Support entire extracted value map in template pipeline stage (#1666)

Refactor template stage processing. This sets all extracted key available, plus if the source is available adds the extra key `Value` for backward compatibility. Signed-off-by: Cyril Tovena <[email protected]> Signed-off-by: Aditya C S <[email protected]> Co-authored-by: Cyril Tovena <[email protected]>
grafana · Feb 14, 2020 · 460bb5b · 460bb5b
1 parent 32d4f6c
commit 460bb5b
Show file tree

Hide file tree

Showing 3 changed files with 98 additions and 36 deletions.
diff --git a/docs/clients/promtail/stages/template.md b/docs/clients/promtail/stages/template.md
@@ -6,9 +6,10 @@ syntax](https://golang.org/pkg/text/template/).
 
 The `template` stage is primarily useful for manipulating data from other stages
 before setting them as labels, such as to replace spaces with underscores or
-converting an uppercase string into a lowercase one.
+converting an uppercase string into a lowercase one. `template` can also be used 
+to construct messages with multiple keys.
 
-The template stage can also create new keys in the extracted map.
+The template stage can also create new keys in the extracted map. 
 
 ## Schema
 
@@ -58,6 +59,32 @@ converts its value to be all lowercase. For example, if the extracted map
 contained `app` with a value of `LOKI`, this pipeline would change its value to
 `loki`.
 
+```yaml
+- template:
+    source: output_msg
+    template: '{{ .level }} for app {{ ToUpper .app }}'
+```
+
+This pipeline takes the current value of `level` and `app` from the extracted map and
+a new key `output_msg` will be added to extracted map with evaluated template. 
+
+For example, if the extracted map contained `app` with a value of `loki`, this pipeline would change its value to `LOKI`. Assuming value of `level` is `warn`. A new key `output_msg` will be added to extracted map with value `warn for app LOKI`. 
+
+Any previously extracted keys can be used in `template`. All extracted keys are available for `template` to expand. 
+
+```yaml
+- template:
+    source: app
+    template: '{{ .level }} for app {{ ToUpper .Value }} in module {{.module}}'
+```
+
+This pipeline takes the current value of `level`, `app` and `module` from the extracted map and
+converts value of `app` to the evaluated template. 
+
+For example, if the extracted map contained `app` with a value of `loki`, this pipeline would change its value to `LOKI`. Assuming value of `level` is `warn` and value of `module` is `test`. Pipeline will change the value of `app` to `warn for app LOKI in module test`. 
+
+Any previously extracted keys can be used in `template`. All extracted keys are available for `template` to expand. Also, if source is available it can be referred as `.Value` in `template`. Here, `app` is provided as `source`. So, it can be referred as `.Value` in `template`. 
+
 ```yaml
 - template:
     source: app

diff --git a/pkg/logentry/stages/template.go b/pkg/logentry/stages/template.go
@@ -71,10 +71,6 @@ func newTemplateStage(logger log.Logger, config interface{}) (*templateStage, er
 	}, nil
 }
 
-type templateData struct {
-	Value string
-}
-
 // templateStage will mutate the incoming entry and set it from extracted data
 type templateStage struct {
 	cfgs     *TemplateConfig
@@ -87,47 +83,37 @@ func (o *templateStage) Process(labels model.LabelSet, extracted map[string]inte
 	if o.cfgs == nil {
 		return
 	}
-	if v, ok := extracted[o.cfgs.Source]; ok {
+	td := make(map[string]interface{})
+	for k, v := range extracted {
 		s, err := getString(v)
 		if err != nil {
 			if Debug {
 				level.Debug(o.logger).Log("msg", "extracted template could not be converted to a string", "err", err, "type", reflect.TypeOf(v).String())
 			}
 			return
 		}
-		td := templateData{s}
-		buf := &bytes.Buffer{}
-		err = o.template.Execute(buf, td)
-		if err != nil {
-			if Debug {
-				level.Debug(o.logger).Log("msg", "failed to execute template on extracted value", "err", err, "value", v)
-			}
-			return
-		}
-		st := buf.String()
-		// If the template evaluates to an empty string, remove the key from the map
-		if st == "" {
-			delete(extracted, o.cfgs.Source)
-		} else {
-			extracted[o.cfgs.Source] = st
+		td[k] = s
+		if k == o.cfgs.Source {
+			td["Value"] = s
 		}
+	}
 
-	} else {
-		td := templateData{}
-		buf := &bytes.Buffer{}
-		err := o.template.Execute(buf, td)
-		if err != nil {
-			if Debug {
-				level.Debug(o.logger).Log("msg", "failed to execute template on extracted value", "err", err, "value", v)
-			}
-			return
-		}
-		st := buf.String()
-		// Do not set extracted data with empty values
-		if st != "" {
-			extracted[o.cfgs.Source] = st
+	buf := &bytes.Buffer{}
+	err := o.template.Execute(buf, td)
+	if err != nil {
+		if Debug {
+			level.Debug(o.logger).Log("msg", "failed to execute template on extracted value", "err", err)
 		}
+		return
+	}
+	st := buf.String()
+	// If the template evaluates to an empty string, remove the key from the map
+	if st == "" {
+		delete(extracted, o.cfgs.Source)
+	} else {
+		extracted[o.cfgs.Source] = st
 	}
+
 }
 
 // Name implements Stage

diff --git a/pkg/logentry/stages/template_test.go b/pkg/logentry/stages/template_test.go
@@ -125,6 +125,55 @@ func TestTemplateStage_Process(t *testing.T) {
 				"missing":    "newval",
 			},
 		},
+		"template with multiple keys": {
+			TemplateConfig{
+				Source:   "message",
+				Template: "{{.Value}} in module {{.module}}",
+			},
+			map[string]interface{}{
+				"level":   "warn",
+				"app":     "loki",
+				"message": "warn for app loki",
+				"module":  "test",
+			},
+			map[string]interface{}{
+				"level":   "warn",
+				"app":     "loki",
+				"module":  "test",
+				"message": "warn for app loki in module test",
+			},
+		},
+		"template with multiple keys with missing source": {
+			TemplateConfig{
+				Source:   "missing",
+				Template: "{{ .level }} for app {{ .app | ToUpper }}",
+			},
+			map[string]interface{}{
+				"level": "warn",
+				"app":   "loki",
+			},
+			map[string]interface{}{
+				"level":   "warn",
+				"app":     "loki",
+				"missing": "warn for app LOKI",
+			},
+		},
+		"template with multiple keys with missing key": {
+			TemplateConfig{
+				Source:   "message",
+				Template: "{{.Value}} in module {{.module}}",
+			},
+			map[string]interface{}{
+				"level":   "warn",
+				"app":     "loki",
+				"message": "warn for app loki",
+			},
+			map[string]interface{}{
+				"level":   "warn",
+				"app":     "loki",
+				"message": "warn for app loki in module <no value>",
+			},
+		},
 		"ToLower": {
 			TemplateConfig{
 				Source:   "testval",