diff --git a/docs/logentry/processing-log-lines.md b/docs/logentry/processing-log-lines.md index 55e740cf84e6f..c066c6dc59499 100644 --- a/docs/logentry/processing-log-lines.md +++ b/docs/logentry/processing-log-lines.md @@ -116,6 +116,10 @@ Extracting data (for use by other stages) * [regex](#regex) - use regex to extract data * [json](#json) - parse a JSON log and extract data +Modifying extracted data + + * [template](#template) - use Go templates to modify extracted data + Filtering stages * [match](#match) - apply selectors to conditionally run stages based on labels @@ -208,6 +212,76 @@ Would create the following `extracted` map: ``` [Example in unit test](../../pkg/logentry/stages/json_test.go) +#### template + +A template stage lets you manipulate the values in the `extracted` data map using [Go's template package](https://golang.org/pkg/text/template/). This can be useful if you want to manipulate data extracted by regex or json stages before setting label values. Maybe to replace all spaces with underscores or make everything lowercase, or append some values to the extracted data. + +You can set values in the extracted map for keys that did not previously exist. + +```yaml +- template: + source: ① + template: ② +``` + +① `source` is **required** and is the key to the value in the `extracted` data map you wish to modify, this key does __not__ have to be present and will be added if missing. +② `template` is **required** and is a [Go template string](https://golang.org/pkg/text/template/) + +The value of the extracted data map is accessed by using `.Value` in your template + +In addition to normal template syntax, several functions have also been mapped to use directly or in a pipe configuration: + +```go +"ToLower": strings.ToLower, +"ToUpper": strings.ToUpper, +"Replace": strings.Replace, +"Trim": strings.Trim, +"TrimLeft": strings.TrimLeft, +"TrimRight": strings.TrimRight, +"TrimPrefix": strings.TrimPrefix, +"TrimSuffix": strings.TrimSuffix, +"TrimSpace": strings.TrimSpace, +``` + +##### Example + +```yaml +- template: + source: app + template: '{{ .Value }}_some_suffix' +``` + +This would take the value of the `app` key in the `extracted` data map and append `_some_suffix` to it. For example, if `app=loki` the new value for `app` in the map would be `loki_some_suffix` + +```yaml +- template: + source: app + template: '{{ ToLower .Value }}' +``` + +This would take the value of `app` from `extracted` data and lowercase all the letters. If `app=LOKI` the new value for `app` would be `loki`. + +The template syntax passes paramters to functions using space delimiters, functions only taking a single argument can also use the pipe syntax: + +```yaml +- template: + source: app + template: '{{ .Value | ToLower }}' +``` + +A more complicated function example: + +```yaml +- template: + source: app + template: '{{ Replace .Value "loki" "bloki" 1 }}' +``` + +The arguments here as described for the [Replace function](https://golang.org/pkg/strings/#Replace), in this example we are saying to Replace in the string `.Value` (which is our extracted value for the `app` key) the occurrence of the string "loki" with the string "bloki" exactly 1 time. + +[More examples in unit test](../../pkg/logentry/stages/template_test.go) + + ### match A match stage will take the provided label `selector` and determine if a group of provided Stages will be executed or not based on labels diff --git a/pkg/logentry/stages/labels_test.go b/pkg/logentry/stages/labels_test.go index af594d73bde67..944bd7665c382 100644 --- a/pkg/logentry/stages/labels_test.go +++ b/pkg/logentry/stages/labels_test.go @@ -140,6 +140,14 @@ func TestLabelStage_Process(t *testing.T) { "testLabel": "testValue", }, }, + "empty_extracted_data": { + LabelsConfig{ + "testLabel": &sourceName, + }, + map[string]interface{}{}, + model.LabelSet{}, + model.LabelSet{}, + }, } for name, test := range tests { test := test diff --git a/pkg/logentry/stages/match_test.go b/pkg/logentry/stages/match_test.go index 68e50ffb081da..9849d064309fa 100644 --- a/pkg/logentry/stages/match_test.go +++ b/pkg/logentry/stages/match_test.go @@ -119,6 +119,7 @@ func TestMatcher(t *testing.T) { {"{foo=\"bar\",bar!=\"test\"}", map[string]string{"foo": "bar", "bar": "test"}, false, false}, {"{foo=\"bar\",bar=~\"te.*\"}", map[string]string{"foo": "bar", "bar": "test"}, true, false}, {"{foo=\"bar\",bar!~\"te.*\"}", map[string]string{"foo": "bar", "bar": "test"}, false, false}, + {"{foo=\"\"}", map[string]string{}, true, false}, } for _, tt := range tests { diff --git a/pkg/logentry/stages/stage.go b/pkg/logentry/stages/stage.go index 44db7f06628d2..c11fba9f3a620 100644 --- a/pkg/logentry/stages/stage.go +++ b/pkg/logentry/stages/stage.go @@ -19,6 +19,7 @@ const ( StageTypeDocker = "docker" StageTypeCRI = "cri" StageTypeMatch = "match" + StageTypeTemplate = "template" ) // Stage takes an existing set of labels, timestamp and log entry and returns either a possibly mutated @@ -86,6 +87,11 @@ func New(logger log.Logger, jobName *string, stageType string, if err != nil { return nil, err } + case StageTypeTemplate: + s, err = newTemplateStage(logger, cfg) + if err != nil { + return nil, err + } default: return nil, errors.Errorf("Unknown stage type: %s", stageType) } diff --git a/pkg/logentry/stages/template.go b/pkg/logentry/stages/template.go new file mode 100644 index 0000000000000..316ce1ff8414b --- /dev/null +++ b/pkg/logentry/stages/template.go @@ -0,0 +1,125 @@ +package stages + +import ( + "bytes" + "errors" + "reflect" + "strings" + "text/template" + "time" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/mitchellh/mapstructure" + "github.com/prometheus/common/model" +) + +// Config Errors +const ( + ErrEmptyTemplateStageConfig = "template stage config cannot be empty" + ErrTemplateSourceRequired = "template source value is required" +) + +var ( + functionMap = template.FuncMap{ + "ToLower": strings.ToLower, + "ToUpper": strings.ToUpper, + "Replace": strings.Replace, + "Trim": strings.Trim, + "TrimLeft": strings.TrimLeft, + "TrimRight": strings.TrimRight, + "TrimPrefix": strings.TrimPrefix, + "TrimSuffix": strings.TrimSuffix, + "TrimSpace": strings.TrimSpace, + } +) + +// TemplateConfig configures template value extraction +type TemplateConfig struct { + Source string `mapstructure:"source"` + Template string `mapstructure:"template"` +} + +// validateTemplateConfig validates the templateStage config +func validateTemplateConfig(cfg *TemplateConfig) (*template.Template, error) { + if cfg == nil { + return nil, errors.New(ErrEmptyTemplateStageConfig) + } + if cfg.Source == "" { + return nil, errors.New(ErrTemplateSourceRequired) + } + + return template.New("pipeline_template").Funcs(functionMap).Parse(cfg.Template) +} + +// newTemplateStage creates a new templateStage +func newTemplateStage(logger log.Logger, config interface{}) (*templateStage, error) { + cfg := &TemplateConfig{} + err := mapstructure.Decode(config, cfg) + if err != nil { + return nil, err + } + t, err := validateTemplateConfig(cfg) + if err != nil { + return nil, err + } + + return &templateStage{ + cfgs: cfg, + logger: logger, + template: t, + }, nil +} + +type templateData struct { + Value string +} + +// templateStage will mutate the incoming entry and set it from extracted data +type templateStage struct { + cfgs *TemplateConfig + logger log.Logger + template *template.Template +} + +// Process implements Stage +func (o *templateStage) Process(labels model.LabelSet, extracted map[string]interface{}, t *time.Time, entry *string) { + if o.cfgs == nil { + return + } + if v, ok := extracted[o.cfgs.Source]; ok { + s, err := getString(v) + if err != nil { + level.Debug(o.logger).Log("msg", "extracted template could not be converted to a string", "err", err, "type", reflect.TypeOf(v).String()) + return + } + td := templateData{s} + buf := &bytes.Buffer{} + err = o.template.Execute(buf, td) + if err != nil { + level.Debug(o.logger).Log("msg", "failed to execute template on extracted value", "err", err, "value", v) + return + } + st := buf.String() + // If the template evaluates to an empty string, remove the key from the map + if st == "" { + delete(extracted, o.cfgs.Source) + } else { + extracted[o.cfgs.Source] = st + } + + } else { + td := templateData{} + buf := &bytes.Buffer{} + err := o.template.Execute(buf, td) + if err != nil { + level.Debug(o.logger).Log("msg", "failed to execute template on extracted value", "err", err, "value", v) + return + } + st := buf.String() + // Do not set extracted data with empty values + if st != "" { + extracted[o.cfgs.Source] = st + } + } +} diff --git a/pkg/logentry/stages/template_test.go b/pkg/logentry/stages/template_test.go new file mode 100644 index 0000000000000..f893a1d3e528c --- /dev/null +++ b/pkg/logentry/stages/template_test.go @@ -0,0 +1,205 @@ +package stages + +import ( + "errors" + "testing" + "time" + + "github.com/cortexproject/cortex/pkg/util" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/assert" +) + +var testTemplateYaml = ` +pipeline_stages: +- json: + expressions: + app: app + level: level +- template: + source: app + template: '{{ .Value | ToUpper }} doki' +- template: + source: level + template: '{{ if eq .Value "WARN" }}{{ Replace .Value "WARN" "OK" -1 }}{{ else }}{{ .Value }}{{ end }}' +- template: + source: notexist + template: "TEST" +- labels: + app: '' + level: '' + type: notexist +` + +var testTemplateLogLine = ` +{ + "time":"2012-11-01T22:08:41+00:00", + "app":"loki", + "component": ["parser","type"], + "level" : "WARN", + "nested" : {"child":"value"}, + "message" : "this is a log line" +} +` + +func TestPipeline_Template(t *testing.T) { + pl, err := NewPipeline(util.Logger, loadConfig(testTemplateYaml), nil, prometheus.DefaultRegisterer) + if err != nil { + t.Fatal(err) + } + lbls := model.LabelSet{} + expectedLbls := model.LabelSet{ + "app": "LOKI doki", + "level": "OK", + "type": "TEST", + } + ts := time.Now() + entry := testTemplateLogLine + extracted := map[string]interface{}{} + pl.Process(lbls, extracted, &ts, &entry) + assert.Equal(t, expectedLbls, lbls) +} + +func TestTemplateValidation(t *testing.T) { + tests := map[string]struct { + config *TemplateConfig + err error + }{ + "missing config": { + config: nil, + err: errors.New(ErrEmptyTemplateStageConfig), + }, + "missing source": { + config: &TemplateConfig{ + Source: "", + }, + err: errors.New(ErrTemplateSourceRequired), + }, + } + for name, test := range tests { + test := test + t.Run(name, func(t *testing.T) { + t.Parallel() + _, err := validateTemplateConfig(test.config) + if (err != nil) != (test.err != nil) { + t.Errorf("validateTemplateConfig() expected error = %v, actual error = %v", test.err, err) + return + } + if (err != nil) && (err.Error() != test.err.Error()) { + t.Errorf("validateTemplateConfig() expected error = %v, actual error = %v", test.err, err) + return + } + }) + } +} + +func TestTemplateStage_Process(t *testing.T) { + tests := map[string]struct { + config TemplateConfig + extracted map[string]interface{} + expectedExtracted map[string]interface{} + }{ + "simple template": { + TemplateConfig{ + Source: "some", + Template: "{{ .Value }} appended", + }, + map[string]interface{}{ + "some": "value", + }, + map[string]interface{}{ + "some": "value appended", + }, + }, + "add missing": { + TemplateConfig{ + Source: "missing", + Template: "newval", + }, + map[string]interface{}{ + "notmissing": "value", + }, + map[string]interface{}{ + "notmissing": "value", + "missing": "newval", + }, + }, + "ToLower": { + TemplateConfig{ + Source: "testval", + Template: "{{ .Value | ToLower }}", + }, + map[string]interface{}{ + "testval": "Value", + }, + map[string]interface{}{ + "testval": "value", + }, + }, + "ToLowerEmptyValue": { + TemplateConfig{ + Source: "testval", + Template: "{{ .Value | ToLower }}", + }, + map[string]interface{}{}, + map[string]interface{}{}, + }, + "ReplaceAllToLower": { + TemplateConfig{ + Source: "testval", + Template: "{{ Replace .Value \" \" \"_\" -1 | ToLower }}", + }, + map[string]interface{}{ + "testval": "Some Silly Value With Lots Of Spaces", + }, + map[string]interface{}{ + "testval": "some_silly_value_with_lots_of_spaces", + }, + }, + "Trim": { + TemplateConfig{ + Source: "testval", + Template: "{{ Trim .Value \"!\" }}", + }, + map[string]interface{}{ + "testval": "!!!!!WOOOOO!!!!!", + }, + map[string]interface{}{ + "testval": "WOOOOO", + }, + }, + "Remove label empty value": { + TemplateConfig{ + Source: "testval", + Template: "", + }, + map[string]interface{}{ + "testval": "WOOOOO", + }, + map[string]interface{}{}, + }, + "Don't add label with empty value": { + TemplateConfig{ + Source: "testval", + Template: "", + }, + map[string]interface{}{}, + map[string]interface{}{}, + }, + } + for name, test := range tests { + test := test + t.Run(name, func(t *testing.T) { + t.Parallel() + st, err := newTemplateStage(util.Logger, test.config) + if err != nil { + t.Fatal(err) + } + lbls := model.LabelSet{} + entry := "not important for this test" + st.Process(lbls, test.extracted, nil, &entry) + assert.Equal(t, test.expectedExtracted, test.extracted) + }) + } +}