From 9d9e83a95716f28762b60a80e2989cae87dc6430 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 30 Aug 2018 21:58:36 -0700 Subject: [PATCH 1/3] Use processor conventions from #4616 --- plugins/processors/strings/README.md | 61 ++- plugins/processors/strings/strings.go | 225 +++++++--- plugins/processors/strings/strings_test.go | 479 ++++++++++++--------- 3 files changed, 475 insertions(+), 290 deletions(-) diff --git a/plugins/processors/strings/README.md b/plugins/processors/strings/README.md index c983d663d906a..fe825436e2b43 100644 --- a/plugins/processors/strings/README.md +++ b/plugins/processors/strings/README.md @@ -1,42 +1,61 @@ # Strings Processor Plugin -The `strings` plugin maps certain go string functions onto tag and field values. If `result_key` parameter is present, it can produce new tags and fields from existing ones. +The `strings` plugin maps certain go string functions onto measurement, tag, and field values. Values can be modified in place or stored in another key. -Implemented functions are: Lowercase, Uppercase, Trim, TrimPrefix, TrimSuffix, TrimRight, TrimLeft +Implemented functions are: +- lowercase +- uppercase +- trim +- trim_prefix +- trim_suffix +- trim_right +- trim_left Please note that in this implementation these are processed in the order that they appear above. -Specify the `tag` or `field` that you want processed in each section and optionally a `result_key` if you want the result stored in a new tag or field. You can specify lots of transformations on data with a single strings processor. Certain functions require an `argument` field to specify how they should process their throughput. - -Functions that require an `argument` are: Trim, TrimPrefix, TrimSuffix, TrimRight, TrimLeft +Specify the `measurement`, `tag` or `field` that you want processed in each section and optionally a `dest` if you want the result stored in a new tag or field. You can specify lots of transformations on data with a single strings processor. ### Configuration: ```toml [[processors.strings]] - namepass = ["uri_stem"] + # [[processors.strings.uppercase]] + # tag = "method" + + # [[processors.strings.lowercase]] + # field = "uri_stem" + # dest = "uri_stem_normalised" + + ## Convert a tag value to lowercase + # [[processors.strings.trim]] + # field = "message" - # Tag and field conversions defined in a separate sub-tables - [[processors.strings.lowercase]] - ## Tag to change - tag = "uri_stem" + # [[processors.strings.trim_left]] + # field = "message" + # cutset = "\t" - [[processors.strings.lowercase]] - ## Multiple tags or fields may be defined - tag = "method" + # [[processors.strings.trim_right]] + # field = "message" + # cutset = "\r\n" - [[processors.strings.uppercase]] - key = "cs-host" - result_key = "cs-host_normalised" + # [[processors.strings.trim_prefix]] + # field = "my_value" + # prefix = "my_" - [[processors.strings.trimprefix]] - tag = "uri_stem" - argument = "/api/" + # [[processors.strings.trim_suffix]] + # field = "read_count" + # suffix = "_count" ``` -### Tags: +#### Trim, TrimLeft, TrimRight + +The `trim`, `trim_left`, and `trim_right` functions take an optional parameter: `cutset`. This value is a string containing the characters to remove from the value. + +#### TrimPrefix, TrimSuffix + +The `trim_prefix` and `trim_suffix` functions remote the given `prefix` or `suffix` +respectively from the string. -No tags are applied by this processor. ### Example Input: ``` diff --git a/plugins/processors/strings/strings.go b/plugins/processors/strings/strings.go index cd96aafa6fb6c..8e68dbc52355f 100644 --- a/plugins/processors/strings/strings.go +++ b/plugins/processors/strings/strings.go @@ -2,109 +2,194 @@ package strings import ( "strings" + "unicode" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/processors" ) type Strings struct { - Lowercase []converter - Uppercase []converter - Trim []converter - TrimLeft []converter - TrimRight []converter - TrimPrefix []converter - TrimSuffix []converter + Lowercase []converter `toml:"lowercase"` + Uppercase []converter `toml:"uppercase"` + Trim []converter `toml:"trim"` + TrimLeft []converter `toml:"trim_left"` + TrimRight []converter `toml:"trim_right"` + TrimPrefix []converter `toml:"trim_prefix"` + TrimSuffix []converter `toml:"trim_suffix"` + + converters []converter + init bool } +type ConvertFunc func(s string) string + type converter struct { + Field string Tag string - Field string - ResultKey string - Argument string + Measurement string + Dest string + Cutset string + Suffix string + Prefix string + + fn ConvertFunc } const sampleConfig = ` - ## Tag and field conversions defined in a separate sub-tables - + ## Convert a tag value to uppercase # [[processors.strings.uppercase]] # tag = "method" + ## Convert a field value to lowercase and store in a new field # [[processors.strings.lowercase]] # field = "uri_stem" - # result_key = "uri_stem_normalised" + # dest = "uri_stem_normalised" + + ## Trim leading and trailing whitespace using the default cutset + # [[processors.strings.trim]] + # field = "message" + + ## Trim leading characters in cutset + # [[processors.strings.trim_left]] + # field = "message" + # cutset = "\t" + + ## Trim trailing characters in cutset + # [[processors.strings.trim_right]] + # field = "message" + # cutset = "\r\n" + + ## Trim the given prefix from the field + # [[processors.strings.trim_prefix]] + # field = "my_value" + # prefix = "my_" + + ## Trim the given suffix from the field + # [[processors.strings.trim_suffix]] + # field = "read_count" + # suffix = "_count" ` -func (r *Strings) SampleConfig() string { +func (s *Strings) SampleConfig() string { return sampleConfig } -func (r *Strings) Description() string { - return "Transforms tag and field values to lower case" +func (s *Strings) Description() string { + return "Perform string processing on tags, fields, and measurements" +} + +func (c *converter) convertTag(metric telegraf.Metric) { + tv, ok := metric.GetTag(c.Tag) + if !ok { + return + } + + dest := c.Tag + if c.Dest != "" { + dest = c.Dest + } + + metric.AddTag(dest, c.fn(tv)) } -func ApplyFunction( - metric telegraf.Metric, - c converter, - fn func(string) string) { - - if value, ok := metric.Tags()[c.Tag]; ok { - metric.AddTag( - getKey(c), - fn(value), - ) - } else if value, ok := metric.Fields()[c.Field]; ok { - switch value := value.(type) { - case string: - metric.AddField( - getKey(c), - fn(value), - ) - } - } +func (c *converter) convertField(metric telegraf.Metric) { + fv, ok := metric.GetField(c.Field) + if !ok { + return + } + + dest := c.Field + if c.Dest != "" { + dest = c.Dest + } + + if fv, ok := fv.(string); ok { + metric.AddField(dest, c.fn(fv)) + } } -func (r *Strings) Apply(in ...telegraf.Metric) []telegraf.Metric { - for _, metric := range in { - for _, converter := range r.Lowercase { - ApplyFunction(metric, converter, strings.ToLower) +func (c *converter) convertMeasurement(metric telegraf.Metric) { + if metric.Name() != c.Measurement { + return + } + + metric.SetName(c.fn(metric.Name())) +} + +func (c *converter) convert(metric telegraf.Metric) { + if c.Field != "" { + c.convertField(metric) + } + + if c.Tag != "" { + c.convertTag(metric) + } + + if c.Measurement != "" { + c.convertMeasurement(metric) + } +} + +func (s *Strings) initOnce() { + if s.init { + return + } + + s.converters = make([]converter, 0) + for _, c := range s.Lowercase { + c.fn = strings.ToLower + s.converters = append(s.converters, c) + } + for _, c := range s.Uppercase { + c.fn = strings.ToUpper + s.converters = append(s.converters, c) + } + for _, c := range s.Trim { + if c.Cutset != "" { + c.fn = func(s string) string { return strings.Trim(s, c.Cutset) } + } else { + c.fn = func(s string) string { return strings.TrimFunc(s, unicode.IsSpace) } } - for _, converter := range r.Uppercase { - ApplyFunction(metric, converter, strings.ToUpper) + s.converters = append(s.converters, c) + } + for _, c := range s.TrimLeft { + if c.Cutset != "" { + c.fn = func(s string) string { return strings.TrimLeft(s, c.Cutset) } + } else { + c.fn = func(s string) string { return strings.TrimLeftFunc(s, unicode.IsSpace) } + } + s.converters = append(s.converters, c) + } + for _, c := range s.TrimRight { + if c.Cutset != "" { + c.fn = func(s string) string { return strings.TrimRight(s, c.Cutset) } + } else { + c.fn = func(s string) string { return strings.TrimRightFunc(s, unicode.IsSpace) } } - for _, converter := range r.Trim { - ApplyFunction(metric, converter, - func(s string) string { return strings.Trim(s, converter.Argument) }) - } - for _, converter := range r.TrimPrefix { - ApplyFunction(metric, converter, - func(s string) string { return strings.TrimPrefix(s, converter.Argument) }) - } - for _, converter := range r.TrimSuffix{ - ApplyFunction(metric, converter, - func(s string) string { return strings.TrimSuffix(s, converter.Argument) }) - } - for _, converter := range r.TrimRight { - ApplyFunction(metric, converter, - func(s string) string { return strings.TrimRight(s, converter.Argument) }) - } - for _, converter := range r.TrimLeft { - ApplyFunction(metric, converter, - func(s string) string { return strings.TrimLeft(s, converter.Argument) }) - } + s.converters = append(s.converters, c) + } + for _, c := range s.TrimPrefix { + c.fn = func(s string) string { return strings.TrimPrefix(s, c.Prefix) } + s.converters = append(s.converters, c) + } + for _, c := range s.TrimSuffix { + c.fn = func(s string) string { return strings.TrimSuffix(s, c.Suffix) } + s.converters = append(s.converters, c) } - return in + s.init = true } -func getKey(c converter) string { - if c.ResultKey != "" { - return c.ResultKey - } else if c.Field != "" { - return c.Field - } else { - return c.Tag - } +func (s *Strings) Apply(in ...telegraf.Metric) []telegraf.Metric { + s.initOnce() + + for _, metric := range in { + for _, converter := range s.converters { + converter.convert(metric) + } + } + + return in } func init() { diff --git a/plugins/processors/strings/strings_test.go b/plugins/processors/strings/strings_test.go index 8509181d097b2..4d02cb64dc03f 100644 --- a/plugins/processors/strings/strings_test.go +++ b/plugins/processors/strings/strings_test.go @@ -1,22 +1,25 @@ package strings import ( + "fmt" "testing" "time" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/metric" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func newM1() telegraf.Metric { m1, _ := metric.New("IIS_log", map[string]string{ "verb": "GET", - "s-computername": "MIXEDCASE_hostname", + "s-computername": "MIXEDCASE_hostname", }, map[string]interface{}{ - "request": "/mixed/CASE/paTH/?from=-1D&to=now", + "request": "/mixed/CASE/paTH/?from=-1D&to=now", + "whitespace": " whitespace\t", }, time.Now(), ) @@ -28,11 +31,11 @@ func newM2() telegraf.Metric { map[string]string{ "verb": "GET", "resp_code": "200", - "s-computername": "MIXEDCASE_hostname", + "s-computername": "MIXEDCASE_hostname", }, map[string]interface{}{ - "request": "/mixed/CASE/paTH/?from=-1D&to=now", - "cs-host": "AAAbbb", + "request": "/mixed/CASE/paTH/?from=-1D&to=now", + "cs-host": "AAAbbb", "ignore_number": int64(200), "ignore_bool": true, }, @@ -43,275 +46,353 @@ func newM2() telegraf.Metric { func TestFieldConversions(t *testing.T) { tests := []struct { - message string - lowercase converter - uppercase converter - trim converter - trimleft converter - trimright converter - trimprefix converter - trimsuffix converter - expectedFields map[string]interface{} + name string + plugin *Strings + check func(t *testing.T, actual telegraf.Metric) }{ { - message: "Should change existing field to lowercase", - lowercase: converter{ - Field: "request", + name: "Should change existing field to lowercase", + plugin: &Strings{ + Lowercase: []converter{ + converter{ + Field: "request", + }, + }, }, - expectedFields: map[string]interface{}{ - "request": "/mixed/case/path/?from=-1d&to=now", + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("request") + require.True(t, ok) + require.Equal(t, "/mixed/case/path/?from=-1d&to=now", fv) }, }, { - message: "Should change existing field to uppercase", - uppercase: converter{ - Field: "request", + name: "Should change existing field to uppercase", + plugin: &Strings{ + Uppercase: []converter{ + converter{ + Field: "request", + }, + }, }, - expectedFields: map[string]interface{}{ - "request": "/MIXED/CASE/PATH/?FROM=-1D&TO=NOW", + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("request") + require.True(t, ok) + require.Equal(t, "/MIXED/CASE/PATH/?FROM=-1D&TO=NOW", fv) }, }, { - message: "Should add new lowercase field", - lowercase: converter{ - Field: "request", - ResultKey: "lowercase_request", + name: "Should add new lowercase field", + plugin: &Strings{ + Lowercase: []converter{ + converter{ + Field: "request", + Dest: "lowercase_request", + }, + }, }, - expectedFields: map[string]interface{}{ - "request": "/mixed/CASE/paTH/?from=-1D&to=now", - "lowercase_request": "/mixed/case/path/?from=-1d&to=now", + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("request") + require.True(t, ok) + require.Equal(t, "/mixed/CASE/paTH/?from=-1D&to=now", fv) + + fv, ok = actual.GetField("lowercase_request") + require.True(t, ok) + require.Equal(t, "/mixed/case/path/?from=-1d&to=now", fv) + }, + }, + { + name: "Should trim from both sides", + plugin: &Strings{ + Trim: []converter{ + converter{ + Field: "request", + Cutset: "/w", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("request") + require.True(t, ok) + require.Equal(t, "mixed/CASE/paTH/?from=-1D&to=no", fv) + }, + }, + { + name: "Should trim from both sides and make lowercase", + plugin: &Strings{ + Trim: []converter{ + converter{ + Field: "request", + Cutset: "/w", + }, + }, + Lowercase: []converter{ + converter{ + Field: "request", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("request") + require.True(t, ok) + require.Equal(t, "mixed/case/path/?from=-1d&to=no", fv) + }, + }, + { + name: "Should trim from left side", + plugin: &Strings{ + TrimLeft: []converter{ + converter{ + Field: "request", + Cutset: "/w", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("request") + require.True(t, ok) + require.Equal(t, "mixed/CASE/paTH/?from=-1D&to=now", fv) }, }, { - message: "Should trim from both sides", - trim: converter{ - Field: "request", - Argument: "/w", + name: "Should trim from right side", + plugin: &Strings{ + TrimRight: []converter{ + converter{ + Field: "request", + Cutset: "/w", + }, + }, }, - expectedFields: map[string]interface{}{ - "request": "mixed/CASE/paTH/?from=-1D&to=no", + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("request") + require.True(t, ok) + require.Equal(t, "/mixed/CASE/paTH/?from=-1D&to=no", fv) }, }, { - message: "Should trim from both sides and make lowercase", - trim: converter{ - Field: "request", - Argument: "/w", + name: "Should trim prefix '/mixed'", + plugin: &Strings{ + TrimPrefix: []converter{ + converter{ + Field: "request", + Prefix: "/mixed", + }, + }, }, - lowercase: converter{ - Field: "request", - }, - expectedFields: map[string]interface{}{ - "request": "mixed/case/path/?from=-1d&to=no", + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("request") + require.True(t, ok) + require.Equal(t, "/CASE/paTH/?from=-1D&to=now", fv) }, }, { - message: "Should trim from left side", - trimleft: converter{ - Field: "request", - Argument: "/w", + name: "Should trim suffix '-1D&to=now'", + plugin: &Strings{ + TrimSuffix: []converter{ + converter{ + Field: "request", + Suffix: "-1D&to=now", + }, + }, }, - expectedFields: map[string]interface{}{ - "request": "mixed/CASE/paTH/?from=-1D&to=now", + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("request") + require.True(t, ok) + require.Equal(t, "/mixed/CASE/paTH/?from=", fv) }, }, { - message: "Should trim from right side", - trimright: converter{ - Field: "request", - Argument: "/w", + name: "Trim without cutset removes whitespace", + plugin: &Strings{ + Trim: []converter{ + converter{ + Field: "whitespace", + }, + }, }, - expectedFields: map[string]interface{}{ - "request": "/mixed/CASE/paTH/?from=-1D&to=no", + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("whitespace") + require.True(t, ok) + require.Equal(t, "whitespace", fv) }, }, { - message: "Should trim prefix '/mixed'", - trimprefix: converter{ - Field: "request", - Argument: "/mixed", + name: "Trim left without cutset removes whitespace", + plugin: &Strings{ + TrimLeft: []converter{ + converter{ + Field: "whitespace", + }, + }, }, - expectedFields: map[string]interface{}{ - "request": "/CASE/paTH/?from=-1D&to=now", + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("whitespace") + require.True(t, ok) + require.Equal(t, "whitespace\t", fv) }, }, { - message: "Should trim suffix '-1D&to=now'", - trimprefix: converter{ - Field: "request", - Argument: "-1D&to=now", + name: "Trim right without cutset removes whitespace", + plugin: &Strings{ + TrimRight: []converter{ + converter{ + Field: "whitespace", + }, + }, }, - expectedFields: map[string]interface{}{ - "request": "/mixed/CASE/paTH/?from=-1D&to=now", + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("whitespace") + require.True(t, ok) + require.Equal(t, " whitespace", fv) + }, + }, + { + name: "No change if field missing", + plugin: &Strings{ + Lowercase: []converter{ + converter{ + Field: "xyzzy", + Suffix: "-1D&to=now", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("request") + require.True(t, ok) + require.Equal(t, "/mixed/CASE/paTH/?from=-1D&to=now", fv) }, }, } - - for _, test := range tests { - strings := &Strings{} - strings.Lowercase = []converter{ - test.lowercase, - } - strings.Uppercase = []converter{ - test.uppercase, - } - strings.Trim = []converter{ - test.trim, - } - strings.TrimLeft = []converter{ - test.trimleft, - } - strings.TrimRight = []converter{ - test.trimright, - } - strings.TrimPrefix = []converter{ - test.trimprefix, - } - strings.TrimSuffix = []converter{ - test.trimsuffix, - } - - processed := strings.Apply(newM1()) - - expectedTags := map[string]string{ - "verb": "GET", - "s-computername": "MIXEDCASE_hostname", - } - - assert.Equal(t, test.expectedFields, processed[0].Fields(), test.message) - assert.Equal(t, expectedTags, processed[0].Tags(), "Should not change tags") - assert.Equal(t, "IIS_log", processed[0].Name(), "Should not change name") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + metrics := tt.plugin.Apply(newM1()) + require.Len(t, metrics, 1) + tt.check(t, metrics[0]) + }) } } func TestTagConversions(t *testing.T) { tests := []struct { - message string - lowercase converter - uppercase converter - expectedTags map[string]string + name string + plugin *Strings + check func(t *testing.T, actual telegraf.Metric) }{ { - message: "Should change existing tag to lowercase", - lowercase: converter{ - Tag: "s-computername", + name: "Should change existing tag to lowercase", + plugin: &Strings{ + Lowercase: []converter{ + converter{ + Tag: "s-computername", + }, + }, }, - expectedTags: map[string]string{ - "verb": "GET", - "s-computername": "mixedcase_hostname", + check: func(t *testing.T, actual telegraf.Metric) { + tv, ok := actual.GetTag("verb") + require.True(t, ok) + require.Equal(t, "GET", tv) + + tv, ok = actual.GetTag("s-computername") + require.True(t, ok) + require.Equal(t, "mixedcase_hostname", tv) }, }, { - message: "Should add new lowercase tag", - lowercase: converter{ - Tag: "s-computername", - ResultKey: "s-computername_lowercase", + name: "Should add new lowercase tag", + plugin: &Strings{ + Lowercase: []converter{ + converter{ + Tag: "s-computername", + Dest: "s-computername_lowercase", + }, + }, }, - expectedTags: map[string]string{ - "verb": "GET", - "s-computername": "MIXEDCASE_hostname", - "s-computername_lowercase": "mixedcase_hostname", + check: func(t *testing.T, actual telegraf.Metric) { + tv, ok := actual.GetTag("verb") + require.True(t, ok) + require.Equal(t, "GET", tv) + + tv, ok = actual.GetTag("s-computername") + require.True(t, ok) + require.Equal(t, "MIXEDCASE_hostname", tv) + + tv, ok = actual.GetTag("s-computername_lowercase") + require.True(t, ok) + require.Equal(t, "mixedcase_hostname", tv) }, }, { - message: "Should add new uppercase tag", - uppercase: converter{ - Tag: "s-computername", - ResultKey: "s-computername_uppercase", + name: "Should add new uppercase tag", + plugin: &Strings{ + Uppercase: []converter{ + converter{ + Tag: "s-computername", + Dest: "s-computername_uppercase", + }, + }, }, - expectedTags: map[string]string{ - "verb": "GET", - "s-computername": "MIXEDCASE_hostname", - "s-computername_uppercase": "MIXEDCASE_HOSTNAME", + check: func(t *testing.T, actual telegraf.Metric) { + tv, ok := actual.GetTag("verb") + require.True(t, ok) + require.Equal(t, "GET", tv) + + tv, ok = actual.GetTag("s-computername") + require.True(t, ok) + require.Equal(t, "MIXEDCASE_hostname", tv) + + tv, ok = actual.GetTag("s-computername_uppercase") + require.True(t, ok) + require.Equal(t, "MIXEDCASE_HOSTNAME", tv) }, }, } - for _, test := range tests { - strings := &Strings{} - strings.Lowercase = []converter{ - test.lowercase, - } - strings.Uppercase = []converter{ - test.uppercase, - } - - processed := strings.Apply(newM1()) - - expectedFields := map[string]interface{}{ - "request": "/mixed/CASE/paTH/?from=-1D&to=now", - } - - assert.Equal(t, expectedFields, processed[0].Fields(), test.message, "Should not change fields") - assert.Equal(t, test.expectedTags, processed[0].Tags(), test.message) - assert.Equal(t, "IIS_log", processed[0].Name(), "Should not change name") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + metrics := tt.plugin.Apply(newM1()) + require.Len(t, metrics, 1) + tt.check(t, metrics[0]) + }) } } func TestMultipleConversions(t *testing.T) { - strings := &Strings{} - strings.Lowercase = []converter{ - { - Tag: "s-computername", - }, - { - Field: "request", + fmt.Println(2) + plugin := &Strings{ + Lowercase: []converter{ + converter{ + Tag: "s-computername", + }, + converter{ + Field: "request", + }, + converter{ + Field: "cs-host", + Dest: "cs-host_lowercase", + }, }, - { - Field: "cs-host", - ResultKey: "cs-host_lowercase", + Uppercase: []converter{ + converter{ + Tag: "verb", + }, }, } - strings.Uppercase = []converter{ - { - Tag: "verb", - }, - } - processed := strings.Apply(newM2()) + processed := plugin.Apply(newM2()) expectedFields := map[string]interface{}{ - "request": "/mixed/case/path/?from=-1d&to=now", + "request": "/mixed/case/path/?from=-1d&to=now", "ignore_number": int64(200), "ignore_bool": true, - "cs-host": "AAAbbb", - "cs-host_lowercase": "aaabbb", + "cs-host": "AAAbbb", + "cs-host_lowercase": "aaabbb", } expectedTags := map[string]string{ "verb": "GET", - "resp_code": "200", - "s-computername": "mixedcase_hostname", + "resp_code": "200", + "s-computername": "mixedcase_hostname", } assert.Equal(t, expectedFields, processed[0].Fields()) assert.Equal(t, expectedTags, processed[0].Tags()) } - -func TestNoKey(t *testing.T) { - tests := []struct { - message string - converter converter - expectedFields map[string]interface{} - }{ - { - message: "Should not change anything if there is no field with given key", - converter: converter{ - Field: "not_exists", - }, - expectedFields: map[string]interface{}{ - "request": "/mixed/CASE/paTH/?from=-1D&to=now", - }, - }, - } - - for _, test := range tests { - strings := &Strings{} - strings.Lowercase = []converter{ - test.converter, - } - - processed := strings.Apply(newM1()) - - assert.Equal(t, test.expectedFields, processed[0].Fields(), test.message) - } -} From b8fadbaac59c0e53cd028776cbca856000b5c3ee Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 30 Aug 2018 22:48:54 -0700 Subject: [PATCH 2/3] Improve readme example and add test for it --- plugins/processors/strings/README.md | 24 +++++- plugins/processors/strings/strings_test.go | 89 +++++++++++++++++++++- 2 files changed, 107 insertions(+), 6 deletions(-) diff --git a/plugins/processors/strings/README.md b/plugins/processors/strings/README.md index fe825436e2b43..f1e7361fea4d2 100644 --- a/plugins/processors/strings/README.md +++ b/plugins/processors/strings/README.md @@ -6,10 +6,10 @@ Implemented functions are: - lowercase - uppercase - trim +- trim_left +- trim_right - trim_prefix - trim_suffix -- trim_right -- trim_left Please note that in this implementation these are processed in the order that they appear above. @@ -56,12 +56,28 @@ The `trim`, `trim_left`, and `trim_right` functions take an optional parameter: The `trim_prefix` and `trim_suffix` functions remote the given `prefix` or `suffix` respectively from the string. +### Example +**Config** +```toml +[[processors.strings]] + [[processors.strings.lowercase]] + field = "uri-stem" + + [[processors.strings.trim_prefix]] + field = "uri_stem" + prefix = "/api/" -### Example Input: + [[processors.strings.uppercase]] + field = "cs-host" + dest = "cs-host_normalised" +``` + +**Input** ``` iis_log,method=get,uri_stem=/API/HealthCheck cs-host="MIXEDCASE_host",referrer="-",ident="-",http_version=1.1,agent="UserAgent",resp_bytes=270i 1519652321000000000 ``` -### Example Output: + +**Output** ``` iis_log,method=get,uri_stem=healthcheck cs-host="MIXEDCASE_host",cs-host_normalised="MIXEDCASE_HOST",referrer="-",ident="-",http_version=1.1,agent="UserAgent",resp_bytes=270i 1519652321000000000 ``` diff --git a/plugins/processors/strings/strings_test.go b/plugins/processors/strings/strings_test.go index 4d02cb64dc03f..2097ac5a836d9 100644 --- a/plugins/processors/strings/strings_test.go +++ b/plugins/processors/strings/strings_test.go @@ -1,7 +1,6 @@ package strings import ( - "fmt" "testing" "time" @@ -356,8 +355,37 @@ func TestTagConversions(t *testing.T) { } } +func TestMeasurementConversions(t *testing.T) { + tests := []struct { + name string + plugin *Strings + check func(t *testing.T, actual telegraf.Metric) + }{ + { + name: "lowercase measurement", + plugin: &Strings{ + Lowercase: []converter{ + converter{ + Measurement: "IIS_log", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + name := actual.Name() + require.Equal(t, "iis_log", name) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + metrics := tt.plugin.Apply(newM1()) + require.Len(t, metrics, 1) + tt.check(t, metrics[0]) + }) + } +} + func TestMultipleConversions(t *testing.T) { - fmt.Println(2) plugin := &Strings{ Lowercase: []converter{ converter{ @@ -396,3 +424,60 @@ func TestMultipleConversions(t *testing.T) { assert.Equal(t, expectedFields, processed[0].Fields()) assert.Equal(t, expectedTags, processed[0].Tags()) } + +func TestReadmeExample(t *testing.T) { + plugin := &Strings{ + Lowercase: []converter{ + converter{ + Tag: "uri_stem", + }, + }, + TrimPrefix: []converter{ + converter{ + Tag: "uri_stem", + Prefix: "/api/", + }, + }, + Uppercase: []converter{ + converter{ + Field: "cs-host", + Dest: "cs-host_normalised", + }, + }, + } + + m, _ := metric.New("iis_log", + map[string]string{ + "verb": "get", + "uri_stem": "/API/HealthCheck", + }, + map[string]interface{}{ + "cs-host": "MIXEDCASE_host", + "referrer": "-", + "ident": "-", + "http_version": "1.1", + "agent": "UserAgent", + "resp_bytes": int64(270), + }, + time.Now(), + ) + + processed := plugin.Apply(m) + + expectedTags := map[string]string{ + "verb": "get", + "uri_stem": "healthcheck", + } + expectedFields := map[string]interface{}{ + "cs-host": "MIXEDCASE_host", + "cs-host_normalised": "MIXEDCASE_HOST", + "referrer": "-", + "ident": "-", + "http_version": "1.1", + "agent": "UserAgent", + "resp_bytes": int64(270), + } + + assert.Equal(t, expectedFields, processed[0].Fields()) + assert.Equal(t, expectedTags, processed[0].Tags()) +} From c67c5344ea9f7e796b5e6cd8383413f3fd3f4539 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Thu, 30 Aug 2018 23:04:30 -0700 Subject: [PATCH 3/3] Fix go fmt issue --- plugins/processors/all/all.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/processors/all/all.go b/plugins/processors/all/all.go index e193e1f41129a..e95ff293abd6d 100644 --- a/plugins/processors/all/all.go +++ b/plugins/processors/all/all.go @@ -6,6 +6,6 @@ import ( _ "github.com/influxdata/telegraf/plugins/processors/override" _ "github.com/influxdata/telegraf/plugins/processors/printer" _ "github.com/influxdata/telegraf/plugins/processors/regex" - _ "github.com/influxdata/telegraf/plugins/processors/topk" _ "github.com/influxdata/telegraf/plugins/processors/strings" + _ "github.com/influxdata/telegraf/plugins/processors/topk" )