From 7ddcb1e47d92fbd91d347c4600045972f461033d Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Thu, 6 Feb 2020 18:55:53 +0100 Subject: [PATCH] Elasticsearch index must be lowercase (#16081) * Index names must be lowercase When indexing into Elasticsearch index names must always be lowercase. If the index or indices setting are configured to produce non-lowercase strings (e.g. by extracting part of the index name from the event contents), we need to normalize them to be lowercase. This change ensure that index names are always converted to lowercase. Static strings are converted to lowercase upfront, while dynamic strings will be post-processed. * update kafka/redis/LS output to guarantee lowercase index * add godoc --- CHANGELOG.next.asciidoc | 2 +- libbeat/outputs/kafka/client.go | 2 +- libbeat/outputs/logstash/config.go | 3 +- libbeat/outputs/logstash/enc.go | 3 + .../logstash/logstash_integration_test.go | 3 +- libbeat/outputs/outil/select.go | 135 ++++++----- libbeat/outputs/outil/select_test.go | 222 +++++++++--------- libbeat/outputs/redis/client.go | 3 +- 8 files changed, 208 insertions(+), 165 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index dcefdc09d7e1..a2e4c2ad1bda 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -51,7 +51,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update replicaset group to apps/v1 {pull}15854[15802] - Fix issue where default go logger is not discarded when either * or stdout is selected. {issue}10251[10251] {pull}15708[15708] - Upgrade go-ucfg to latest v0.8.1. {pull}15937{15937} -- Remove superfluous use of number_of_routing_shards setting from the default template. {pull}16038[16038] +- Fix index names for indexing not always guaranteed to be lower case. {pull}16081[16081] *Auditbeat* diff --git a/libbeat/outputs/kafka/client.go b/libbeat/outputs/kafka/client.go index 1191b1ae0113..ad1ff3e45de0 100644 --- a/libbeat/outputs/kafka/client.go +++ b/libbeat/outputs/kafka/client.go @@ -79,7 +79,7 @@ func newKafkaClient( hosts: hosts, topic: topic, key: key, - index: index, + index: strings.ToLower(index), codec: writer, config: *cfg, } diff --git a/libbeat/outputs/logstash/config.go b/libbeat/outputs/logstash/config.go index 598413f5814f..6d2a30d39ea6 100644 --- a/libbeat/outputs/logstash/config.go +++ b/libbeat/outputs/logstash/config.go @@ -18,6 +18,7 @@ package logstash import ( + "strings" "time" "github.com/elastic/beats/libbeat/beat" @@ -80,7 +81,7 @@ func readConfig(cfg *common.Config, info beat.Info) (*Config, error) { } if c.Index == "" { - c.Index = info.IndexPrefix + c.Index = strings.ToLower(info.IndexPrefix) } return &c, nil diff --git a/libbeat/outputs/logstash/enc.go b/libbeat/outputs/logstash/enc.go index b9fa409b6e9c..fb42626ec5cd 100644 --- a/libbeat/outputs/logstash/enc.go +++ b/libbeat/outputs/logstash/enc.go @@ -18,6 +18,8 @@ package logstash import ( + "strings" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/outputs/codec/json" ) @@ -27,6 +29,7 @@ func makeLogstashEventEncoder(info beat.Info, escapeHTML bool, index string) fun Pretty: false, EscapeHTML: escapeHTML, }) + index = strings.ToLower(index) return func(event interface{}) (d []byte, err error) { d, err = enc.Encode(index, event.(*beat.Event)) if err != nil { diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index 8941984b922a..6aca15337fe8 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -92,7 +92,8 @@ func esConnect(t *testing.T, index string) *esConnection { host := getElasticsearchHost() indexFmt := fmtstr.MustCompileEvent(fmt.Sprintf("%s-%%{+yyyy.MM.dd}", index)) - indexSel := outil.MakeSelector(outil.FmtSelectorExpr(indexFmt, "")) + indexFmtExpr, _ := outil.FmtSelectorExpr(indexFmt, "") + indexSel := outil.MakeSelector(indexFmtExpr) index, _ = indexSel.Select(&beat.Event{ Timestamp: ts, }) diff --git a/libbeat/outputs/outil/select.go b/libbeat/outputs/outil/select.go index d6ff4931b64e..d06ee4e32091 100644 --- a/libbeat/outputs/outil/select.go +++ b/libbeat/outputs/outil/select.go @@ -19,6 +19,7 @@ package outil import ( "fmt" + "strings" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" @@ -26,10 +27,14 @@ import ( "github.com/elastic/beats/libbeat/conditions" ) +// Selector is used to produce a string based on the contents of a Beats event. +// A selector supports multiple rules that need to be configured. type Selector struct { sel SelectorExpr } +// Settings configures how BuildSelectorFromConfig creates a Selector from +// a given configuration object. type Settings struct { // single selector key and default option keyword Key string @@ -44,6 +49,8 @@ type Settings struct { FailEmpty bool } +// SelectorExpr represents an expression object that can be composed with other +// expressions in order to build a Selector. type SelectorExpr interface { sel(evt *beat.Event) (string, error) } @@ -76,6 +83,7 @@ type mapSelector struct { var nilSelector SelectorExpr = &emptySelector{} +// MakeSelector creates a selector from a set of selector expressions. func MakeSelector(es ...SelectorExpr) Selector { switch len(es) { case 0: @@ -95,10 +103,12 @@ func (s Selector) Select(evt *beat.Event) (string, error) { return s.sel.sel(evt) } +// IsEmpty checks if the selector is not configured and will always return an empty string. func (s Selector) IsEmpty() bool { return s.sel == nilSelector || s.sel == nil } +// IsConst checks if the selector will always return the same string. func (s Selector) IsConst() bool { if s.sel == nilSelector { return true @@ -108,6 +118,7 @@ func (s Selector) IsConst() bool { return ok } +// BuildSelectorFromConfig creates a selector from a configuration object. func BuildSelectorFromConfig( cfg *common.Config, settings Settings, @@ -156,17 +167,13 @@ func BuildSelectorFromConfig( return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key)) } - if fmtstr.IsConst() { - str, err := fmtstr.Run(nil) - if err != nil { - return Selector{}, err - } + fmtsel, err := FmtSelectorExpr(fmtstr, "") + if err != nil { + return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key)) + } - if str != "" { - sel = append(sel, ConstSelectorExpr(str)) - } - } else { - sel = append(sel, FmtSelectorExpr(fmtstr, "")) + if fmtsel != nilSelector { + sel = append(sel, fmtsel) } } @@ -183,22 +190,44 @@ func BuildSelectorFromConfig( return MakeSelector(sel...), nil } +// EmptySelectorExpr create a selector expression that returns an empty string. func EmptySelectorExpr() SelectorExpr { return nilSelector } +// ConstSelectorExpr creates a selector expression that always returns the configured string. func ConstSelectorExpr(s string) SelectorExpr { - return &constSelector{s} + if s == "" { + return EmptySelectorExpr() + } + return &constSelector{strings.ToLower(s)} } -func FmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) SelectorExpr { - return &fmtSelector{*fmt, fallback} +// FmtSelectorExpr creates a selector expression using a format string. If the +// event can not be applied the default fallback constant string will be returned. +func FmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) (SelectorExpr, error) { + if fmt.IsConst() { + str, err := fmt.Run(nil) + if err != nil { + return nil, err + } + if str == "" { + str = fallback + } + return ConstSelectorExpr(str), nil + } + + return &fmtSelector{*fmt, strings.ToLower(fallback)}, nil } +// ConcatSelectorExpr combines multiple expressions that are run one after the other. +// The first expression that returns a string wins. func ConcatSelectorExpr(s ...SelectorExpr) SelectorExpr { return &listSelector{s} } +// ConditionalSelectorExpr executes the given expression only if the event +// matches the given condition. func ConditionalSelectorExpr( s SelectorExpr, cond conditions.Condition, @@ -206,12 +235,39 @@ func ConditionalSelectorExpr( return &condSelector{s, cond} } +// LookupSelectorExpr replaces the produced string with an table entry. +// If there is no entry in the table the default fallback string will be reported. func LookupSelectorExpr( - s SelectorExpr, + evtfmt *fmtstr.EventFormatString, table map[string]string, fallback string, -) SelectorExpr { - return &mapSelector{s, fallback, table} +) (SelectorExpr, error) { + if evtfmt.IsConst() { + str, err := evtfmt.Run(nil) + if err != nil { + return nil, err + } + + str = table[strings.ToLower(str)] + if str == "" { + str = fallback + } + return ConstSelectorExpr(str), nil + } + + return &mapSelector{ + from: &fmtSelector{f: *evtfmt}, + to: table, + otherwise: fallback, + }, nil +} + +func lowercaseTable(table map[string]string) map[string]string { + tmp := make(map[string]string, len(table)) + for k, v := range table { + tmp[strings.ToLower(k)] = strings.ToLower(v) + } + return tmp } func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { @@ -239,7 +295,7 @@ func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { if err != nil { return nil, err } - otherwise = tmp + otherwise = strings.ToLower(tmp) } // 3. extract optional `mapping` @@ -276,45 +332,14 @@ func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { // 5. build selector from available fields var sel SelectorExpr if len(mapping.Table) > 0 { - if evtfmt.IsConst() { - str, err := evtfmt.Run(nil) - if err != nil { - return nil, err - } - - str = mapping.Table[str] - if str == "" { - str = otherwise - } - - if str == "" { - sel = nilSelector - } else { - sel = ConstSelectorExpr(str) - } - } else { - sel = &mapSelector{ - from: FmtSelectorExpr(evtfmt, ""), - to: mapping.Table, - otherwise: otherwise, - } - } + sel, err = LookupSelectorExpr(evtfmt, lowercaseTable(mapping.Table), otherwise) } else { - if evtfmt.IsConst() { - str, err := evtfmt.Run(nil) - if err != nil { - return nil, err - } - - if str == "" { - sel = nilSelector - } else { - sel = ConstSelectorExpr(str) - } - } else { - sel = FmtSelectorExpr(evtfmt, otherwise) - } + sel, err = FmtSelectorExpr(evtfmt, otherwise) } + if err != nil { + return nil, err + } + if cond != nil && sel != nilSelector { sel = ConditionalSelectorExpr(sel, cond) } @@ -363,7 +388,7 @@ func (s *fmtSelector) sel(evt *beat.Event) (string, error) { if n == "" { return s.otherwise, nil } - return n, nil + return strings.ToLower(n), nil } func (s *mapSelector) sel(evt *beat.Event) (string, error) { diff --git a/libbeat/outputs/outil/select_test.go b/libbeat/outputs/outil/select_test.go index d4093ed56606..f6e837966b2d 100644 --- a/libbeat/outputs/outil/select_test.go +++ b/libbeat/outputs/outil/select_test.go @@ -31,74 +31,92 @@ import ( type node map[string]interface{} func TestSelector(t *testing.T) { - tests := []struct { - title string + tests := map[string]struct { config string event common.MapStr expected string }{ - { - "constant key", + "constant key": { `key: value`, common.MapStr{}, "value", }, - { - "format string key", + "lowercase constant key": { + `key: VaLuE`, + common.MapStr{}, + "value", + }, + "format string key": { `key: '%{[key]}'`, common.MapStr{"key": "value"}, "value", }, - { - "key with empty keys", + "lowercase format string key": { + `key: '%{[key]}'`, + common.MapStr{"key": "VaLuE"}, + "value", + }, + "key with empty keys": { `{key: value, keys: }`, common.MapStr{}, "value", }, - { - "constant in multi key", + "lowercase key with empty keys": { + `{key: vAlUe, keys: }`, + common.MapStr{}, + "value", + }, + "constant in multi key": { `keys: [key: 'value']`, common.MapStr{}, "value", }, - { - "format string in multi key", + "format string in multi key": { `keys: [key: '%{[key]}']`, common.MapStr{"key": "value"}, "value", }, - { - "missing format string key with default in rule", + "missing format string key with default in rule": { `keys: - key: '%{[key]}' default: value`, common.MapStr{}, "value", }, - { - "empty format string key with default in rule", + "lowercase missing format string key with default in rule": { + `keys: + - key: '%{[key]}' + default: vAlUe`, + common.MapStr{}, + "value", + }, + "empty format string key with default in rule": { `keys: - key: '%{[key]}' default: value`, common.MapStr{"key": ""}, "value", }, - { - "missing format string key with constant in next rule", + "lowercase empty format string key with default in rule": { + `keys: + - key: '%{[key]}' + default: vAluE`, + common.MapStr{"key": ""}, + "value", + }, + "missing format string key with constant in next rule": { `keys: - key: '%{[key]}' - key: value`, common.MapStr{}, "value", }, - { - "missing format string key with constant in top-level rule", + "missing format string key with constant in top-level rule": { `{ key: value, keys: [key: '%{[key]}']}`, common.MapStr{}, "value", }, - { - "apply mapping", + "apply mapping": { `keys: - key: '%{[key]}' mappings: @@ -106,8 +124,15 @@ func TestSelector(t *testing.T) { common.MapStr{"key": "v"}, "value", }, - { - "apply mapping with default on empty key", + "lowercase applied mapping": { + `keys: + - key: '%{[key]}' + mappings: + v: vAlUe`, + common.MapStr{"key": "v"}, + "value", + }, + "apply mapping with default on empty key": { `keys: - key: '%{[key]}' default: value @@ -116,8 +141,16 @@ func TestSelector(t *testing.T) { common.MapStr{"key": ""}, "value", }, - { - "apply mapping with default on empty lookup", + "lowercase apply mapping with default on empty key": { + `keys: + - key: '%{[key]}' + default: vAluE + mappings: + v: 'v'`, + common.MapStr{"key": ""}, + "value", + }, + "apply mapping with default on empty lookup": { `keys: - key: '%{[key]}' default: value @@ -126,8 +159,7 @@ func TestSelector(t *testing.T) { common.MapStr{"key": "v"}, "value", }, - { - "apply mapping without match", + "apply mapping without match": { `keys: - key: '%{[key]}' mappings: @@ -136,8 +168,7 @@ func TestSelector(t *testing.T) { common.MapStr{"key": "x"}, "value", }, - { - "mapping with constant key", + "mapping with constant key": { `keys: - key: k mappings: @@ -145,8 +176,7 @@ func TestSelector(t *testing.T) { common.MapStr{}, "value", }, - { - "mapping with missing constant key", + "mapping with missing constant key": { `keys: - key: unknown mappings: {k: wrong} @@ -154,8 +184,7 @@ func TestSelector(t *testing.T) { common.MapStr{}, "value", }, - { - "mapping with missing constant key, but default", + "mapping with missing constant key, but default": { `keys: - key: unknown default: value @@ -163,16 +192,14 @@ func TestSelector(t *testing.T) { common.MapStr{}, "value", }, - { - "matching condition", + "matching condition": { `keys: - key: value when.equals.test: test`, common.MapStr{"test": "test"}, "value", }, - { - "failing condition", + "failing condition": { `keys: - key: wrong when.equals.test: test @@ -182,113 +209,98 @@ func TestSelector(t *testing.T) { }, } - for i, test := range tests { - t.Logf("run (%v): %v", i, test.title) - - yaml := strings.Replace(test.config, "\t", " ", -1) - cfg, err := common.NewConfigWithYAML([]byte(yaml), "test") - if err != nil { - t.Errorf("YAML parse error: %v\n%v", err, yaml) - continue - } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + yaml := strings.Replace(test.config, "\t", " ", -1) + cfg, err := common.NewConfigWithYAML([]byte(yaml), "test") + if err != nil { + t.Fatalf("YAML parse error: %v\n%v", err, yaml) + } - sel, err := BuildSelectorFromConfig(cfg, Settings{ - Key: "key", - MultiKey: "keys", - EnableSingleOnly: true, - FailEmpty: true, - }) - if err != nil { - t.Error(err) - continue - } + sel, err := BuildSelectorFromConfig(cfg, Settings{ + Key: "key", + MultiKey: "keys", + EnableSingleOnly: true, + FailEmpty: true, + }) + if err != nil { + t.Fatal(err) + } - event := beat.Event{ - Timestamp: time.Now(), - Fields: test.event, - } - actual, err := sel.Select(&event) - if err != nil { - t.Error(err) - continue - } + event := beat.Event{ + Timestamp: time.Now(), + Fields: test.event, + } + actual, err := sel.Select(&event) + if err != nil { + t.Fatal(err) + } - assert.Equal(t, test.expected, actual) + assert.Equal(t, test.expected, actual) + }) } } func TestSelectorInitFail(t *testing.T) { - tests := []struct { - title string + tests := map[string]struct { config string }{ - { - "keys missing", + "keys missing": { `test: no key`, }, - { - "invalid keys type", + "invalid keys type": { `keys: 5`, }, - { - "invaid keys element type", + "invaid keys element type": { `keys: [5]`, }, - { - "invalid key type", + "invalid key type": { `key: {}`, }, - { - "missing key in list", + "missing key in list": { `keys: [default: value]`, }, - { - "invalid key type in list", + "invalid key type in list": { `keys: [key: {}]`, }, - { - "fail on invalid format string", + "fail on invalid format string": { `key: '%{[abc}'`, }, - { - "fail on invalid format string in list", + "fail on invalid format string in list": { `keys: [key: '%{[abc}']`, }, - { - "default value type mismatch", + "default value type mismatch": { `keys: [{key: ok, default: {}}]`, }, - { - "mappings type mismatch", + "mappings type mismatch": { `keys: - key: '%{[k]}' mappings: {v: {}}`, }, - { - "condition empty", + "condition empty": { `keys: - key: value when:`, }, } - for i, test := range tests { - t.Logf("run (%v): %v", i, test.title) + for name, test := range tests { + t.Run(name, func(t *testing.T) { + cfg, err := common.NewConfigWithYAML([]byte(test.config), "test") + if err != nil { + t.Fatal(err) + } - cfg, err := common.NewConfigWithYAML([]byte(test.config), "test") - if err != nil { - t.Error(err) - continue - } + _, err = BuildSelectorFromConfig(cfg, Settings{ + Key: "key", + MultiKey: "keys", + EnableSingleOnly: true, + FailEmpty: true, + }) - _, err = BuildSelectorFromConfig(cfg, Settings{ - Key: "key", - MultiKey: "keys", - EnableSingleOnly: true, - FailEmpty: true, + assert.Error(t, err) + t.Log(err) }) - assert.Error(t, err) - t.Log(err) } } diff --git a/libbeat/outputs/redis/client.go b/libbeat/outputs/redis/client.go index 7238ba367fd0..1764d9e60f99 100644 --- a/libbeat/outputs/redis/client.go +++ b/libbeat/outputs/redis/client.go @@ -21,6 +21,7 @@ import ( "errors" "regexp" "strconv" + "strings" "time" "github.com/garyburd/redigo/redis" @@ -77,7 +78,7 @@ func newClient( observer: observer, timeout: timeout, password: pass, - index: index, + index: strings.ToLower(index), db: db, dataType: dt, key: key,