From b301961edc9edb3c1785e4a2c7b5dcce01c47607 Mon Sep 17 00:00:00 2001 From: ruflin Date: Wed, 28 Jun 2017 09:30:14 +0200 Subject: [PATCH] Add support for analyzers and multifields This PR enhances fields.yml to support `analyzer`, `search_analyzer`, `norms` and `multi_fields`. Currently `multi_fields` are only supported for `text` fields. Norms is still disabled by default but can now be overwritten. Below is an example on the usage: ``` fields: - name: phrase type: text multi_fields: - name: raw type: keyword - name: english type: text analyzer: english search_analyzer: englishenhanced norms: true ``` --- CHANGELOG.asciidoc | 3 +- libbeat/template/field.go | 72 ++++++++++++++++++++++------------ libbeat/template/field_test.go | 67 +++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 28b49fb98146..9e077252d18e 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -58,7 +58,8 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha2...master[Check the HEAD d *Affecting all Beats* - New cli subcommands interface. {pull}4420[4420] -- Allow source path matching in `add_docker_metadata` processor {pull}4495[4495] +- Allow source path matching in `add_docker_metadata` processor. {pull}4495[4495] +- Add support for analyzers and multifields in fields.yml. {pull}4574[4574] *Filebeat* diff --git a/libbeat/template/field.go b/libbeat/template/field.go index 45f27bb9f25f..f36415ceb3d2 100644 --- a/libbeat/template/field.go +++ b/libbeat/template/field.go @@ -11,14 +11,18 @@ var ( ) type Field struct { - Name string `config:"name"` - Type string `config:"type"` - Description string `config:"description"` - Format string `config:"format"` - ScalingFactor int `config:"scaling_factor"` - Fields Fields `config:"fields"` - ObjectType string `config:"object_type"` - Enabled *bool `config:"enabled"` + Name string `config:"name"` + Type string `config:"type"` + Description string `config:"description"` + Format string `config:"format"` + ScalingFactor int `config:"scaling_factor"` + Fields Fields `config:"fields"` + MultiFields Fields `config:"multi_fields"` + ObjectType string `config:"object_type"` + Enabled *bool `config:"enabled"` + Analyzer string `config:"analyzer"` + SearchAnalyzer string `config:"search_analyzer"` + Norms bool `config:"norms"` path string esVersion Version @@ -94,20 +98,37 @@ func (f *Field) keyword() common.MapStr { } func (f *Field) text() common.MapStr { - property := f.getDefaultProperties() + properties := f.getDefaultProperties() - property["type"] = "text" + properties["type"] = "text" if f.esVersion.IsMajor(2) { - property["type"] = "string" - property["index"] = "analyzed" - property["norms"] = common.MapStr{ - "enabled": false, + properties["type"] = "string" + properties["index"] = "analyzed" + if !f.Norms { + properties["norms"] = common.MapStr{ + "enabled": false, + } } } else { - property["norms"] = false + if !f.Norms { + properties["norms"] = false + } } - return property + + if f.Analyzer != "" { + properties["analyzer"] = f.Analyzer + } + + if f.SearchAnalyzer != "" { + properties["search_analyzer"] = f.SearchAnalyzer + } + + if len(f.MultiFields) > 0 { + properties["fields"] = f.MultiFields.process("", f.esVersion) + } + + return properties } func (f *Field) array() common.MapStr { @@ -152,6 +173,16 @@ func (f *Field) addDynamicTemplate(properties common.MapStr, matchType string) { dynamicTemplates = append(dynamicTemplates, template) } +func (f *Field) getDefaultProperties() common.MapStr { + // Currently no defaults exist + property := common.MapStr{} + if f.Enabled != nil { + property["enabled"] = *f.Enabled + } + + return property +} + // Recursively generates the correct key based on the dots // The mapping requires "properties" between each layer. This is added here. func generateKey(key string) string { @@ -161,12 +192,3 @@ func generateKey(key string) string { } return key } - -func (f *Field) getDefaultProperties() common.MapStr { - // Currently no defaults exist - property := common.MapStr{} - if f.Enabled != nil { - property["enabled"] = *f.Enabled - } - return property -} diff --git a/libbeat/template/field_test.go b/libbeat/template/field_test.go index 4b3b9d3bedb0..da40fe5b1239 100644 --- a/libbeat/template/field_test.go +++ b/libbeat/template/field_test.go @@ -65,6 +65,73 @@ func TestField(t *testing.T) { "enabled": false, }, }, + { + field: Field{Type: "text", Analyzer: "autocomplete"}, + method: func(f Field) common.MapStr { return f.text() }, + output: common.MapStr{ + "type": "text", + "analyzer": "autocomplete", + "norms": false, + }, + }, + { + field: Field{Type: "text", Analyzer: "autocomplete", Norms: true}, + method: func(f Field) common.MapStr { return f.text() }, + output: common.MapStr{ + "type": "text", + "analyzer": "autocomplete", + }, + }, + { + field: Field{Type: "text", SearchAnalyzer: "standard", Norms: true}, + method: func(f Field) common.MapStr { return f.text() }, + output: common.MapStr{ + "type": "text", + "search_analyzer": "standard", + }, + }, + { + field: Field{Type: "text", Analyzer: "autocomplete", SearchAnalyzer: "standard", Norms: true}, + method: func(f Field) common.MapStr { return f.text() }, + output: common.MapStr{ + "type": "text", + "analyzer": "autocomplete", + "search_analyzer": "standard", + }, + }, + { + field: Field{Type: "text", MultiFields: Fields{Field{Name: "raw", Type: "keyword"}}, Norms: true}, + method: func(f Field) common.MapStr { return f.text() }, + output: common.MapStr{ + "type": "text", + "fields": common.MapStr{ + "raw": common.MapStr{ + "type": "keyword", + "ignore_above": 1024, + }, + }, + }, + }, + { + field: Field{Type: "text", MultiFields: Fields{ + Field{Name: "raw", Type: "keyword"}, + Field{Name: "indexed", Type: "text"}, + }, Norms: true}, + method: func(f Field) common.MapStr { return f.text() }, + output: common.MapStr{ + "type": "text", + "fields": common.MapStr{ + "raw": common.MapStr{ + "type": "keyword", + "ignore_above": 1024, + }, + "indexed": common.MapStr{ + "type": "text", + "norms": false, + }, + }, + }, + }, } for _, test := range tests {