From 3d5beb0fe8f76cfae98141b946b806a17b6f9c90 Mon Sep 17 00:00:00 2001 From: simitt Date: Fri, 23 Nov 2018 11:15:11 +0100 Subject: [PATCH 1/4] WIP multiple templates --- libbeat/cmd/export/template.go | 50 ++++++----- libbeat/cmd/instance/beat.go | 9 +- libbeat/cmd/instance/setup.go | 26 ------ libbeat/template/config.go | 30 +++++-- libbeat/template/load.go | 105 +++++++++++----------- libbeat/template/load_integration_test.go | 14 +-- libbeat/template/template.go | 9 +- libbeat/template/template_test.go | 4 +- metricbeat/metricbeat.yml | 6 ++ 9 files changed, 128 insertions(+), 125 deletions(-) delete mode 100644 libbeat/cmd/instance/setup.go diff --git a/libbeat/cmd/export/template.go b/libbeat/cmd/export/template.go index bf3cbbb9b27..4e7eeb803e8 100644 --- a/libbeat/cmd/export/template.go +++ b/libbeat/cmd/export/template.go @@ -30,9 +30,11 @@ import ( ) func GenTemplateConfigCmd(settings instance.Settings, name, idxPrefix, beatVersion string) *cobra.Command { + fmt.Println(name) + fmt.Println(idxPrefix) genTemplateConfigCmd := &cobra.Command{ Use: "template", - Short: "Export index template to stdout", + Short: "Export index template(s) to stdout", Run: func(cmd *cobra.Command, args []string) { version, _ := cmd.Flags().GetString("es.version") index, _ := cmd.Flags().GetString("index") @@ -48,38 +50,40 @@ func GenTemplateConfigCmd(settings instance.Settings, name, idxPrefix, beatVersi os.Exit(1) } - cfg := template.DefaultConfig + templatesCfg := template.DefaultConfig if b.Config.Template.Enabled() { - err = b.Config.Template.Unpack(&cfg) + err = b.Config.Template.Unpack(&templatesCfg) if err != nil { fmt.Fprintf(os.Stderr, "Error getting template settings: %+v", err) os.Exit(1) } } - tmpl, err := template.New(b.Info.Version, index, version, cfg) - if err != nil { - fmt.Fprintf(os.Stderr, "Error generating template: %+v", err) - os.Exit(1) - } + for _, cfg := range templatesCfg.Templates { + tmpl, err := template.NewTemplate(b.Info.Version, index, version, cfg) + if err != nil { + fmt.Fprintf(os.Stderr, "Error generating template: %+v", err) + os.Exit(1) + } - var templateString common.MapStr - if cfg.Fields != "" { - fieldsPath := paths.Resolve(paths.Config, cfg.Fields) - templateString, err = tmpl.LoadFile(fieldsPath) - } else { - templateString, err = tmpl.LoadBytes(b.Fields) - } + var templateString common.MapStr + if cfg.Fields != "" { + fieldsPath := paths.Resolve(paths.Config, cfg.Fields) + templateString, err = tmpl.LoadFile(fieldsPath) + } else { + templateString, err = tmpl.LoadBytes(b.Fields) + } - if err != nil { - fmt.Fprintf(os.Stderr, "Error generating template: %+v", err) - os.Exit(1) - } + if err != nil { + fmt.Fprintf(os.Stderr, "Error generating template: %+v", err) + os.Exit(1) + } - _, err = os.Stdout.WriteString(templateString.StringToPrint() + "\n") - if err != nil { - fmt.Fprintf(os.Stderr, "Error writing template: %+v", err) - os.Exit(1) + _, err = os.Stdout.WriteString(templateString.StringToPrint() + "\n") + if err != nil { + fmt.Fprintf(os.Stderr, "Error writing template: %+v", err) + os.Exit(1) + } } }, } diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 9e9c23d756e..ac990189631 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -719,7 +719,7 @@ func (b *Beat) loadDashboards(ctx context.Context, force bool) error { // the elasticsearch output. It is important the the registration happens before // the publisher is created. func (b *Beat) registerTemplateLoading() error { - var cfg template.TemplateConfig + var cfg template.TemplatesConfig // Check if outputting to file is enabled, and output to file if it is if b.Config.Template.Enabled() { @@ -741,13 +741,14 @@ func (b *Beat) registerTemplateLoading() error { return err } - if esCfg.Index != "" && (cfg.Name == "" || cfg.Pattern == "") && (b.Config.Template == nil || b.Config.Template.Enabled()) { + //TODO: what to do with this check with Indices? + if esCfg.Index != "" && (len(cfg.Templates) == 0 || cfg.Templates[0].Name == "" || cfg.Templates[0].Pattern == "") && (b.Config.Template == nil || b.Config.Template.Enabled()) { return fmt.Errorf("setup.template.name and setup.template.pattern have to be set if index name is modified.") } if b.Config.Template == nil || (b.Config.Template != nil && b.Config.Template.Enabled()) { - // load template through callback to make sure it is also loaded + // load templates through callback to make sure it is also loaded // on reconnecting callback, err := b.templateLoadingCallback() if err != nil { @@ -760,7 +761,7 @@ func (b *Beat) registerTemplateLoading() error { return nil } -// Build and return a callback to load index template into ES +// Build and return a callback to load index templates into ES func (b *Beat) templateLoadingCallback() (func(esClient *elasticsearch.Client) error, error) { callback := func(esClient *elasticsearch.Client) error { if b.Config.Template == nil { diff --git a/libbeat/cmd/instance/setup.go b/libbeat/cmd/instance/setup.go deleted file mode 100644 index a058100f92c..00000000000 --- a/libbeat/cmd/instance/setup.go +++ /dev/null @@ -1,26 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package instance - -type TemplateConfig struct { - Enabled bool `config:"enabled"` - Name string `config:"name"` - Fields string `config:"fields"` - Overwrite bool `config:"overwrite"` - Settings map[string]string `config:"settings"` -} diff --git a/libbeat/template/config.go b/libbeat/template/config.go index 77e93d72011..841fbd64af2 100644 --- a/libbeat/template/config.go +++ b/libbeat/template/config.go @@ -20,7 +20,8 @@ package template import "github.com/elastic/beats/libbeat/common" type TemplateConfig struct { - Enabled bool `config:"enabled"` + AppendFields common.Fields `config:"append_fields"` + Name string `config:"name"` Pattern string `config:"pattern"` Fields string `config:"fields"` @@ -29,9 +30,19 @@ type TemplateConfig struct { Path string `config:"path"` Name string `config:"name"` } `config:"json"` - AppendFields common.Fields `config:"append_fields"` - Overwrite bool `config:"overwrite"` - Settings TemplateSettings `config:"settings"` + + Settings TemplateSettings + Enabled bool + Overwrite bool +} + +type TemplatesConfig struct { + Enabled bool `config:"enabled"` + Overwrite bool `config:"overwrite"` + + Templates []TemplateConfig `config:"templates"` + + Settings TemplateSettings `config:"settings"` } type TemplateSettings struct { @@ -41,8 +52,15 @@ type TemplateSettings struct { var ( // DefaultConfig for index template - DefaultConfig = TemplateConfig{ + DefaultConfig = TemplatesConfig{ Enabled: true, - Fields: "", + Templates: []TemplateConfig{ + TemplateConfig{Fields: ""}, + }, } + + // Defaults used in the template + defaultDateDetection = false + defaultTotalFieldsLimit = 10000 + defaultNumberOfRoutingShards = 30 ) diff --git a/libbeat/template/load.go b/libbeat/template/load.go index 157e2c7e915..7953b582727 100644 --- a/libbeat/template/load.go +++ b/libbeat/template/load.go @@ -38,7 +38,7 @@ type ESClient interface { } type Loader struct { - config TemplateConfig + config TemplatesConfig client ESClient beatInfo beat.Info fields []byte @@ -46,6 +46,7 @@ type Loader struct { // NewLoader creates a new template loader func NewLoader(cfg *common.Config, client ESClient, beatInfo beat.Info, fields []byte) (*Loader, error) { + //TODO: change init config := DefaultConfig err := cfg.Unpack(&config) @@ -66,67 +67,69 @@ func NewLoader(cfg *common.Config, client ESClient, beatInfo beat.Info, fields [ // template is written to index func (l *Loader) Load() error { - tmpl, err := New(l.beatInfo.Version, l.beatInfo.IndexPrefix, l.client.GetVersion(), l.config) - if err != nil { - return fmt.Errorf("error creating template instance: %v", err) - } - - templateName := tmpl.GetName() - if l.config.JSON.Enabled { - templateName = l.config.JSON.Name - } - // Check if template already exist or should be overwritten - exists := l.CheckTemplate(templateName) - if !exists || l.config.Overwrite { + for _, cfg := range l.config.Templates { + tmpl, err := NewTemplate(l.beatInfo.Version, l.beatInfo.IndexPrefix, l.client.GetVersion(), cfg) + if err != nil { + return fmt.Errorf("error creating template instance: %v", err) + } - logp.Info("Loading template for Elasticsearch version: %s", l.client.GetVersion()) - if l.config.Overwrite { - logp.Info("Existing template will be overwritten, as overwrite is enabled.") + templateName := tmpl.GetName() + if cfg.JSON.Enabled { + templateName = cfg.JSON.Name } + // Check if template already exist or should be overwritten + exists := l.CheckTemplate(templateName) + if !exists || cfg.Overwrite { - var template map[string]interface{} - if l.config.JSON.Enabled { - jsonPath := paths.Resolve(paths.Config, l.config.JSON.Path) - if _, err := os.Stat(jsonPath); err != nil { - return fmt.Errorf("error checking for json template: %s", err) + logp.Info("Loading template for Elasticsearch version: %s", l.client.GetVersion()) + if cfg.Overwrite { + logp.Info("Existing template will be overwritten, as overwrite is enabled.") } - logp.Info("Loading json template from file %s", jsonPath) - - content, err := ioutil.ReadFile(jsonPath) - if err != nil { - return fmt.Errorf("error reading file. Path: %s, Error: %s", jsonPath, err) - + var template map[string]interface{} + if cfg.JSON.Enabled { + jsonPath := paths.Resolve(paths.Config, cfg.JSON.Path) + if _, err := os.Stat(jsonPath); err != nil { + return fmt.Errorf("error checking for json template: %s", err) + } + + logp.Info("Loading json template from file %s", jsonPath) + + content, err := ioutil.ReadFile(jsonPath) + if err != nil { + return fmt.Errorf("error reading file. Path: %s, Error: %s", jsonPath, err) + + } + err = json.Unmarshal(content, &template) + if err != nil { + return fmt.Errorf("could not unmarshal json template: %s", err) + } + // Load fields from path + } else if cfg.Fields != "" { + logp.Debug("template", "Load fields.yml from file: %s", cfg.Fields) + + fieldsPath := paths.Resolve(paths.Config, cfg.Fields) + + template, err = tmpl.LoadFile(fieldsPath) + if err != nil { + return fmt.Errorf("error creating template from file %s: %v", fieldsPath, err) + } + } else { + logp.Debug("template", "Load default fields") + template, err = tmpl.LoadBytes(l.fields) + if err != nil { + return fmt.Errorf("error creating template: %v", err) + } } - err = json.Unmarshal(content, &template) - if err != nil { - return fmt.Errorf("could not unmarshal json template: %s", err) - } - // Load fields from path - } else if l.config.Fields != "" { - logp.Debug("template", "Load fields.yml from file: %s", l.config.Fields) - - fieldsPath := paths.Resolve(paths.Config, l.config.Fields) - template, err = tmpl.LoadFile(fieldsPath) - if err != nil { - return fmt.Errorf("error creating template from file %s: %v", fieldsPath, err) - } - } else { - logp.Debug("template", "Load default fields.yml") - template, err = tmpl.LoadBytes(l.fields) + err = l.LoadTemplate(templateName, template) if err != nil { - return fmt.Errorf("error creating template: %v", err) + return fmt.Errorf("could not load template. Elasticsearch returned: %v. Template is: %s", err, template) } - } - err = l.LoadTemplate(templateName, template) - if err != nil { - return fmt.Errorf("could not load template. Elasticsearch returned: %v. Template is: %s", err, template) + } else { + logp.Info("Template already exists and will not be overwritten.") } - - } else { - logp.Info("Template already exists and will not be overwritten.") } return nil diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index 12d8568c534..92457c1c11b 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -61,7 +61,7 @@ func TestLoadTemplate(t *testing.T) { fieldsPath := absPath + "/fields.yml" index := "testbeat" - tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{}) + tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplatesConfig{}) assert.NoError(t, err) content, err := tmpl.LoadFile(fieldsPath) assert.NoError(t, err) @@ -149,7 +149,7 @@ func TestLoadBeatsTemplate(t *testing.T) { fieldsPath := absPath + "/fields.yml" index := beat - tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{}) + tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplatesConfig{}) assert.NoError(t, err) content, err := tmpl.LoadFile(fieldsPath) assert.NoError(t, err) @@ -195,7 +195,7 @@ func TestTemplateSettings(t *testing.T) { "enabled": false, }, } - config := TemplateConfig{ + config := TemplatesConfig{ Settings: settings, } tmpl, err := New(version.GetDefaultVersion(), "testbeat", client.GetVersion(), config) @@ -250,7 +250,7 @@ func TestOverwrite(t *testing.T) { client.Request("DELETE", "/_template/"+templateName, "", nil, nil) // Load template - config := newConfigFrom(t, TemplateConfig{ + config := newConfigFrom(t, TemplatesConfig{ Enabled: true, Fields: absPath + "/fields.yml", }) @@ -260,7 +260,7 @@ func TestOverwrite(t *testing.T) { assert.NoError(t, err) // Load template again, this time with custom settings - config = newConfigFrom(t, TemplateConfig{ + config = newConfigFrom(t, TemplatesConfig{ Enabled: true, Fields: absPath + "/fields.yml", Settings: TemplateSettings{ @@ -280,7 +280,7 @@ func TestOverwrite(t *testing.T) { assert.Error(t, err) // Load template again, this time with custom settings AND overwrite: true - config = newConfigFrom(t, TemplateConfig{ + config = newConfigFrom(t, TemplatesConfig{ Enabled: true, Overwrite: true, Fields: absPath + "/fields.yml", @@ -356,7 +356,7 @@ func TestTemplateWithData(t *testing.T) { // Setup ES client := estest.GetTestingElasticsearch(t) - tmpl, err := New(version.GetDefaultVersion(), "testindex", client.GetVersion(), TemplateConfig{}) + tmpl, err := New(version.GetDefaultVersion(), "testindex", client.GetVersion(), TemplatesConfig{}) assert.NoError(t, err) content, err := tmpl.LoadFile(fieldsPath) assert.NoError(t, err) diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 49e274f3727..dcab9a0629d 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -30,17 +30,14 @@ import ( ) var ( - // Defaults used in the template - defaultDateDetection = false - defaultTotalFieldsLimit = 10000 - defaultNumberOfRoutingShards = 30 - // Array to store dynamicTemplate parts in dynamicTemplates []common.MapStr defaultFields []string ) +type Templates []Template + type Template struct { sync.Mutex name string @@ -51,7 +48,7 @@ type Template struct { } // New creates a new template instance -func New(beatVersion string, beatName string, esVersion string, config TemplateConfig) (*Template, error) { +func NewTemplate(beatVersion string, beatName string, esVersion string, config TemplateConfig) (*Template, error) { bV, err := common.NewVersion(beatVersion) if err != nil { return nil, err diff --git a/libbeat/template/template_test.go b/libbeat/template/template_test.go index 09eb50b93a1..26bfcf4a663 100644 --- a/libbeat/template/template_test.go +++ b/libbeat/template/template_test.go @@ -31,7 +31,7 @@ func TestNumberOfRoutingShards(t *testing.T) { beatVersion := "6.1.0" beatName := "testbeat" - config := TemplateConfig{} + config := TemplatesConfig{} // Test it exists in 6.1 template, err := New(beatVersion, beatName, "6.1.0", config) @@ -57,7 +57,7 @@ func TestNumberOfRoutingShardsOverwrite(t *testing.T) { beatVersion := "6.1.0" beatName := "testbeat" - config := TemplateConfig{ + config := TemplatesConfig{ Settings: TemplateSettings{ Index: map[string]interface{}{"number_of_routing_shards": 5}, }, diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index 259092c66df..b642e488428 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -21,6 +21,10 @@ metricbeat.config.modules: #==================== Elasticsearch template setting ========================== +setup.template.name: "metricbeat-simi" +setup.template.pattern: "metricbeat-simi*" +#setup.template.fields: "${path.config}/simi-fields.yml" + setup.template.settings: index.number_of_shards: 1 index.codec: best_compression @@ -97,6 +101,8 @@ output.elasticsearch: #protocol: "https" #username: "elastic" #password: "changeme" + + index: "metricbeat-simi-%{+yyyy.MM.dd}" #----------------------------- Logstash output -------------------------------- #output.logstash: From 33d118ef6c5713821d981e14997fa1dfe23044cd Mon Sep 17 00:00:00 2001 From: simitt Date: Fri, 23 Nov 2018 14:31:49 +0100 Subject: [PATCH 2/4] WIP remove fields from beat --- libbeat/asset/registry.go | 23 ++++++++++++++++++++++- libbeat/beat/beat.go | 2 +- libbeat/cmd/export/template.go | 19 ++++++++++++++++++- libbeat/cmd/instance/beat.go | 10 ++-------- libbeat/template/config.go | 9 ++++++--- libbeat/template/load.go | 31 +++++++++++++++++++++++++++---- libbeat/template/template.go | 12 ++++++++---- metricbeat/metricbeat.yml | 2 +- 8 files changed, 85 insertions(+), 23 deletions(-) diff --git a/libbeat/asset/registry.go b/libbeat/asset/registry.go index 634e4e3878e..9b042b1904c 100644 --- a/libbeat/asset/registry.go +++ b/libbeat/asset/registry.go @@ -21,6 +21,8 @@ import ( "bytes" "compress/zlib" "encoding/base64" + "errors" + "fmt" "io/ioutil" ) @@ -42,7 +44,7 @@ func SetFields(beat, name string, asset func() string) error { return nil } -// GetFields returns a byte array contains all fields for the given beat +// GetFields returns a byte array containing all fields for the given beat func GetFields(beat string) ([]byte, error) { var fields []byte for _, data := range FieldsRegistry[beat] { @@ -56,6 +58,25 @@ func GetFields(beat string) ([]byte, error) { return fields, nil } +// GetFieldsFor returns a byte array containing all fields for the given beat +// and given names +func GetFieldsFor(beat string, names []string) ([]byte, error) { + var fields []byte + for _, name := range names { + data, ok := FieldsRegistry[beat][name] + if !ok { + return nil, errors.New(fmt.Sprintf("No fields available for %s", name)) + } + output, err := DecodeData(data) + if err != nil { + return nil, err + } + + fields = append(fields, output...) + } + return fields, nil +} + // EncodeData compresses the data with zlib and base64 encodes it func EncodeData(data string) (string, error) { var zlibBuf bytes.Buffer diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go index 9423b774dd1..2bc37b41016 100644 --- a/libbeat/beat/beat.go +++ b/libbeat/beat/beat.go @@ -64,7 +64,7 @@ type Beat struct { BeatConfig *common.Config // The beat's own configuration section - Fields []byte // Data from fields.yml + //Fields []byte // Data from fields.yml ConfigManager management.ConfigManager // config manager } diff --git a/libbeat/cmd/export/template.go b/libbeat/cmd/export/template.go index 4e7eeb803e8..ac412a9f99e 100644 --- a/libbeat/cmd/export/template.go +++ b/libbeat/cmd/export/template.go @@ -20,9 +20,11 @@ package export import ( "fmt" "os" + "strings" "github.com/spf13/cobra" + "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/cmd/instance" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/paths" @@ -35,6 +37,7 @@ func GenTemplateConfigCmd(settings instance.Settings, name, idxPrefix, beatVersi genTemplateConfigCmd := &cobra.Command{ Use: "template", Short: "Export index template(s) to stdout", + //TODO: generally use Loader here Run: func(cmd *cobra.Command, args []string) { version, _ := cmd.Flags().GetString("es.version") index, _ := cmd.Flags().GetString("index") @@ -70,8 +73,22 @@ func GenTemplateConfigCmd(settings instance.Settings, name, idxPrefix, beatVersi if cfg.Fields != "" { fieldsPath := paths.Resolve(paths.Config, cfg.Fields) templateString, err = tmpl.LoadFile(fieldsPath) + } else if cfg.Modules != "" { + + fields, err := asset.GetFieldsFor(name, strings.Split(cfg.Modules, ",")) + if err != nil { + fmt.Fprintf(os.Stderr, "Error generating template: %+v", err) + os.Exit(1) + } + templateString, err = tmpl.LoadBytes(fields) } else { - templateString, err = tmpl.LoadBytes(b.Fields) + + fields, err := asset.GetFields(name) + if err != nil { + fmt.Fprintf(os.Stderr, "Error generating template: %+v", err) + os.Exit(1) + } + templateString, err = tmpl.LoadBytes(fields) } if err != nil { diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index ac990189631..00de502fe15 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -38,7 +38,6 @@ import ( "go.uber.org/zap" "github.com/elastic/beats/libbeat/api" - "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/cfgfile" "github.com/elastic/beats/libbeat/cloudid" @@ -196,11 +195,6 @@ func NewBeat(name, indexPrefix, v string) (*Beat, error) { return nil, err } - fields, err := asset.GetFields(name) - if err != nil { - return nil, err - } - id, err := uuid.NewV4() if err != nil { return nil, err @@ -215,7 +209,7 @@ func NewBeat(name, indexPrefix, v string) (*Beat, error) { Hostname: hostname, UUID: id, }, - Fields: fields, + //Fields: fields, } return &Beat{Beat: b}, nil @@ -768,7 +762,7 @@ func (b *Beat) templateLoadingCallback() (func(esClient *elasticsearch.Client) e b.Config.Template = common.NewConfig() } - loader, err := template.NewLoader(b.Config.Template, esClient, b.Info, b.Fields) + loader, err := template.NewLoader(b.Config.Template, esClient, b.Info) if err != nil { return fmt.Errorf("Error creating Elasticsearch template loader: %v", err) } diff --git a/libbeat/template/config.go b/libbeat/template/config.go index 841fbd64af2..bcd69cc83e2 100644 --- a/libbeat/template/config.go +++ b/libbeat/template/config.go @@ -25,18 +25,21 @@ type TemplateConfig struct { Name string `config:"name"` Pattern string `config:"pattern"` Fields string `config:"fields"` + Modules string `config:"modules"` JSON struct { Enabled bool `config:"enabled"` Path string `config:"path"` Name string `config:"name"` } `config:"json"` - Settings TemplateSettings - Enabled bool - Overwrite bool + //TODO: check for overwrites + Settings TemplateSettings `config:"settings"` + Enabled bool `config:"enabled"` + Overwrite bool `config:"overwrite"` } type TemplatesConfig struct { + //TODO: check for this attribute Enabled bool `config:"enabled"` Overwrite bool `config:"overwrite"` diff --git a/libbeat/template/load.go b/libbeat/template/load.go index 7953b582727..2bf7b823d10 100644 --- a/libbeat/template/load.go +++ b/libbeat/template/load.go @@ -22,7 +22,9 @@ import ( "fmt" "io/ioutil" "os" + "strings" + "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" @@ -41,11 +43,10 @@ type Loader struct { config TemplatesConfig client ESClient beatInfo beat.Info - fields []byte } // NewLoader creates a new template loader -func NewLoader(cfg *common.Config, client ESClient, beatInfo beat.Info, fields []byte) (*Loader, error) { +func NewLoader(cfg *common.Config, client ESClient, beatInfo beat.Info) (*Loader, error) { //TODO: change init config := DefaultConfig @@ -58,7 +59,6 @@ func NewLoader(cfg *common.Config, client ESClient, beatInfo beat.Info, fields [ config: config, client: client, beatInfo: beatInfo, - fields: fields, }, nil } @@ -68,6 +68,10 @@ func NewLoader(cfg *common.Config, client ESClient, beatInfo beat.Info, fields [ func (l *Loader) Load() error { for _, cfg := range l.config.Templates { + fmt.Println("------------- SIMI TEMPLATE") + fmt.Println(cfg) + fmt.Println(cfg.Enabled) + fmt.Println(cfg.Overwrite) tmpl, err := NewTemplate(l.beatInfo.Version, l.beatInfo.IndexPrefix, l.client.GetVersion(), cfg) if err != nil { return fmt.Errorf("error creating template instance: %v", err) @@ -114,9 +118,28 @@ func (l *Loader) Load() error { if err != nil { return fmt.Errorf("error creating template from file %s: %v", fieldsPath, err) } + + // Load fields for modules + } else if cfg.Modules != "" { + logp.Debug("template", "Load fields for %s", cfg.Modules) + + fields, err := asset.GetFieldsFor(l.beatInfo.Name, strings.Split(cfg.Modules, ",")) + if err != nil { + return err + } + template, err = tmpl.LoadBytes(fields) + if err != nil { + return fmt.Errorf("error creating template: %v", err) + } + + // Load default fields } else { logp.Debug("template", "Load default fields") - template, err = tmpl.LoadBytes(l.fields) + fields, err := asset.GetFields(l.beatInfo.Name) + if err != nil { + return err + } + template, err = tmpl.LoadBytes(fields) if err != nil { return fmt.Errorf("error creating template: %v", err) } diff --git a/libbeat/template/template.go b/libbeat/template/template.go index dcab9a0629d..6497dbac199 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -152,10 +152,14 @@ func (t *Template) LoadFile(file string) (common.MapStr, error) { } // LoadBytes loads the the template from the given byte array -func (t *Template) LoadBytes(data []byte) (common.MapStr, error) { - fields, err := loadYamlByte(data) - if err != nil { - return nil, err +func (t *Template) LoadBytes(data ...[]byte) (common.MapStr, error) { + var fields common.Fields + for _, d := range data { + f, err := loadYamlByte(d) + if err != nil { + return nil, err + } + fields = append(fields, f...) } return t.load(fields) diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index b642e488428..3ddb994d8ef 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -102,7 +102,7 @@ output.elasticsearch: #username: "elastic" #password: "changeme" - index: "metricbeat-simi-%{+yyyy.MM.dd}" + #index: "metricbeat-simi-%{+yyyy.MM.dd}" #----------------------------- Logstash output -------------------------------- #output.logstash: From 49ca0ad23b867985e324a0c4f3f219a4e5671f0f Mon Sep 17 00:00:00 2001 From: simitt Date: Mon, 26 Nov 2018 18:15:27 +0100 Subject: [PATCH 3/4] backwardscompatible load config; export cmd uses loader --- libbeat/cmd/export/template.go | 63 +++++--------------------------- libbeat/cmd/instance/beat.go | 2 +- libbeat/template/config.go | 54 ++++++++++++++++++++------- libbeat/template/load.go | 67 ++++++++++++++++++++++------------ 4 files changed, 93 insertions(+), 93 deletions(-) diff --git a/libbeat/cmd/export/template.go b/libbeat/cmd/export/template.go index ac412a9f99e..125ab60e594 100644 --- a/libbeat/cmd/export/template.go +++ b/libbeat/cmd/export/template.go @@ -20,29 +20,22 @@ package export import ( "fmt" "os" - "strings" "github.com/spf13/cobra" - "github.com/elastic/beats/libbeat/asset" "github.com/elastic/beats/libbeat/cmd/instance" - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/paths" "github.com/elastic/beats/libbeat/template" ) func GenTemplateConfigCmd(settings instance.Settings, name, idxPrefix, beatVersion string) *cobra.Command { - fmt.Println(name) - fmt.Println(idxPrefix) genTemplateConfigCmd := &cobra.Command{ Use: "template", Short: "Export index template(s) to stdout", - //TODO: generally use Loader here Run: func(cmd *cobra.Command, args []string) { version, _ := cmd.Flags().GetString("es.version") index, _ := cmd.Flags().GetString("index") - b, err := instance.NewBeat(name, idxPrefix, beatVersion) + b, err := instance.NewBeat(name, index, version) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) @@ -53,54 +46,16 @@ func GenTemplateConfigCmd(settings instance.Settings, name, idxPrefix, beatVersi os.Exit(1) } - templatesCfg := template.DefaultConfig - if b.Config.Template.Enabled() { - err = b.Config.Template.Unpack(&templatesCfg) - if err != nil { - fmt.Fprintf(os.Stderr, "Error getting template settings: %+v", err) - os.Exit(1) - } + loader, err := template.NewConsoleLoader(b.Config.Template, b.Info, version) + if err != nil { + fmt.Fprintf(os.Stderr, "Error generating template loader: %+v", err) + os.Exit(1) } - for _, cfg := range templatesCfg.Templates { - tmpl, err := template.NewTemplate(b.Info.Version, index, version, cfg) - if err != nil { - fmt.Fprintf(os.Stderr, "Error generating template: %+v", err) - os.Exit(1) - } - - var templateString common.MapStr - if cfg.Fields != "" { - fieldsPath := paths.Resolve(paths.Config, cfg.Fields) - templateString, err = tmpl.LoadFile(fieldsPath) - } else if cfg.Modules != "" { - - fields, err := asset.GetFieldsFor(name, strings.Split(cfg.Modules, ",")) - if err != nil { - fmt.Fprintf(os.Stderr, "Error generating template: %+v", err) - os.Exit(1) - } - templateString, err = tmpl.LoadBytes(fields) - } else { - - fields, err := asset.GetFields(name) - if err != nil { - fmt.Fprintf(os.Stderr, "Error generating template: %+v", err) - os.Exit(1) - } - templateString, err = tmpl.LoadBytes(fields) - } - - if err != nil { - fmt.Fprintf(os.Stderr, "Error generating template: %+v", err) - os.Exit(1) - } - - _, err = os.Stdout.WriteString(templateString.StringToPrint() + "\n") - if err != nil { - fmt.Fprintf(os.Stderr, "Error writing template: %+v", err) - os.Exit(1) - } + err = loader.Load() + if err != nil { + fmt.Fprintf(os.Stderr, "Error generating template: %+v", err) + os.Exit(1) } }, } diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 00de502fe15..47f115e1506 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -762,7 +762,7 @@ func (b *Beat) templateLoadingCallback() (func(esClient *elasticsearch.Client) e b.Config.Template = common.NewConfig() } - loader, err := template.NewLoader(b.Config.Template, esClient, b.Info) + loader, err := template.NewESLoader(b.Config.Template, esClient, b.Info) if err != nil { return fmt.Errorf("Error creating Elasticsearch template loader: %v", err) } diff --git a/libbeat/template/config.go b/libbeat/template/config.go index bcd69cc83e2..01276fe5a63 100644 --- a/libbeat/template/config.go +++ b/libbeat/template/config.go @@ -17,7 +17,37 @@ package template -import "github.com/elastic/beats/libbeat/common" +import ( + "github.com/elastic/beats/libbeat/common" +) + +func Unpack(c *common.Config) (*TemplatesConfig, error) { + var templatesRaw = struct { + Templates []*common.Config `config:"templates"` + }{} + + if err := c.Unpack(&templatesRaw); err != nil { + return nil, err + } + + var tc TemplatesConfig + + // use `settings.template.templates` if configured + for _, t := range templatesRaw.Templates { + var tmplCfg = DefaultTemplateCfg() + t.Unpack(&tmplCfg) + tc.Templates = append(tc.Templates, tmplCfg) + } + + // fallback if no `settings.template.templates` was configured + if len(tc.Templates) == 0 { + var tmplCfg = DefaultTemplateCfg() + c.Unpack(&tmplCfg) + tc.Templates = append(tc.Templates, tmplCfg) + } + + return &tc, nil +} type TemplateConfig struct { AppendFields common.Fields `config:"append_fields"` @@ -39,11 +69,7 @@ type TemplateConfig struct { } type TemplatesConfig struct { - //TODO: check for this attribute - Enabled bool `config:"enabled"` - Overwrite bool `config:"overwrite"` - - Templates []TemplateConfig `config:"templates"` + Templates []TemplateConfig `config:"-"` Settings TemplateSettings `config:"settings"` } @@ -54,16 +80,16 @@ type TemplateSettings struct { } var ( - // DefaultConfig for index template - DefaultConfig = TemplatesConfig{ - Enabled: true, - Templates: []TemplateConfig{ - TemplateConfig{Fields: ""}, - }, - } - // Defaults used in the template defaultDateDetection = false defaultTotalFieldsLimit = 10000 defaultNumberOfRoutingShards = 30 ) + +func DefaultTemplateCfg() TemplateConfig { + return TemplateConfig{ + Enabled: true, + Overwrite: false, + Fields: "", + } +} diff --git a/libbeat/template/load.go b/libbeat/template/load.go index 2bf7b823d10..7e196cb90be 100644 --- a/libbeat/template/load.go +++ b/libbeat/template/load.go @@ -40,25 +40,33 @@ type ESClient interface { } type Loader struct { - config TemplatesConfig - client ESClient - beatInfo beat.Info + config *TemplatesConfig + client ESClient + beatInfo beat.Info + esVersion string } -// NewLoader creates a new template loader -func NewLoader(cfg *common.Config, client ESClient, beatInfo beat.Info) (*Loader, error) { - //TODO: change init - config := DefaultConfig +// NewESLoader creates a new Elasticsearch template loader +func NewESLoader(cfg *common.Config, client ESClient, beatInfo beat.Info) (*Loader, error) { + return newLoader(cfg, client, beatInfo, client.GetVersion()) +} + +// NewConsoleLoader creates a new Console template loader +func NewConsoleLoader(cfg *common.Config, beatInfo beat.Info, esVersion string) (*Loader, error) { + return newLoader(cfg, nil, beatInfo, esVersion) +} - err := cfg.Unpack(&config) +func newLoader(cfg *common.Config, client ESClient, beatInfo beat.Info, esVersion string) (*Loader, error) { + templatesCfg, err := Unpack(cfg) if err != nil { return nil, err } return &Loader{ - config: config, - client: client, - beatInfo: beatInfo, + config: templatesCfg, + client: client, + beatInfo: beatInfo, + esVersion: esVersion, }, nil } @@ -68,11 +76,11 @@ func NewLoader(cfg *common.Config, client ESClient, beatInfo beat.Info) (*Loader func (l *Loader) Load() error { for _, cfg := range l.config.Templates { - fmt.Println("------------- SIMI TEMPLATE") - fmt.Println(cfg) - fmt.Println(cfg.Enabled) - fmt.Println(cfg.Overwrite) - tmpl, err := NewTemplate(l.beatInfo.Version, l.beatInfo.IndexPrefix, l.client.GetVersion(), cfg) + if !cfg.Enabled { + continue + } + + tmpl, err := NewTemplate(l.beatInfo.Version, l.beatInfo.IndexPrefix, l.esVersion, cfg) if err != nil { return fmt.Errorf("error creating template instance: %v", err) } @@ -82,15 +90,15 @@ func (l *Loader) Load() error { templateName = cfg.JSON.Name } // Check if template already exist or should be overwritten - exists := l.CheckTemplate(templateName) - if !exists || cfg.Overwrite { + loaded := l.templateLoaded(templateName) + if !loaded || cfg.Overwrite { - logp.Info("Loading template for Elasticsearch version: %s", l.client.GetVersion()) + logp.Info("Loading template for Elasticsearch version: %s", l.esVersion) if cfg.Overwrite { logp.Info("Existing template will be overwritten, as overwrite is enabled.") } - var template map[string]interface{} + var template common.MapStr if cfg.JSON.Enabled { jsonPath := paths.Resolve(paths.Config, cfg.JSON.Path) if _, err := os.Stat(jsonPath); err != nil { @@ -123,7 +131,7 @@ func (l *Loader) Load() error { } else if cfg.Modules != "" { logp.Debug("template", "Load fields for %s", cfg.Modules) - fields, err := asset.GetFieldsFor(l.beatInfo.Name, strings.Split(cfg.Modules, ",")) + fields, err := asset.GetFieldsFor(l.beatInfo.Beat, strings.Split(cfg.Modules, ",")) if err != nil { return err } @@ -135,7 +143,7 @@ func (l *Loader) Load() error { // Load default fields } else { logp.Debug("template", "Load default fields") - fields, err := asset.GetFields(l.beatInfo.Name) + fields, err := asset.GetFields(l.beatInfo.Beat) if err != nil { return err } @@ -161,8 +169,15 @@ func (l *Loader) Load() error { // LoadTemplate loads a template into Elasticsearch overwriting the existing // template if it exists. If you wish to not overwrite an existing template // then use CheckTemplate prior to calling this method. -func (l *Loader) LoadTemplate(templateName string, template map[string]interface{}) error { +func (l *Loader) LoadTemplate(templateName string, template common.MapStr) error { logp.Debug("template", "Try loading template with name: %s", templateName) + if l.client == nil { + if _, err := os.Stdout.WriteString(template.StringToPrint() + "\n"); err != nil { + return fmt.Errorf("Error writing template: %v", err) + } + return nil + } + path := "/_template/" + templateName body, err := l.client.LoadJSON(path, template) if err != nil { @@ -174,7 +189,11 @@ func (l *Loader) LoadTemplate(templateName string, template map[string]interface // CheckTemplate checks if a given template already exist. It returns true if // and only if Elasticsearch returns with HTTP status code 200. -func (l *Loader) CheckTemplate(templateName string) bool { +// If no ES client is configured it returns false. +func (l *Loader) templateLoaded(templateName string) bool { + if l.client == nil { + return false + } status, _, _ := l.client.Request("HEAD", "/_template/"+templateName, "", nil, nil) if status != 200 { From 7ff9c687751ca970678a2856fe19fc14ec6f1356 Mon Sep 17 00:00:00 2001 From: simitt Date: Tue, 27 Nov 2018 10:30:48 +0100 Subject: [PATCH 4/4] Example config. --- metricbeat/metricbeat.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index 3ddb994d8ef..a06d227959a 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -21,9 +21,24 @@ metricbeat.config.modules: #==================== Elasticsearch template setting ========================== -setup.template.name: "metricbeat-simi" -setup.template.pattern: "metricbeat-simi*" -#setup.template.fields: "${path.config}/simi-fields.yml" +# Example config for templates +# +# When `setup.template.templates` is configured, the old configuration is ignored. +#setup.template.name: "metricbeat" +#setup.template.pattern: "metricbeat*" +#setup.template.fields: "${path.config}/fields.yml" +#setup.template.enabled: false +#setup.template.overwrite: true +setup.template.templates: + - name: "metricbeat-mix" + pattern: "*" + modules: "aerospike,apache,system" + overwrite: true + - name: "metricbeat-docker" + pattern: "docker*" + modules: "docker" + enabled: true + overwrite: true setup.template.settings: index.number_of_shards: 1 @@ -101,8 +116,6 @@ output.elasticsearch: #protocol: "https" #username: "elastic" #password: "changeme" - - #index: "metricbeat-simi-%{+yyyy.MM.dd}" #----------------------------- Logstash output -------------------------------- #output.logstash: