diff --git a/metricbeat/helper/elastic/elastic.go b/metricbeat/helper/elastic/elastic.go index fa466a5fee1c..e6cdea008f82 100644 --- a/metricbeat/helper/elastic/elastic.go +++ b/metricbeat/helper/elastic/elastic.go @@ -138,7 +138,7 @@ func FixTimestampField(m mapstr.M, field string) error { } // NewModule returns a new Elastic stack module with the appropriate metricsets configured. -func NewModule(base *mb.BaseModule, xpackEnabledMetricsets []string, logger *logp.Logger) (*mb.BaseModule, error) { +func NewModule(base *mb.BaseModule, xpackEnabledMetricsets []string, optionalXpackMetricsets []string, logger *logp.Logger) (*mb.BaseModule, error) { moduleName := base.Name() config := struct { @@ -158,8 +158,35 @@ func NewModule(base *mb.BaseModule, xpackEnabledMetricsets []string, logger *log return nil, errors.Wrapf(err, "could not unpack configuration for module %v", moduleName) } - // These metricsets are exactly the ones required if xpack.enabled == true - raw["metricsets"] = xpackEnabledMetricsets + // Ensure all required metricsets are enabled when xpack.enabled == true, and add any additional which are optional + cfgdMetricsets, err := raw.GetValue("metricsets") + metricsets := xpackEnabledMetricsets + if err == nil && cfgdMetricsets != nil { + // Type cast the metricsets to a slice of strings + cfgdMetricsetsSlice := cfgdMetricsets.([]interface{}) + cfgdMetricsetsStrings := make([]string, len(cfgdMetricsetsSlice)) + for i := range cfgdMetricsetsSlice { + cfgdMetricsetsStrings[i] = cfgdMetricsetsSlice[i].(string) + } + + // Add any optional metricsets which are not already configured + for _, cfgdMs := range cfgdMetricsetsStrings { + found := false + for _, ms := range optionalXpackMetricsets { + if ms == cfgdMs { + found = true + break + } + } + + if found { + metricsets = append(metricsets, cfgdMs) + } + } + + } + + raw["metricsets"] = metricsets newConfig, err := conf.NewConfigFrom(raw) if err != nil { diff --git a/metricbeat/helper/elastic/elastic_test.go b/metricbeat/helper/elastic/elastic_test.go index a7cc38a5680e..be5529928029 100644 --- a/metricbeat/helper/elastic/elastic_test.go +++ b/metricbeat/helper/elastic/elastic_test.go @@ -157,11 +157,14 @@ func TestConfigureModule(t *testing.T) { require.NoError(t, err) err = mockRegistry.AddMetricSet(moduleName, "baz", mockMetricSetFactory) require.NoError(t, err) + err = mockRegistry.AddMetricSet(moduleName, "thud", mockMetricSetFactory) + require.NoError(t, err) tests := map[string]struct { - initConfig metricSetConfig - xpackEnabledMetricsets []string - newConfig metricSetConfig + initConfig metricSetConfig + xpackEnabledMetricsets []string + optionalXpackMetricsets []string + newConfig metricSetConfig }{ "no_xpack_enabled": { metricSetConfig{ @@ -169,6 +172,7 @@ func TestConfigureModule(t *testing.T) { MetricSets: []string{"foo", "bar"}, }, []string{"baz", "qux", "foo"}, + []string{}, metricSetConfig{ Module: moduleName, MetricSets: []string{"foo", "bar"}, @@ -181,12 +185,27 @@ func TestConfigureModule(t *testing.T) { MetricSets: []string{"foo", "bar"}, }, []string{"baz", "qux", "foo"}, + []string{}, metricSetConfig{ Module: moduleName, XPackEnabled: true, MetricSets: []string{"baz", "qux", "foo"}, }, }, + "xpack_enabled_with_optional": { + metricSetConfig{ + Module: moduleName, + XPackEnabled: true, + MetricSets: []string{"foo", "bar", "thud"}, + }, + []string{"baz", "qux", "foo"}, + []string{"bar"}, + metricSetConfig{ + Module: moduleName, + XPackEnabled: true, + MetricSets: []string{"baz", "qux", "foo", "bar"}, // include optional, exclude others + }, + }, } for name, test := range tests { @@ -200,7 +219,7 @@ func TestConfigureModule(t *testing.T) { require.Fail(t, "expecting module to be base module") } - newM, err := NewModule(bm, test.xpackEnabledMetricsets, logp.L()) + newM, err := NewModule(bm, test.xpackEnabledMetricsets, test.optionalXpackMetricsets, logp.L()) require.NoError(t, err) var newConfig metricSetConfig diff --git a/metricbeat/module/beat/beat.go b/metricbeat/module/beat/beat.go index c58e33974454..138e2d99750b 100644 --- a/metricbeat/module/beat/beat.go +++ b/metricbeat/module/beat/beat.go @@ -41,7 +41,7 @@ var metricSets = []string{"state", "stats"} // NewModule creates a new module func NewModule(base mb.BaseModule) (mb.Module, error) { - return elastic.NewModule(&base, metricSets, logp.NewLogger(ModuleName)) + return elastic.NewModule(&base, metricSets, []string{}, logp.NewLogger(ModuleName)) } // ModuleName is the name of this module. diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index cd7dc0ec47ed..56e2d5bedf87 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -56,7 +56,7 @@ func NewModule(base mb.BaseModule) (mb.Module, error) { "node_stats", "shard", } - return elastic.NewModule(&base, xpackEnabledMetricSets, logp.NewLogger(ModuleName)) + return elastic.NewModule(&base, xpackEnabledMetricSets, []string{}, logp.NewLogger(ModuleName)) } var ( diff --git a/metricbeat/module/kibana/kibana.go b/metricbeat/module/kibana/kibana.go index 781a0fccfda9..9a3fa60b12e8 100644 --- a/metricbeat/module/kibana/kibana.go +++ b/metricbeat/module/kibana/kibana.go @@ -92,7 +92,7 @@ func NewModule(base mb.BaseModule) (mb.Module, error) { xpackEnabledMetricSets := []string{ "stats", "cluster_rules", "node_rules", "cluster_actions", "node_actions", } - return elastic.NewModule(&base, xpackEnabledMetricSets, logp.NewLogger(ModuleName)) + return elastic.NewModule(&base, xpackEnabledMetricSets, []string{}, logp.NewLogger(ModuleName)) } // GetVersion returns the version of the Kibana instance diff --git a/metricbeat/module/logstash/logstash.go b/metricbeat/module/logstash/logstash.go index 6008b8d947ad..8623bcd9a803 100644 --- a/metricbeat/module/logstash/logstash.go +++ b/metricbeat/module/logstash/logstash.go @@ -40,7 +40,7 @@ func init() { // NewModule creates a new module func NewModule(base mb.BaseModule) (mb.Module, error) { - return elastic.NewModule(&base, []string{"node", "node_stats"}, logp.NewLogger(ModuleName)) + return elastic.NewModule(&base, []string{"node", "node_stats"}, []string{}, logp.NewLogger(ModuleName)) } // ModuleName is the name of this module.