support defining metrics_filters for prometheus module in annotations (…

…elastic#24264)
urso · Mar 4, 2021 · 9dd84a9 · 9dd84a9
1 parent 7eab148
commit 9dd84a9
Show file tree

Hide file tree

Showing 4 changed files with 79 additions and 11 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -1006,6 +1006,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Enrich events of `state_service` metricset with kubernetes services' metadata. {pull}23730[23730]
 - Add support for Darwin/arm M1. {pull}24019[24019]
 - Check fields are documented in aws metricsets. {pull}23887[23887]
+- Add support for defining metrics_filters for prometheus module in hints. {pull}24264[24264]
 
 *Packetbeat*
 

diff --git a/metricbeat/autodiscover/builder/hints/metrics.go b/metricbeat/autodiscover/builder/hints/metrics.go
@@ -38,16 +38,17 @@ func init() {
 }
 
 const (
-	module      = "module"
-	namespace   = "namespace"
-	hosts       = "hosts"
-	metricsets  = "metricsets"
-	period      = "period"
-	timeout     = "timeout"
-	ssl         = "ssl"
-	metricspath = "metrics_path"
-	username    = "username"
-	password    = "password"
+	module         = "module"
+	namespace      = "namespace"
+	hosts          = "hosts"
+	metricsets     = "metricsets"
+	period         = "period"
+	timeout        = "timeout"
+	ssl            = "ssl"
+	metricsfilters = "metrics_filters"
+	metricspath    = "metrics_path"
+	username       = "username"
+	password       = "password"
 
 	defaultTimeout = "3s"
 	defaultPeriod  = "1m"
@@ -140,6 +141,10 @@ func (m *metricHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*c
 			"processors": procs,
 		}
 
+		if mod == "prometheus" {
+			moduleConfig[metricsfilters] = m.getMetricsFilters(hint)
+		}
+
 		if ns != "" {
 			moduleConfig["namespace"] = ns
 		}
@@ -300,6 +305,14 @@ func (m *metricHints) getSSLConfig(hints common.MapStr) common.MapStr {
 	return builder.GetHintMapStr(hints, m.Key, ssl)
 }
 
+func (m *metricHints) getMetricsFilters(hints common.MapStr) common.MapStr {
+	mf := common.MapStr{}
+	for k := range builder.GetHintMapStr(hints, m.Key, metricsfilters) {
+		mf[k] = builder.GetHintAsList(hints, m.Key, metricsfilters+"."+k)
+	}
+	return mf
+}
+
 func (m *metricHints) getModuleConfigs(hints common.MapStr) []common.MapStr {
 	return builder.GetHintAsConfigs(hints, m.Key)
 }

diff --git a/metricbeat/autodiscover/builder/hints/metrics_test.go b/metricbeat/autodiscover/builder/hints/metrics_test.go
@@ -548,13 +548,47 @@ func TestGenerateHints(t *testing.T) {
 				},
 			},
 		},
+		{
+			message: "exclude/exclude in metrics filters are parsed as a list",
+			event: bus.Event{
+				"host": "1.2.3.4",
+				"hints": common.MapStr{
+					"metrics": common.MapStr{
+						"module":    "prometheus",
+						"namespace": "test",
+						"hosts":     "${data.host}:9090",
+						"metrics_filters": common.MapStr{
+							"exclude": "foo, bar",
+							"include": "xxx, yyy",
+						},
+					},
+				},
+			},
+			len: 1,
+			result: []common.MapStr{
+				{
+					"module":     "prometheus",
+					"namespace":  "test",
+					"metricsets": []string{"collector"},
+					"timeout":    "3s",
+					"period":     "1m",
+					"enabled":    true,
+					"hosts":      []interface{}{"1.2.3.4:9090"},
+					"metrics_filters": map[string]interface{}{
+						"exclude": []interface{}{"foo", "bar"},
+						"include": []interface{}{"xxx", "yyy"},
+					},
+				},
+			},
+		},
 	}
 	for _, test := range tests {
 		mockRegister := mb.NewRegister()
 		mockRegister.MustAddMetricSet("mockmodule", "one", NewMockMetricSet, mb.DefaultMetricSet())
 		mockRegister.MustAddMetricSet("mockmodule", "two", NewMockMetricSet, mb.DefaultMetricSet())
 		mockRegister.MustAddMetricSet("mockmoduledefaults", "default", NewMockMetricSet, mb.DefaultMetricSet())
 		mockRegister.MustAddMetricSet("mockmoduledefaults", "other", NewMockMetricSet)
+		mockRegister.MustAddMetricSet("prometheus", "collector", NewMockMetricSet)
 
 		m := metricHints{
 			Key:      defaultConfig().Key,
@@ -679,8 +713,16 @@ func (ms *MockMetricSet) Fetch(report mb.Reporter) {
 
 }
 
+type MockPrometheus struct {
+	*MockMetricSet
+}
+
+func NewMockPrometheus(base mb.BaseMetricSet) (mb.MetricSet, error) {
+	return &MockPrometheus{}, nil
+}
+
 // create a keystore with an existing key
-/// `PASSWORD` with the value of `secret` variable.
+// `PASSWORD` with the value of `secret` variable.
 func createAnExistingKeystore(path string, secret string) keystore.Keystore {
 	keyStore, err := keystore.NewFileKeystore(path)
 	// Fail fast in the test suite

diff --git a/metricbeat/docs/autodiscover-hints.asciidoc b/metricbeat/docs/autodiscover-hints.asciidoc
@@ -56,6 +56,18 @@ can make use of Kuberentes Secrets as described in <<kubernetes-secrets>>.
 
 SSL parameters, as seen in <<configuration-ssl>>.
 
+[float]
+===== `co.elastic.metrics/metrics_filters.*`
+
+Metrics filters (for prometheus module only).
+
+["source","yaml",subs="attributes"]
+-------------------------------------------------------------------------------------
+co.elastic.metrics/module: prometheus
+co.elastic.metrics/metrics_filters.include: node_filesystem_*
+co.elastic.metrics/metrics_filters.exclude: node_filesystem_device_foo,node_filesystem_device_bar
+-------------------------------------------------------------------------------------
+
 [float]
 ===== `co.elastic.metrics/raw`
 When an entire module configuration needs to be completely set the `raw` hint can be used. You can provide a