[Metricbeat/Kibana/stats] exclude_usage=true by default

CHANGELOG.next.asciidoc

@@ -732,6 +732,7 @@ same journal. {pull}18467[18467]
 
 *Metricbeat*
 
+- `kibana` module: `stats` metricset now skips usage collection by default. Set `stats.exclude_usage: false` in the config flag to revert these changes. {pull}22654[22654]
 - Move the windows pdh implementation from perfmon to a shared location in order for future modules/metricsets to make use of. {pull}15503[15503]
 - Add DynamoDB AWS Metricbeat light module {pull}15097[15097]
 - Release elb module as GA. {pull}15485[15485]

metricbeat/module/kibana/config.go

@@ -19,12 +19,14 @@ package kibana
 
 // Config defines the structure for the Kibana module configuration options
 type Config struct {
-	XPackEnabled bool `config:"xpack.enabled"`
+	XPackEnabled      bool `config:"xpack.enabled"`
+	StatsExcludeUsage bool `config:"stats.exclude_usage"`
 }
 
 // DefaultConfig returns the default configuration for the Kibana module
 func DefaultConfig() Config {
 	return Config{
-		XPackEnabled: false,
+		XPackEnabled:      false,
+		StatsExcludeUsage: true,
 	}
 }

metricbeat/module/kibana/metricset.go

@@ -24,7 +24,8 @@ import (
 // MetricSet can be used to build other metricsets within the Kibana module.
 type MetricSet struct {
 	mb.BaseMetricSet
-	XPackEnabled bool
+	XPackEnabled      bool
+	StatsExcludeUsage bool
 }
 
 // NewMetricSet creates a metricset that can be used to build other metricsets
@@ -38,5 +39,6 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 	return &MetricSet{
 		base,
 		config.XPackEnabled,
+		config.StatsExcludeUsage,
 	}, nil
 }

metricbeat/module/kibana/mtest/testing.go

@@ -18,7 +18,7 @@
 package mtest
 
 // GetConfig returns config for kibana module
-func GetConfig(metricset string, host string, xpackEnabled bool) map[string]interface{} {
+func GetConfig(metricset string, host string, xpackEnabled bool, statsExcludeUsage bool) map[string]interface{} {
 	config := map[string]interface{}{
 		"module":     "kibana",
 		"metricsets": []string{metricset},
@@ -27,6 +27,9 @@ func GetConfig(metricset string, host string, xpackEnabled bool) map[string]inte
 	if xpackEnabled {
 		config["xpack.enabled"] = true
 	}
+	if statsExcludeUsage == false {
+		config["stats.exclude_usage"] = false
+	}
 
 	return config
 }

metricbeat/module/kibana/stats/stats.go

@@ -60,6 +60,7 @@ type MetricSet struct {
 	usageLastCollectedOn time.Time
 	usageNextCollectOn   time.Time
 	isUsageExcludable    bool
+	statsExcludeUsage    bool
 }
 
 // New create a new instance of the MetricSet
@@ -148,6 +149,8 @@ func (m *MetricSet) init() error {
 	m.statsHTTP = statsHTTP
 	m.settingsHTTP = settingsHTTP
 	m.isUsageExcludable = kibana.IsUsageExcludable(kibanaVersion)
+	// Copying the config to an internal property to make tests easier to implement
+	m.statsExcludeUsage = m.StatsExcludeUsage
 
 	return nil
 }
@@ -221,5 +224,6 @@ func (m *MetricSet) calculateIntervalMs() int64 {
 }
 
 func (m *MetricSet) shouldCollectUsage(now time.Time) bool {
-	return now.Sub(m.usageLastCollectedOn) > usageCollectionPeriod && now.Sub(m.usageNextCollectOn) > 0
+	return m.statsExcludeUsage == false && // If the config claims we should exclude the usage, there is no need to check for the time
+		now.Sub(m.usageLastCollectedOn) > usageCollectionPeriod && now.Sub(m.usageNextCollectOn) > 0
 }

metricbeat/module/kibana/stats/stats_integration_test.go

@@ -38,7 +38,7 @@ import (
 func TestFetch(t *testing.T) {
 	service := compose.EnsureUpWithTimeout(t, 570, "kibana")
 
-	config := mtest.GetConfig("stats", service.Host(), false)
+	config := mtest.GetConfig("stats", service.Host(), false, true)
 	host := config["hosts"].([]string)[0]
 	version, err := getKibanaVersion(t, host)
 	require.NoError(t, err)
@@ -63,7 +63,7 @@ func TestFetch(t *testing.T) {
 func TestData(t *testing.T) {
 	service := compose.EnsureUp(t, "kibana")
 
-	config := mtest.GetConfig("stats", service.Host(), false)
+	config := mtest.GetConfig("stats", service.Host(), false, true)
 	host := config["hosts"].([]string)[0]
 	version, err := getKibanaVersion(t, host)
 	require.NoError(t, err)

metricbeat/module/kibana/stats/stats_test.go

@@ -64,7 +64,7 @@ func TestFetchUsage(t *testing.T) {
 	}))
 	defer kib.Close()
 
-	config := mtest.GetConfig("stats", kib.URL, true)
+	config := mtest.GetConfig("stats", kib.URL, true, false)
 
 	f := mbtest.NewReportingMetricSetV2Error(t, config)
 
@@ -84,29 +84,40 @@ func TestShouldCollectUsage(t *testing.T) {
 	cases := map[string]struct {
 		usageLastCollectedOn time.Time
 		usageNextCollectOn   time.Time
+		statsExcludeUsage    bool
 		expectedResult       bool
 	}{
 		"within_usage_collection_period": {
 			usageLastCollectedOn: now.Add(-1 * usageCollectionPeriod),
+			statsExcludeUsage:    false,
 			expectedResult:       false,
 		},
 		"after_usage_collection_period_but_before_next_scheduled_collection": {
 			usageLastCollectedOn: now.Add(-2 * usageCollectionPeriod),
 			usageNextCollectOn:   now.Add(3 * time.Hour),
+			statsExcludeUsage:    false,
 			expectedResult:       false,
 		},
 		"after_usage_collection_period_and_after_next_scheduled_collection": {
 			usageLastCollectedOn: now.Add(-2 * usageCollectionPeriod),
 			usageNextCollectOn:   now.Add(-1 * time.Hour),
+			statsExcludeUsage:    false,
 			expectedResult:       true,
 		},
+		"any_time_when_exclude_usage_is_true": {
+			usageLastCollectedOn: now.Add(-2 * usageCollectionPeriod),
+			usageNextCollectOn:   now.Add(-1 * time.Hour),
+			statsExcludeUsage:    true,
+			expectedResult:       false,
+		},
 	}
 
 	for name, test := range cases {
 		t.Run(name, func(t *testing.T) {
 			m := MetricSet{
 				usageLastCollectedOn: test.usageLastCollectedOn,
 				usageNextCollectOn:   test.usageNextCollectOn,
+				statsExcludeUsage:    test.statsExcludeUsage,
 			}
 
 			actualResult := m.shouldCollectUsage(now)

metricbeat/module/kibana/status/status_integration_test.go

@@ -32,7 +32,7 @@ import (
 func TestFetch(t *testing.T) {
 	service := compose.EnsureUpWithTimeout(t, 570, "kibana")
 
-	f := mbtest.NewReportingMetricSetV2Error(t, mtest.GetConfig("status", service.Host(), false))
+	f := mbtest.NewReportingMetricSetV2Error(t, mtest.GetConfig("status", service.Host(), false, true))
 	events, errs := mbtest.ReportingFetchV2Error(f)
 
 	require.Empty(t, errs)