From 0721f1090ca5ffe822189d957accbe6d76d418b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Ferna=CC=81ndez=20Haro?= Date: Wed, 18 Nov 2020 15:43:21 +0100 Subject: [PATCH] [Metricbeat/Kibana/stats] `exclude_usage=true` by default --- CHANGELOG.next.asciidoc | 1 + metricbeat/module/kibana/config.go | 6 ++++-- metricbeat/module/kibana/metricset.go | 4 +++- metricbeat/module/kibana/mtest/testing.go | 5 ++++- metricbeat/module/kibana/stats/stats.go | 6 +++++- .../module/kibana/stats/stats_integration_test.go | 4 ++-- metricbeat/module/kibana/stats/stats_test.go | 13 ++++++++++++- .../module/kibana/status/status_integration_test.go | 2 +- 8 files changed, 32 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index cdf8d58d31c..4474a90eb89 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -732,6 +732,7 @@ same journal. {pull}18467[18467] *Metricbeat* +- `kibana` module: `stats` metricset now skips usage collection by default. Set `stats.exclude_usage: false` in the config flag to revert these changes. {pull}22654[22654] - Move the windows pdh implementation from perfmon to a shared location in order for future modules/metricsets to make use of. {pull}15503[15503] - Add DynamoDB AWS Metricbeat light module {pull}15097[15097] - Release elb module as GA. {pull}15485[15485] diff --git a/metricbeat/module/kibana/config.go b/metricbeat/module/kibana/config.go index 748ba6bb5e3..5c022f953bc 100644 --- a/metricbeat/module/kibana/config.go +++ b/metricbeat/module/kibana/config.go @@ -19,12 +19,14 @@ package kibana // Config defines the structure for the Kibana module configuration options type Config struct { - XPackEnabled bool `config:"xpack.enabled"` + XPackEnabled bool `config:"xpack.enabled"` + StatsExcludeUsage bool `config:"stats.exclude_usage"` } // DefaultConfig returns the default configuration for the Kibana module func DefaultConfig() Config { return Config{ - XPackEnabled: false, + XPackEnabled: false, + StatsExcludeUsage: true, } } diff --git a/metricbeat/module/kibana/metricset.go b/metricbeat/module/kibana/metricset.go index 1a68669fe58..b504667b4ce 100644 --- a/metricbeat/module/kibana/metricset.go +++ b/metricbeat/module/kibana/metricset.go @@ -24,7 +24,8 @@ import ( // MetricSet can be used to build other metricsets within the Kibana module. type MetricSet struct { mb.BaseMetricSet - XPackEnabled bool + XPackEnabled bool + StatsExcludeUsage bool } // NewMetricSet creates a metricset that can be used to build other metricsets @@ -38,5 +39,6 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { return &MetricSet{ base, config.XPackEnabled, + config.StatsExcludeUsage, }, nil } diff --git a/metricbeat/module/kibana/mtest/testing.go b/metricbeat/module/kibana/mtest/testing.go index 7a96f4c7a3f..e9f65ae0c13 100644 --- a/metricbeat/module/kibana/mtest/testing.go +++ b/metricbeat/module/kibana/mtest/testing.go @@ -18,7 +18,7 @@ package mtest // GetConfig returns config for kibana module -func GetConfig(metricset string, host string, xpackEnabled bool) map[string]interface{} { +func GetConfig(metricset string, host string, xpackEnabled bool, statsExcludeUsage bool) map[string]interface{} { config := map[string]interface{}{ "module": "kibana", "metricsets": []string{metricset}, @@ -27,6 +27,9 @@ func GetConfig(metricset string, host string, xpackEnabled bool) map[string]inte if xpackEnabled { config["xpack.enabled"] = true } + if statsExcludeUsage == false { + config["stats.exclude_usage"] = false + } return config } diff --git a/metricbeat/module/kibana/stats/stats.go b/metricbeat/module/kibana/stats/stats.go index a6e19d50f42..f1eb0ec1c5d 100644 --- a/metricbeat/module/kibana/stats/stats.go +++ b/metricbeat/module/kibana/stats/stats.go @@ -60,6 +60,7 @@ type MetricSet struct { usageLastCollectedOn time.Time usageNextCollectOn time.Time isUsageExcludable bool + statsExcludeUsage bool } // New create a new instance of the MetricSet @@ -148,6 +149,8 @@ func (m *MetricSet) init() error { m.statsHTTP = statsHTTP m.settingsHTTP = settingsHTTP m.isUsageExcludable = kibana.IsUsageExcludable(kibanaVersion) + // Copying the config to an internal property to make tests easier to implement + m.statsExcludeUsage = m.StatsExcludeUsage return nil } @@ -221,5 +224,6 @@ func (m *MetricSet) calculateIntervalMs() int64 { } func (m *MetricSet) shouldCollectUsage(now time.Time) bool { - return now.Sub(m.usageLastCollectedOn) > usageCollectionPeriod && now.Sub(m.usageNextCollectOn) > 0 + return m.statsExcludeUsage == false && // If the config claims we should exclude the usage, there is no need to check for the time + now.Sub(m.usageLastCollectedOn) > usageCollectionPeriod && now.Sub(m.usageNextCollectOn) > 0 } diff --git a/metricbeat/module/kibana/stats/stats_integration_test.go b/metricbeat/module/kibana/stats/stats_integration_test.go index 8bbf584000d..8ac21fd1891 100644 --- a/metricbeat/module/kibana/stats/stats_integration_test.go +++ b/metricbeat/module/kibana/stats/stats_integration_test.go @@ -38,7 +38,7 @@ import ( func TestFetch(t *testing.T) { service := compose.EnsureUpWithTimeout(t, 570, "kibana") - config := mtest.GetConfig("stats", service.Host(), false) + config := mtest.GetConfig("stats", service.Host(), false, true) host := config["hosts"].([]string)[0] version, err := getKibanaVersion(t, host) require.NoError(t, err) @@ -63,7 +63,7 @@ func TestFetch(t *testing.T) { func TestData(t *testing.T) { service := compose.EnsureUp(t, "kibana") - config := mtest.GetConfig("stats", service.Host(), false) + config := mtest.GetConfig("stats", service.Host(), false, true) host := config["hosts"].([]string)[0] version, err := getKibanaVersion(t, host) require.NoError(t, err) diff --git a/metricbeat/module/kibana/stats/stats_test.go b/metricbeat/module/kibana/stats/stats_test.go index 56cbfc17e1f..197924ffe82 100644 --- a/metricbeat/module/kibana/stats/stats_test.go +++ b/metricbeat/module/kibana/stats/stats_test.go @@ -64,7 +64,7 @@ func TestFetchUsage(t *testing.T) { })) defer kib.Close() - config := mtest.GetConfig("stats", kib.URL, true) + config := mtest.GetConfig("stats", kib.URL, true, false) f := mbtest.NewReportingMetricSetV2Error(t, config) @@ -84,22 +84,32 @@ func TestShouldCollectUsage(t *testing.T) { cases := map[string]struct { usageLastCollectedOn time.Time usageNextCollectOn time.Time + statsExcludeUsage bool expectedResult bool }{ "within_usage_collection_period": { usageLastCollectedOn: now.Add(-1 * usageCollectionPeriod), + statsExcludeUsage: false, expectedResult: false, }, "after_usage_collection_period_but_before_next_scheduled_collection": { usageLastCollectedOn: now.Add(-2 * usageCollectionPeriod), usageNextCollectOn: now.Add(3 * time.Hour), + statsExcludeUsage: false, expectedResult: false, }, "after_usage_collection_period_and_after_next_scheduled_collection": { usageLastCollectedOn: now.Add(-2 * usageCollectionPeriod), usageNextCollectOn: now.Add(-1 * time.Hour), + statsExcludeUsage: false, expectedResult: true, }, + "any_time_when_exclude_usage_is_true": { + usageLastCollectedOn: now.Add(-2 * usageCollectionPeriod), + usageNextCollectOn: now.Add(-1 * time.Hour), + statsExcludeUsage: true, + expectedResult: false, + }, } for name, test := range cases { @@ -107,6 +117,7 @@ func TestShouldCollectUsage(t *testing.T) { m := MetricSet{ usageLastCollectedOn: test.usageLastCollectedOn, usageNextCollectOn: test.usageNextCollectOn, + statsExcludeUsage: test.statsExcludeUsage, } actualResult := m.shouldCollectUsage(now) diff --git a/metricbeat/module/kibana/status/status_integration_test.go b/metricbeat/module/kibana/status/status_integration_test.go index 003dd34f452..1770bcce6e0 100644 --- a/metricbeat/module/kibana/status/status_integration_test.go +++ b/metricbeat/module/kibana/status/status_integration_test.go @@ -32,7 +32,7 @@ import ( func TestFetch(t *testing.T) { service := compose.EnsureUpWithTimeout(t, 570, "kibana") - f := mbtest.NewReportingMetricSetV2Error(t, mtest.GetConfig("status", service.Host(), false)) + f := mbtest.NewReportingMetricSetV2Error(t, mtest.GetConfig("status", service.Host(), false, true)) events, errs := mbtest.ReportingFetchV2Error(f) require.Empty(t, errs)