From 208dbbd617f3cf08c51f19169c663b98d99aea2b Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 14 May 2021 21:03:13 -0400 Subject: [PATCH 1/4] Report dataset metrics in monitoring log reporter This adds periodic reporting of "dataset" metrics to the logs. This can be disabled by configuring `logging.metrics.namespaces`. These are the same metrics available from the HTTP monitoring endpoint under the /dataset path. This primarily improves visibility of Filebeat and Metricbeat because they make use of the "dataset" namespace for input and metricset instance metrics. Prior to 7.10 the Filebeat input metrics were reported in the logged metrics, but this was lost when the metrics moved to the "dataset" namespace. --- CHANGELOG.next.asciidoc | 6 + auditbeat/auditbeat.reference.yml | 3 + filebeat/filebeat.reference.yml | 3 + heartbeat/heartbeat.reference.yml | 3 + journalbeat/journalbeat.reference.yml | 3 + .../_meta/config/logging.reference.yml.tmpl | 3 + libbeat/docs/loggingconfig.asciidoc | 8 +- libbeat/monitoring/report/log/config.go | 10 +- libbeat/monitoring/report/log/log.go | 122 ++++++++++++------ libbeat/monitoring/report/log/log_test.go | 17 +-- libbeat/processors/dns/resolver.go | 4 +- libbeat/tests/system/test_base.py | 2 +- metricbeat/metricbeat.reference.yml | 3 + packetbeat/packetbeat.reference.yml | 3 + winlogbeat/winlogbeat.reference.yml | 3 + x-pack/auditbeat/auditbeat.reference.yml | 3 + x-pack/filebeat/filebeat.reference.yml | 3 + .../functionbeat/functionbeat.reference.yml | 3 + x-pack/heartbeat/heartbeat.reference.yml | 3 + x-pack/metricbeat/metricbeat.reference.yml | 3 + x-pack/osquerybeat/osquerybeat.reference.yml | 3 + x-pack/packetbeat/packetbeat.reference.yml | 3 + x-pack/winlogbeat/winlogbeat.reference.yml | 3 + 23 files changed, 164 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 15aaa4f2848f..614dcbfc9628 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -498,6 +498,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add proxy support for AWS functions. {pull}26832[26832] - Added policies to the elasticsearch output for non indexible events {pull}26952[26952] - Add sha256 digests to RPM packages. {issue}23670[23670] +- Add support for defining explicitly named dynamic templates without path/type match criteria {pull}25422[25422] +- Improve ES output error insights. {pull}25825[25825] +- Add orchestrator.cluster.name/url fields as k8s metadata {pull}26056[26056] +- Libbeat: report beat version to monitoring. {pull}26214[26214] +- Ensure common proxy settings support in HTTP clients: proxy_disabled, proxy_url, proxy_headers and typical environment variables HTTP_PROXY, HTTPS_PROXY, NOPROXY. {pull}25219[25219] +- Add `logging.metrics.namespaces` config option to control what metric groups are reported in logs. {pull}25727[25727] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index f62d0737e00e..3794db33e659 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -1405,6 +1405,9 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s +# A list of metrics namespaces to report in the logs. +#logging.metrics.namespaces: [stats, dataset] + # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index de1917041bd9..de1e6ed5f3df 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -2317,6 +2317,9 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s +# A list of metrics namespaces to report in the logs. +#logging.metrics.namespaces: [stats, dataset] + # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 14597f2a27c5..70cf7aec2cba 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -1536,6 +1536,9 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s +# A list of metrics namespaces to report in the logs. +#logging.metrics.namespaces: [stats, dataset] + # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index dae323d89af0..e7cd78cb3b53 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -1348,6 +1348,9 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s +# A list of metrics namespaces to report in the logs. +#logging.metrics.namespaces: [stats, dataset] + # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true diff --git a/libbeat/_meta/config/logging.reference.yml.tmpl b/libbeat/_meta/config/logging.reference.yml.tmpl index e85e3c0bba37..c41a4e8d99af 100644 --- a/libbeat/_meta/config/logging.reference.yml.tmpl +++ b/libbeat/_meta/config/logging.reference.yml.tmpl @@ -30,6 +30,9 @@ # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s +# A list of metrics namespaces to report in the logs. +#logging.metrics.namespaces: [stats, dataset] + # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true diff --git a/libbeat/docs/loggingconfig.asciidoc b/libbeat/docs/loggingconfig.asciidoc index 3c2083dd0cb0..62fa48855997 100644 --- a/libbeat/docs/loggingconfig.asciidoc +++ b/libbeat/docs/loggingconfig.asciidoc @@ -173,7 +173,7 @@ endif::serverless[] By default, {beatname_uc} periodically logs its internal metrics that have changed in the last period. For each metric that changed, the delta from the value at the beginning of the period is logged. Also, the total values for all -non-zero internal metrics are logged on shutdown. Set this to false to disable +non-zero internal metrics are logged on shutdown. Set this to false to disable this behavior. The default is true. Here is an example log line: @@ -191,6 +191,12 @@ metrics and for this reason they are also not documented. The period after which to log the internal metrics. The default is 30s. +[float] +==== `logging.metrics.namespaces` + +A list of metrics namespaces to report in the logs. Defaults to +`[stats, dataset]`. + ifndef::serverless[] [float] ==== `logging.files.path` diff --git a/libbeat/monitoring/report/log/config.go b/libbeat/monitoring/report/log/config.go index 4b6425187bef..ab601a64e9a6 100644 --- a/libbeat/monitoring/report/log/config.go +++ b/libbeat/monitoring/report/log/config.go @@ -22,9 +22,13 @@ import ( ) type config struct { - Period time.Duration `config:"period"` + Period time.Duration `config:"period"` + Namespaces []string `config:"namespaces"` } -var defaultConfig = config{ - Period: 30 * time.Second, +func defaultConfig() config { + return config{ + Period: 30 * time.Second, + Namespaces: []string{"stats", "dataset"}, + } } diff --git a/libbeat/monitoring/report/log/log.go b/libbeat/monitoring/report/log/log.go index f05214cd932a..eae53fe707bd 100644 --- a/libbeat/monitoring/report/log/log.go +++ b/libbeat/monitoring/report/log/log.go @@ -18,6 +18,7 @@ package log import ( + "strings" "sync" "time" @@ -63,6 +64,18 @@ var gauges = map[string]bool{ "system.load.norm.15": true, } +// isGauge returns true when the given metric key name represents a gauge value. +// Any metric name suffixed in '_gauge' or containing '.histogram.' is +// treated as a gauge. Other metrics can specifically be marked as gauges +// through the list maintained in this package. +func isGauge(key string) bool { + if strings.HasSuffix(key, "_gauge") || strings.Contains(key, ".histogram.") { + return true + } + _, found := gauges[key] + return found +} + // TODO: Change this when gauges are refactored, too. var strConsts = map[string]bool{ "beat.info.ephemeral_id": true, @@ -75,10 +88,10 @@ var ( ) type reporter struct { - wg sync.WaitGroup - done chan struct{} - period time.Duration - registry *monitoring.Registry + config + wg sync.WaitGroup + done chan struct{} + registries map[string]*monitoring.Registry // output logger *logp.Logger @@ -87,7 +100,7 @@ type reporter struct { // MakeReporter returns a new Reporter that periodically reports metrics via // logp. If cfg is nil defaults will be used. func MakeReporter(beat beat.Info, cfg *common.Config) (report.Reporter, error) { - config := defaultConfig + config := defaultConfig() if cfg != nil { if err := cfg.Unpack(&config); err != nil { return nil, err @@ -95,10 +108,21 @@ func MakeReporter(beat beat.Info, cfg *common.Config) (report.Reporter, error) { } r := &reporter{ - done: make(chan struct{}), - period: config.Period, - logger: logp.NewLogger("monitoring"), - registry: monitoring.Default, + config: config, + done: make(chan struct{}), + logger: logp.NewLogger("monitoring"), + registries: map[string]*monitoring.Registry{}, + } + + for _, ns := range r.config.Namespaces { + reg := monitoring.GetNamespace(ns).GetRegistry() + + // That 'stats' namespace is reported as 'metrics' in the Elasticsearch + // reporter so use the same name for consistency. + if ns == "stats" { + ns = "metrics" + } + r.registries[ns] = reg } r.wg.Add(1) @@ -115,16 +139,21 @@ func (r *reporter) Stop() { } func (r *reporter) snapshotLoop() { - r.logger.Infof("Starting metrics logging every %v", r.period) + r.logger.Infof("Starting metrics logging every %v", r.Period) defer r.logger.Infof("Stopping metrics logging.") defer func() { - r.logTotals(makeDeltaSnapshot(monitoring.MakeFlatSnapshot(), makeSnapshot(r.registry))) + snaps := map[string]monitoring.FlatSnapshot{} + for name, reg := range r.registries { + snap := makeSnapshot(reg) + snaps[name] = snap + } + r.logTotals(snaps) }() - ticker := time.NewTicker(r.period) + ticker := time.NewTicker(r.Period) defer ticker.Stop() - var last monitoring.FlatSnapshot + lastSnaps := map[string]monitoring.FlatSnapshot{} for { select { case <-r.done: @@ -132,31 +161,41 @@ func (r *reporter) snapshotLoop() { case <-ticker.C: } - cur := makeSnapshot(r.registry) - delta := makeDeltaSnapshot(last, cur) - last = cur + snaps := make(map[string]monitoring.FlatSnapshot, len(r.registries)) + for name, reg := range r.registries { + snap := makeSnapshot(reg) + lastSnap := lastSnaps[name] + lastSnaps[name] = snap + delta := makeDeltaSnapshot(lastSnap, snap) + snaps[name] = delta + } - r.logSnapshot(delta) + r.logSnapshot(snaps) } } -func (r *reporter) logSnapshot(s monitoring.FlatSnapshot) { - if snapshotLen(s) > 0 { - r.logger.Infow("Non-zero metrics in the last "+r.period.String(), toKeyValuePairs(s)...) +func (r *reporter) logSnapshot(snaps map[string]monitoring.FlatSnapshot) { + var snapsLen int + for _, s := range snaps { + snapsLen += snapshotLen(s) + } + + if snapsLen > 0 { + r.logger.Infow("Non-zero metrics in the last "+r.Period.String(), toKeyValuePairs(snaps)...) return } - r.logger.Infof("No non-zero metrics in the last %v", r.period) + r.logger.Infof("No non-zero metrics in the last %v", r.Period) } -func (r *reporter) logTotals(s monitoring.FlatSnapshot) { - r.logger.Infow("Total non-zero metrics", toKeyValuePairs(s)...) +func (r *reporter) logTotals(snaps map[string]monitoring.FlatSnapshot) { + r.logger.Infow("Total metrics", toKeyValuePairs(snaps)...) r.logger.Infof("Uptime: %v", time.Since(StartTime)) } func makeSnapshot(R *monitoring.Registry) monitoring.FlatSnapshot { mode := monitoring.Full - return monitoring.CollectFlatSnapshot(R, mode, true) + return monitoring.CollectFlatSnapshot(R, mode, false) } func makeDeltaSnapshot(prev, cur monitoring.FlatSnapshot) monitoring.FlatSnapshot { @@ -201,20 +240,27 @@ func snapshotLen(s monitoring.FlatSnapshot) int { return len(s.Bools) + len(s.Floats) + len(s.Ints) + len(s.Strings) } -func toKeyValuePairs(s monitoring.FlatSnapshot) []interface{} { - data := make(common.MapStr, snapshotLen(s)) - for k, v := range s.Bools { - data.Put(k, v) - } - for k, v := range s.Floats { - data.Put(k, v) - } - for k, v := range s.Ints { - data.Put(k, v) - } - for k, v := range s.Strings { - data.Put(k, v) +func toKeyValuePairs(snaps map[string]monitoring.FlatSnapshot) []interface{} { + args := []interface{}{logp.Namespace("monitoring")} + + for name, snap := range snaps { + data := make(common.MapStr, snapshotLen(snap)) + for k, v := range snap.Bools { + data.Put(k, v) + } + for k, v := range snap.Floats { + data.Put(k, v) + } + for k, v := range snap.Ints { + data.Put(k, v) + } + for k, v := range snap.Strings { + data.Put(k, v) + } + if len(data) > 0 { + args = append(args, logp.Reflect(name, data)) + } } - return []interface{}{logp.Namespace("monitoring"), logp.Reflect("metrics", data)} + return args } diff --git a/libbeat/monitoring/report/log/log_test.go b/libbeat/monitoring/report/log/log_test.go index 4d0e215d6c8c..75671459185d 100644 --- a/libbeat/monitoring/report/log/log_test.go +++ b/libbeat/monitoring/report/log/log_test.go @@ -19,7 +19,6 @@ package log import ( "testing" - "time" "github.com/stretchr/testify/assert" @@ -72,18 +71,20 @@ func TestMakeDeltaSnapshot(t *testing.T) { func TestReporterLog(t *testing.T) { logp.DevelopmentSetup(logp.ToObserverOutput()) - reporter := reporter{period: 30 * time.Second, logger: logp.NewLogger("monitoring")} + reporter := reporter{config: defaultConfig(), logger: logp.NewLogger("monitoring")} - reporter.logSnapshot(monitoring.FlatSnapshot{}) + reporter.logSnapshot(map[string]monitoring.FlatSnapshot{}) logs := logp.ObserverLogs().TakeAll() if assert.Len(t, logs, 1) { assert.Equal(t, "No non-zero metrics in the last 30s", logs[0].Message) } reporter.logSnapshot( - monitoring.FlatSnapshot{ - Bools: map[string]bool{ - "running": true, + map[string]monitoring.FlatSnapshot{ + "metrics": monitoring.FlatSnapshot{ + Bools: map[string]bool{ + "running": true, + }, }, }, ) @@ -93,10 +94,10 @@ func TestReporterLog(t *testing.T) { assertMapHas(t, logs[0].ContextMap(), "monitoring.metrics.running", true) } - reporter.logTotals(curSnap) + reporter.logTotals(map[string]monitoring.FlatSnapshot{"metrics": curSnap}) logs = logp.ObserverLogs().TakeAll() if assert.Len(t, logs, 2) { - assert.Equal(t, "Total non-zero metrics", logs[0].Message) + assert.Equal(t, "Total metrics", logs[0].Message) assertMapHas(t, logs[0].ContextMap(), "monitoring.metrics.count", 20) assertMapHas(t, logs[0].ContextMap(), "monitoring.metrics.new", 1) assert.Contains(t, logs[1].Message, "Uptime: ") diff --git a/libbeat/processors/dns/resolver.go b/libbeat/processors/dns/resolver.go index 366f00e165f9..5a259baffafe 100644 --- a/libbeat/processors/dns/resolver.go +++ b/libbeat/processors/dns/resolver.go @@ -216,8 +216,8 @@ func (res *MiekgResolver) getOrCreateNameserverStats(ns string) *nameserverStats failure: monitoring.NewInt(reg, "failure"), ptrResponse: metrics.NewUniformSample(1028), } - adapter.NewGoMetrics(reg, "response", adapter.Accept). - Register("ptr", metrics.NewHistogram(stats.ptrResponse)) + adapter.NewGoMetrics(reg, "response.ptr", adapter.Accept). + Register("histogram", metrics.NewHistogram(stats.ptrResponse)) res.nsStats[ns] = stats return stats diff --git a/libbeat/tests/system/test_base.py b/libbeat/tests/system/test_base.py index 330b831041a9..97615e7ec00e 100644 --- a/libbeat/tests/system/test_base.py +++ b/libbeat/tests/system/test_base.py @@ -138,7 +138,7 @@ def test_logging_metrics(self): max_timeout=2) proc.check_kill_and_wait() self.wait_until( - lambda: self.log_contains("Total non-zero metrics"), + lambda: self.log_contains("Total metrics"), max_timeout=2) def test_persistent_uuid(self): diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index c3012225a8e0..7d4dad67049a 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -2217,6 +2217,9 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s +# A list of metrics namespaces to report in the logs. +#logging.metrics.namespaces: [stats, dataset] + # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 2dd7bdf9e6dc..e2125dce1942 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1900,6 +1900,9 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s +# A list of metrics namespaces to report in the logs. +#logging.metrics.namespaces: [stats, dataset] + # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 809c68e6ad9a..844c74d93527 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -1328,6 +1328,9 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s +# A list of metrics namespaces to report in the logs. +#logging.metrics.namespaces: [stats, dataset] + # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index 616112a7d8be..af6453be0d36 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -1461,6 +1461,9 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s +# A list of metrics namespaces to report in the logs. +#logging.metrics.namespaces: [stats, dataset] + # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 9a797d337fcd..65de0730d48e 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -4536,6 +4536,9 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s +# A list of metrics namespaces to report in the logs. +#logging.metrics.namespaces: [stats, dataset] + # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index 5a760f97046a..b7e7a07c748a 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -1307,6 +1307,9 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s +# A list of metrics namespaces to report in the logs. +#logging.metrics.namespaces: [stats, dataset] + # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index 14597f2a27c5..70cf7aec2cba 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -1536,6 +1536,9 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s +# A list of metrics namespaces to report in the logs. +#logging.metrics.namespaces: [stats, dataset] + # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 65f84166d726..1eb54db5acc1 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -2738,6 +2738,9 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s +# A list of metrics namespaces to report in the logs. +#logging.metrics.namespaces: [stats, dataset] + # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml index eb98402f328f..c07e147a1fee 100644 --- a/x-pack/osquerybeat/osquerybeat.reference.yml +++ b/x-pack/osquerybeat/osquerybeat.reference.yml @@ -920,6 +920,9 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s +# A list of metrics namespaces to report in the logs. +#logging.metrics.namespaces: [stats, dataset] + # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index 2dd7bdf9e6dc..e2125dce1942 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -1900,6 +1900,9 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s +# A list of metrics namespaces to report in the logs. +#logging.metrics.namespaces: [stats, dataset] + # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 3b94dd6d807e..c1784126fa68 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -1371,6 +1371,9 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s +# A list of metrics namespaces to report in the logs. +#logging.metrics.namespaces: [stats, dataset] + # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true From 51fc35b552434b4a74f295f80892cf695b8ebb7c Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 11 Aug 2021 23:44:22 -0400 Subject: [PATCH 2/4] Disable dataset by default --- auditbeat/auditbeat.reference.yml | 10 ++++++---- filebeat/filebeat.reference.yml | 10 ++++++---- heartbeat/heartbeat.reference.yml | 10 ++++++---- journalbeat/journalbeat.reference.yml | 10 ++++++---- libbeat/_meta/config/general.reference.yml.tmpl | 4 ++-- libbeat/_meta/config/logging.reference.yml.tmpl | 6 ++++-- libbeat/docs/loggingconfig.asciidoc | 5 +++-- libbeat/monitoring/report/log/config.go | 2 +- metricbeat/metricbeat.reference.yml | 10 ++++++---- packetbeat/packetbeat.reference.yml | 10 ++++++---- winlogbeat/winlogbeat.reference.yml | 10 ++++++---- x-pack/auditbeat/auditbeat.reference.yml | 10 ++++++---- x-pack/filebeat/filebeat.reference.yml | 10 ++++++---- x-pack/functionbeat/functionbeat.reference.yml | 10 ++++++---- x-pack/heartbeat/heartbeat.reference.yml | 10 ++++++---- x-pack/metricbeat/metricbeat.reference.yml | 10 ++++++---- x-pack/osquerybeat/osquerybeat.reference.yml | 10 ++++++---- x-pack/packetbeat/packetbeat.reference.yml | 10 ++++++---- x-pack/winlogbeat/winlogbeat.reference.yml | 10 ++++++---- 19 files changed, 100 insertions(+), 67 deletions(-) diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 3794db33e659..cbd799ce1627 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -161,14 +161,14 @@ auditbeat.modules: # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < `flush.min_events`. #flush.timeout: 1s - + # The disk queue stores incoming events on disk until the output is # ready for them. This allows a higher event limit than the memory-only # queue and lets pending events persist through a restart. #disk: # The directory path to store the queue's data. #path: "${path.data}/diskqueue" - + # The maximum space the queue should occupy on disk. Depending on # input settings, events that exceed this limit are delayed or discarded. #max_size: 10GB @@ -1405,8 +1405,10 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# A list of metrics namespaces to report in the logs. -#logging.metrics.namespaces: [stats, dataset] +# A list of metrics namespaces to report in the logs. Defaults to [stats]. +# `stats` contains general Beat metrics. `dataset` may be present in some +# Beats and contains module or input metrics. +#logging.metrics.namespaces: [stats] # Logging to rotating files. Set logging.to_files to false to disable logging to # files. diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index de1e6ed5f3df..aa3b9149a66a 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1073,14 +1073,14 @@ filebeat.inputs: # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < `flush.min_events`. #flush.timeout: 1s - + # The disk queue stores incoming events on disk until the output is # ready for them. This allows a higher event limit than the memory-only # queue and lets pending events persist through a restart. #disk: # The directory path to store the queue's data. #path: "${path.data}/diskqueue" - + # The maximum space the queue should occupy on disk. Depending on # input settings, events that exceed this limit are delayed or discarded. #max_size: 10GB @@ -2317,8 +2317,10 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# A list of metrics namespaces to report in the logs. -#logging.metrics.namespaces: [stats, dataset] +# A list of metrics namespaces to report in the logs. Defaults to [stats]. +# `stats` contains general Beat metrics. `dataset` may be present in some +# Beats and contains module or input metrics. +#logging.metrics.namespaces: [stats] # Logging to rotating files. Set logging.to_files to false to disable logging to # files. diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 70cf7aec2cba..dac3b877fd95 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -292,14 +292,14 @@ heartbeat.scheduler: # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < `flush.min_events`. #flush.timeout: 1s - + # The disk queue stores incoming events on disk until the output is # ready for them. This allows a higher event limit than the memory-only # queue and lets pending events persist through a restart. #disk: # The directory path to store the queue's data. #path: "${path.data}/diskqueue" - + # The maximum space the queue should occupy on disk. Depending on # input settings, events that exceed this limit are delayed or discarded. #max_size: 10GB @@ -1536,8 +1536,10 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# A list of metrics namespaces to report in the logs. -#logging.metrics.namespaces: [stats, dataset] +# A list of metrics namespaces to report in the logs. Defaults to [stats]. +# `stats` contains general Beat metrics. `dataset` may be present in some +# Beats and contains module or input metrics. +#logging.metrics.namespaces: [stats] # Logging to rotating files. Set logging.to_files to false to disable logging to # files. diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index e7cd78cb3b53..85393dca55b5 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -104,14 +104,14 @@ setup.template.settings: # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < `flush.min_events`. #flush.timeout: 1s - + # The disk queue stores incoming events on disk until the output is # ready for them. This allows a higher event limit than the memory-only # queue and lets pending events persist through a restart. #disk: # The directory path to store the queue's data. #path: "${path.data}/diskqueue" - + # The maximum space the queue should occupy on disk. Depending on # input settings, events that exceed this limit are delayed or discarded. #max_size: 10GB @@ -1348,8 +1348,10 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# A list of metrics namespaces to report in the logs. -#logging.metrics.namespaces: [stats, dataset] +# A list of metrics namespaces to report in the logs. Defaults to [stats]. +# `stats` contains general Beat metrics. `dataset` may be present in some +# Beats and contains module or input metrics. +#logging.metrics.namespaces: [stats] # Logging to rotating files. Set logging.to_files to false to disable logging to # files. diff --git a/libbeat/_meta/config/general.reference.yml.tmpl b/libbeat/_meta/config/general.reference.yml.tmpl index 60d15e686389..cc266128c025 100644 --- a/libbeat/_meta/config/general.reference.yml.tmpl +++ b/libbeat/_meta/config/general.reference.yml.tmpl @@ -41,14 +41,14 @@ # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < `flush.min_events`. #flush.timeout: 1s - + # The disk queue stores incoming events on disk until the output is # ready for them. This allows a higher event limit than the memory-only # queue and lets pending events persist through a restart. #disk: # The directory path to store the queue's data. #path: "${path.data}/diskqueue" - + # The maximum space the queue should occupy on disk. Depending on # input settings, events that exceed this limit are delayed or discarded. #max_size: 10GB diff --git a/libbeat/_meta/config/logging.reference.yml.tmpl b/libbeat/_meta/config/logging.reference.yml.tmpl index c41a4e8d99af..556ae8bc8dc3 100644 --- a/libbeat/_meta/config/logging.reference.yml.tmpl +++ b/libbeat/_meta/config/logging.reference.yml.tmpl @@ -30,8 +30,10 @@ # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# A list of metrics namespaces to report in the logs. -#logging.metrics.namespaces: [stats, dataset] +# A list of metrics namespaces to report in the logs. Defaults to [stats]. +# `stats` contains general Beat metrics. `dataset` may be present in some +# Beats and contains module or input metrics. +#logging.metrics.namespaces: [stats] # Logging to rotating files. Set logging.to_files to false to disable logging to # files. diff --git a/libbeat/docs/loggingconfig.asciidoc b/libbeat/docs/loggingconfig.asciidoc index 62fa48855997..49ec4beaa110 100644 --- a/libbeat/docs/loggingconfig.asciidoc +++ b/libbeat/docs/loggingconfig.asciidoc @@ -194,8 +194,9 @@ The period after which to log the internal metrics. The default is 30s. [float] ==== `logging.metrics.namespaces` -A list of metrics namespaces to report in the logs. Defaults to -`[stats, dataset]`. +A list of metrics namespaces to report in the logs. Defaults to `[stats]`. +`stats` contains general Beat metrics. `dataset` may be present in some +Beats and contains module or input metrics. ifndef::serverless[] [float] diff --git a/libbeat/monitoring/report/log/config.go b/libbeat/monitoring/report/log/config.go index ab601a64e9a6..34ee7d84bb74 100644 --- a/libbeat/monitoring/report/log/config.go +++ b/libbeat/monitoring/report/log/config.go @@ -29,6 +29,6 @@ type config struct { func defaultConfig() config { return config{ Period: 30 * time.Second, - Namespaces: []string{"stats", "dataset"}, + Namespaces: []string{"stats"}, } } diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 7d4dad67049a..7a10d5d269df 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -973,14 +973,14 @@ metricbeat.modules: # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < `flush.min_events`. #flush.timeout: 1s - + # The disk queue stores incoming events on disk until the output is # ready for them. This allows a higher event limit than the memory-only # queue and lets pending events persist through a restart. #disk: # The directory path to store the queue's data. #path: "${path.data}/diskqueue" - + # The maximum space the queue should occupy on disk. Depending on # input settings, events that exceed this limit are delayed or discarded. #max_size: 10GB @@ -2217,8 +2217,10 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# A list of metrics namespaces to report in the logs. -#logging.metrics.namespaces: [stats, dataset] +# A list of metrics namespaces to report in the logs. Defaults to [stats]. +# `stats` contains general Beat metrics. `dataset` may be present in some +# Beats and contains module or input metrics. +#logging.metrics.namespaces: [stats] # Logging to rotating files. Set logging.to_files to false to disable logging to # files. diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index e2125dce1942..8e67ba141ece 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -656,14 +656,14 @@ packetbeat.ignore_outgoing: false # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < `flush.min_events`. #flush.timeout: 1s - + # The disk queue stores incoming events on disk until the output is # ready for them. This allows a higher event limit than the memory-only # queue and lets pending events persist through a restart. #disk: # The directory path to store the queue's data. #path: "${path.data}/diskqueue" - + # The maximum space the queue should occupy on disk. Depending on # input settings, events that exceed this limit are delayed or discarded. #max_size: 10GB @@ -1900,8 +1900,10 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# A list of metrics namespaces to report in the logs. -#logging.metrics.namespaces: [stats, dataset] +# A list of metrics namespaces to report in the logs. Defaults to [stats]. +# `stats` contains general Beat metrics. `dataset` may be present in some +# Beats and contains module or input metrics. +#logging.metrics.namespaces: [stats] # Logging to rotating files. Set logging.to_files to false to disable logging to # files. diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 844c74d93527..d895016d63f9 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -84,14 +84,14 @@ winlogbeat.event_logs: # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < `flush.min_events`. #flush.timeout: 1s - + # The disk queue stores incoming events on disk until the output is # ready for them. This allows a higher event limit than the memory-only # queue and lets pending events persist through a restart. #disk: # The directory path to store the queue's data. #path: "${path.data}/diskqueue" - + # The maximum space the queue should occupy on disk. Depending on # input settings, events that exceed this limit are delayed or discarded. #max_size: 10GB @@ -1328,8 +1328,10 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# A list of metrics namespaces to report in the logs. -#logging.metrics.namespaces: [stats, dataset] +# A list of metrics namespaces to report in the logs. Defaults to [stats]. +# `stats` contains general Beat metrics. `dataset` may be present in some +# Beats and contains module or input metrics. +#logging.metrics.namespaces: [stats] # Logging to rotating files. Set logging.to_files to false to disable logging to # files. diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index af6453be0d36..63ec6f03a40b 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -217,14 +217,14 @@ auditbeat.modules: # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < `flush.min_events`. #flush.timeout: 1s - + # The disk queue stores incoming events on disk until the output is # ready for them. This allows a higher event limit than the memory-only # queue and lets pending events persist through a restart. #disk: # The directory path to store the queue's data. #path: "${path.data}/diskqueue" - + # The maximum space the queue should occupy on disk. Depending on # input settings, events that exceed this limit are delayed or discarded. #max_size: 10GB @@ -1461,8 +1461,10 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# A list of metrics namespaces to report in the logs. -#logging.metrics.namespaces: [stats, dataset] +# A list of metrics namespaces to report in the logs. Defaults to [stats]. +# `stats` contains general Beat metrics. `dataset` may be present in some +# Beats and contains module or input metrics. +#logging.metrics.namespaces: [stats] # Logging to rotating files. Set logging.to_files to false to disable logging to # files. diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 65de0730d48e..62aca03a64a0 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -3292,14 +3292,14 @@ filebeat.inputs: # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < `flush.min_events`. #flush.timeout: 1s - + # The disk queue stores incoming events on disk until the output is # ready for them. This allows a higher event limit than the memory-only # queue and lets pending events persist through a restart. #disk: # The directory path to store the queue's data. #path: "${path.data}/diskqueue" - + # The maximum space the queue should occupy on disk. Depending on # input settings, events that exceed this limit are delayed or discarded. #max_size: 10GB @@ -4536,8 +4536,10 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# A list of metrics namespaces to report in the logs. -#logging.metrics.namespaces: [stats, dataset] +# A list of metrics namespaces to report in the logs. Defaults to [stats]. +# `stats` contains general Beat metrics. `dataset` may be present in some +# Beats and contains module or input metrics. +#logging.metrics.namespaces: [stats] # Logging to rotating files. Set logging.to_files to false to disable logging to # files. diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index b7e7a07c748a..0b1397cab757 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -447,14 +447,14 @@ functionbeat.provider.gcp.functions: # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < `flush.min_events`. #flush.timeout: 1s - + # The disk queue stores incoming events on disk until the output is # ready for them. This allows a higher event limit than the memory-only # queue and lets pending events persist through a restart. #disk: # The directory path to store the queue's data. #path: "${path.data}/diskqueue" - + # The maximum space the queue should occupy on disk. Depending on # input settings, events that exceed this limit are delayed or discarded. #max_size: 10GB @@ -1307,8 +1307,10 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# A list of metrics namespaces to report in the logs. -#logging.metrics.namespaces: [stats, dataset] +# A list of metrics namespaces to report in the logs. Defaults to [stats]. +# `stats` contains general Beat metrics. `dataset` may be present in some +# Beats and contains module or input metrics. +#logging.metrics.namespaces: [stats] # Logging to rotating files. Set logging.to_files to false to disable logging to # files. diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index 70cf7aec2cba..dac3b877fd95 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -292,14 +292,14 @@ heartbeat.scheduler: # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < `flush.min_events`. #flush.timeout: 1s - + # The disk queue stores incoming events on disk until the output is # ready for them. This allows a higher event limit than the memory-only # queue and lets pending events persist through a restart. #disk: # The directory path to store the queue's data. #path: "${path.data}/diskqueue" - + # The maximum space the queue should occupy on disk. Depending on # input settings, events that exceed this limit are delayed or discarded. #max_size: 10GB @@ -1536,8 +1536,10 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# A list of metrics namespaces to report in the logs. -#logging.metrics.namespaces: [stats, dataset] +# A list of metrics namespaces to report in the logs. Defaults to [stats]. +# `stats` contains general Beat metrics. `dataset` may be present in some +# Beats and contains module or input metrics. +#logging.metrics.namespaces: [stats] # Logging to rotating files. Set logging.to_files to false to disable logging to # files. diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 1eb54db5acc1..971d720b10aa 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1494,14 +1494,14 @@ metricbeat.modules: # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < `flush.min_events`. #flush.timeout: 1s - + # The disk queue stores incoming events on disk until the output is # ready for them. This allows a higher event limit than the memory-only # queue and lets pending events persist through a restart. #disk: # The directory path to store the queue's data. #path: "${path.data}/diskqueue" - + # The maximum space the queue should occupy on disk. Depending on # input settings, events that exceed this limit are delayed or discarded. #max_size: 10GB @@ -2738,8 +2738,10 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# A list of metrics namespaces to report in the logs. -#logging.metrics.namespaces: [stats, dataset] +# A list of metrics namespaces to report in the logs. Defaults to [stats]. +# `stats` contains general Beat metrics. `dataset` may be present in some +# Beats and contains module or input metrics. +#logging.metrics.namespaces: [stats] # Logging to rotating files. Set logging.to_files to false to disable logging to # files. diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml index c07e147a1fee..dce18b7079a2 100644 --- a/x-pack/osquerybeat/osquerybeat.reference.yml +++ b/x-pack/osquerybeat/osquerybeat.reference.yml @@ -60,14 +60,14 @@ seccomp.enabled: false # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < `flush.min_events`. #flush.timeout: 1s - + # The disk queue stores incoming events on disk until the output is # ready for them. This allows a higher event limit than the memory-only # queue and lets pending events persist through a restart. #disk: # The directory path to store the queue's data. #path: "${path.data}/diskqueue" - + # The maximum space the queue should occupy on disk. Depending on # input settings, events that exceed this limit are delayed or discarded. #max_size: 10GB @@ -920,8 +920,10 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# A list of metrics namespaces to report in the logs. -#logging.metrics.namespaces: [stats, dataset] +# A list of metrics namespaces to report in the logs. Defaults to [stats]. +# `stats` contains general Beat metrics. `dataset` may be present in some +# Beats and contains module or input metrics. +#logging.metrics.namespaces: [stats] # Logging to rotating files. Set logging.to_files to false to disable logging to # files. diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index e2125dce1942..8e67ba141ece 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -656,14 +656,14 @@ packetbeat.ignore_outgoing: false # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < `flush.min_events`. #flush.timeout: 1s - + # The disk queue stores incoming events on disk until the output is # ready for them. This allows a higher event limit than the memory-only # queue and lets pending events persist through a restart. #disk: # The directory path to store the queue's data. #path: "${path.data}/diskqueue" - + # The maximum space the queue should occupy on disk. Depending on # input settings, events that exceed this limit are delayed or discarded. #max_size: 10GB @@ -1900,8 +1900,10 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# A list of metrics namespaces to report in the logs. -#logging.metrics.namespaces: [stats, dataset] +# A list of metrics namespaces to report in the logs. Defaults to [stats]. +# `stats` contains general Beat metrics. `dataset` may be present in some +# Beats and contains module or input metrics. +#logging.metrics.namespaces: [stats] # Logging to rotating files. Set logging.to_files to false to disable logging to # files. diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index c1784126fa68..6ea0efb08acb 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -127,14 +127,14 @@ winlogbeat.event_logs: # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < `flush.min_events`. #flush.timeout: 1s - + # The disk queue stores incoming events on disk until the output is # ready for them. This allows a higher event limit than the memory-only # queue and lets pending events persist through a restart. #disk: # The directory path to store the queue's data. #path: "${path.data}/diskqueue" - + # The maximum space the queue should occupy on disk. Depending on # input settings, events that exceed this limit are delayed or discarded. #max_size: 10GB @@ -1371,8 +1371,10 @@ setup.kibana: # The period after which to log the internal metrics. The default is 30s. #logging.metrics.period: 30s -# A list of metrics namespaces to report in the logs. -#logging.metrics.namespaces: [stats, dataset] +# A list of metrics namespaces to report in the logs. Defaults to [stats]. +# `stats` contains general Beat metrics. `dataset` may be present in some +# Beats and contains module or input metrics. +#logging.metrics.namespaces: [stats] # Logging to rotating files. Set logging.to_files to false to disable logging to # files. From ae08c7a4017048075dcb2ff05bd2381f666cd6cd Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 11 Aug 2021 23:47:02 -0400 Subject: [PATCH 3/4] Fix changelog --- CHANGELOG.next.asciidoc | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 614dcbfc9628..131312724bda 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -497,13 +497,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Allow node/namespace metadata to be disabled on kubernetes metagen and ensure add_kubernetes_metadata honors host {pull}23012[23012] - Add proxy support for AWS functions. {pull}26832[26832] - Added policies to the elasticsearch output for non indexible events {pull}26952[26952] -- Add sha256 digests to RPM packages. {issue}23670[23670] -- Add support for defining explicitly named dynamic templates without path/type match criteria {pull}25422[25422] -- Improve ES output error insights. {pull}25825[25825] -- Add orchestrator.cluster.name/url fields as k8s metadata {pull}26056[26056] -- Libbeat: report beat version to monitoring. {pull}26214[26214] -- Ensure common proxy settings support in HTTP clients: proxy_disabled, proxy_url, proxy_headers and typical environment variables HTTP_PROXY, HTTPS_PROXY, NOPROXY. {pull}25219[25219] - Add `logging.metrics.namespaces` config option to control what metric groups are reported in logs. {pull}25727[25727] +- Add sha256 digests to RPM packages. {issue}23670[23670] *Auditbeat* From 87b1c072f0d56c46c3d7a99c063c852b3d77869b Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 12 Aug 2021 07:32:48 -0400 Subject: [PATCH 4/4] Verify isGauge is used with a test --- libbeat/monitoring/report/log/log.go | 4 ++-- libbeat/monitoring/report/log/log_test.go | 22 ++++++++++++++-------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/libbeat/monitoring/report/log/log.go b/libbeat/monitoring/report/log/log.go index eae53fe707bd..edce95b8ff0a 100644 --- a/libbeat/monitoring/report/log/log.go +++ b/libbeat/monitoring/report/log/log.go @@ -216,7 +216,7 @@ func makeDeltaSnapshot(prev, cur monitoring.FlatSnapshot) monitoring.FlatSnapsho } for k, i := range cur.Ints { - if _, found := gauges[k]; found { + if isGauge(k) { delta.Ints[k] = i } else { if p := prev.Ints[k]; p != i { @@ -226,7 +226,7 @@ func makeDeltaSnapshot(prev, cur monitoring.FlatSnapshot) monitoring.FlatSnapsho } for k, f := range cur.Floats { - if _, found := gauges[k]; found { + if isGauge(k) { delta.Floats[k] = f } else if p := prev.Floats[k]; p != f { delta.Floats[k] = f - p diff --git a/libbeat/monitoring/report/log/log_test.go b/libbeat/monitoring/report/log/log_test.go index 75671459185d..70a15cd5e1b7 100644 --- a/libbeat/monitoring/report/log/log_test.go +++ b/libbeat/monitoring/report/log/log_test.go @@ -31,22 +31,26 @@ import ( var ( prevSnap = monitoring.FlatSnapshot{ Ints: map[string]int64{ - "count": 10, - "gone": 1, + "count": 10, + "gone": 1, + "active_gauge": 6, }, Floats: map[string]float64{ - "system.load.1": 2.0, - "float_counter": 1, + "system.load.1": 2.0, + "float_counter": 1, + "foo.histogram.p99": 4.0, }, } curSnap = monitoring.FlatSnapshot{ Ints: map[string]int64{ - "count": 20, - "new": 1, + "count": 20, + "new": 1, + "active_gauge": 5, }, Floats: map[string]float64{ - "system.load.1": 1.2, - "float_counter": 3, + "system.load.1": 1.2, + "float_counter": 3, + "foo.histogram.p99": 4.1, }, } ) @@ -66,6 +70,8 @@ func TestMakeDeltaSnapshot(t *testing.T) { assert.EqualValues(t, 1, delta.Ints["new"]) assert.EqualValues(t, 1.2, delta.Floats["system.load.1"]) assert.EqualValues(t, 2, delta.Floats["float_counter"]) + assert.EqualValues(t, 5, delta.Ints["active_gauge"]) + assert.EqualValues(t, 4.1, delta.Floats["foo.histogram.p99"]) assert.NotContains(t, delta.Ints, "gone") }