Report dataset metrics in monitoring log reporter

CHANGELOG.next.asciidoc

@@ -497,6 +497,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Allow node/namespace metadata to be disabled on kubernetes metagen and ensure add_kubernetes_metadata honors host {pull}23012[23012]
 - Add proxy support for AWS functions. {pull}26832[26832]
 - Added policies to the elasticsearch output for non indexible events {pull}26952[26952]
+- Add `logging.metrics.namespaces` config option to control what metric groups are reported in logs. {pull}25727[25727]
 - Add sha256 digests to RPM packages. {issue}23670[23670]
 
 *Auditbeat*

auditbeat/auditbeat.reference.yml

@@ -161,14 +161,14 @@ auditbeat.modules:
     # Maximum duration after which events are available to the outputs,
     # if the number of events stored in the queue is < `flush.min_events`.
     #flush.timeout: 1s
-  
+
   # The disk queue stores incoming events on disk until the output is
   # ready for them. This allows a higher event limit than the memory-only
   # queue and lets pending events persist through a restart.
   #disk:
     # The directory path to store the queue's data.
     #path: "${path.data}/diskqueue"
-    
+
     # The maximum space the queue should occupy on disk. Depending on
     # input settings, events that exceed this limit are delayed or discarded.
     #max_size: 10GB
@@ -1405,6 +1405,11 @@ setup.kibana:
 # The period after which to log the internal metrics. The default is 30s.
 #logging.metrics.period: 30s
 
+# A list of metrics namespaces to report in the logs. Defaults to [stats].
+# `stats` contains general Beat metrics. `dataset` may be present in some
+# Beats and contains module or input metrics.
+#logging.metrics.namespaces: [stats]
+
 # Logging to rotating files. Set logging.to_files to false to disable logging to
 # files.
 logging.to_files: true

filebeat/filebeat.reference.yml

@@ -1073,14 +1073,14 @@ filebeat.inputs:
     # Maximum duration after which events are available to the outputs,
     # if the number of events stored in the queue is < `flush.min_events`.
     #flush.timeout: 1s
-  
+
   # The disk queue stores incoming events on disk until the output is
   # ready for them. This allows a higher event limit than the memory-only
   # queue and lets pending events persist through a restart.
   #disk:
     # The directory path to store the queue's data.
     #path: "${path.data}/diskqueue"
-    
+
     # The maximum space the queue should occupy on disk. Depending on
     # input settings, events that exceed this limit are delayed or discarded.
     #max_size: 10GB
@@ -2317,6 +2317,11 @@ setup.kibana:
 # The period after which to log the internal metrics. The default is 30s.
 #logging.metrics.period: 30s
 
+# A list of metrics namespaces to report in the logs. Defaults to [stats].
+# `stats` contains general Beat metrics. `dataset` may be present in some
+# Beats and contains module or input metrics.
+#logging.metrics.namespaces: [stats]
+
 # Logging to rotating files. Set logging.to_files to false to disable logging to
 # files.
 logging.to_files: true

heartbeat/heartbeat.reference.yml

@@ -292,14 +292,14 @@ heartbeat.scheduler:
     # Maximum duration after which events are available to the outputs,
     # if the number of events stored in the queue is < `flush.min_events`.
     #flush.timeout: 1s
-  
+
   # The disk queue stores incoming events on disk until the output is
   # ready for them. This allows a higher event limit than the memory-only
   # queue and lets pending events persist through a restart.
   #disk:
     # The directory path to store the queue's data.
     #path: "${path.data}/diskqueue"
-    
+
     # The maximum space the queue should occupy on disk. Depending on
     # input settings, events that exceed this limit are delayed or discarded.
     #max_size: 10GB
@@ -1536,6 +1536,11 @@ setup.kibana:
 # The period after which to log the internal metrics. The default is 30s.
 #logging.metrics.period: 30s
 
+# A list of metrics namespaces to report in the logs. Defaults to [stats].
+# `stats` contains general Beat metrics. `dataset` may be present in some
+# Beats and contains module or input metrics.
+#logging.metrics.namespaces: [stats]
+
 # Logging to rotating files. Set logging.to_files to false to disable logging to
 # files.
 logging.to_files: true

journalbeat/journalbeat.reference.yml

@@ -104,14 +104,14 @@ setup.template.settings:
     # Maximum duration after which events are available to the outputs,
     # if the number of events stored in the queue is < `flush.min_events`.
     #flush.timeout: 1s
-  
+
   # The disk queue stores incoming events on disk until the output is
   # ready for them. This allows a higher event limit than the memory-only
   # queue and lets pending events persist through a restart.
   #disk:
     # The directory path to store the queue's data.
     #path: "${path.data}/diskqueue"
-    
+
     # The maximum space the queue should occupy on disk. Depending on
     # input settings, events that exceed this limit are delayed or discarded.
     #max_size: 10GB
@@ -1348,6 +1348,11 @@ setup.kibana:
 # The period after which to log the internal metrics. The default is 30s.
 #logging.metrics.period: 30s
 
+# A list of metrics namespaces to report in the logs. Defaults to [stats].
+# `stats` contains general Beat metrics. `dataset` may be present in some
+# Beats and contains module or input metrics.
+#logging.metrics.namespaces: [stats]
+
 # Logging to rotating files. Set logging.to_files to false to disable logging to
 # files.
 logging.to_files: true

libbeat/_meta/config/general.reference.yml.tmpl

@@ -41,14 +41,14 @@
     # Maximum duration after which events are available to the outputs,
     # if the number of events stored in the queue is < `flush.min_events`.
     #flush.timeout: 1s
-  
+
   # The disk queue stores incoming events on disk until the output is
   # ready for them. This allows a higher event limit than the memory-only
   # queue and lets pending events persist through a restart.
   #disk:
     # The directory path to store the queue's data.
     #path: "${path.data}/diskqueue"
-    
+
     # The maximum space the queue should occupy on disk. Depending on
     # input settings, events that exceed this limit are delayed or discarded.
     #max_size: 10GB

libbeat/_meta/config/logging.reference.yml.tmpl

@@ -30,6 +30,11 @@
 # The period after which to log the internal metrics. The default is 30s.
 #logging.metrics.period: 30s
 
+# A list of metrics namespaces to report in the logs. Defaults to [stats].
+# `stats` contains general Beat metrics. `dataset` may be present in some
+# Beats and contains module or input metrics.
+#logging.metrics.namespaces: [stats]
+
 # Logging to rotating files. Set logging.to_files to false to disable logging to
 # files.
 logging.to_files: true

libbeat/docs/loggingconfig.asciidoc

@@ -173,7 +173,7 @@ endif::serverless[]
 By default, {beatname_uc} periodically logs its internal metrics that have
 changed in the last period. For each metric that changed, the delta from the
 value at the beginning of the period is logged. Also, the total values for all
-non-zero internal metrics are logged on shutdown. Set this to false to disable 
+non-zero internal metrics are logged on shutdown. Set this to false to disable
 this behavior. The default is true.
 
 Here is an example log line:
@@ -191,6 +191,13 @@ metrics and for this reason they are also not documented.
 
 The period after which to log the internal metrics. The default is 30s.
 
+[float]
+==== `logging.metrics.namespaces`
+
+A list of metrics namespaces to report in the logs. Defaults to `[stats]`.
+`stats` contains general Beat metrics. `dataset` may be present in some
+Beats and contains module or input metrics.
+
 ifndef::serverless[]
 [float]
 ==== `logging.files.path`

libbeat/monitoring/report/log/config.go

@@ -22,9 +22,13 @@ import (
 )
 
 type config struct {
-	Period time.Duration `config:"period"`
+	Period     time.Duration `config:"period"`
+	Namespaces []string      `config:"namespaces"`
 }
 
-var defaultConfig = config{
-	Period: 30 * time.Second,
+func defaultConfig() config {
+	return config{
+		Period:     30 * time.Second,
+		Namespaces: []string{"stats"},
+	}
 }