Add metricset.period to Metricbeat events (#13242)

Having the period as part of each event makes it possible for Kibana or ML to predict when the next event is potentially missing or delayed based on the period of the previous events. It can always be that the period changed but as soon as the next event comes in, this can be used as the new expected period. This field is not added to push fetchers, as they don't do periodic fetching. Fixes #12616 Co-authored-by: ruflin <[email protected]>
elastic · Aug 23, 2019 · fea2db3 · fea2db3
1 parent 9cfc4e3
commit fea2db3
Show file tree

Hide file tree

Showing 71 changed files with 18,745 additions and 17,325 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -319,6 +319,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add cgroup memory stats to docker/memory metricset {pull}12916[12916]
 - Add AWS elb metricset. {pull}12952[12952] {issue}11701[11701]
 - Add AWS ebs metricset. {pull}13167[13167] {issue}11699[11699]
+- Add `metricset.period` field with the configured fetching period. {pull}13242[13242] {issue}12616[12616]
 - Add rate metrics for ec2 metricset. {pull}13203[13203]
 
 *Packetbeat*

diff --git a/metricbeat/_meta/fields.common.yml b/metricbeat/_meta/fields.common.yml
@@ -15,6 +15,11 @@
       description: >
         The name of the metricset that generated the event.
 
+    - name: metricset.period
+      type: integer
+      description: >
+        Current data collection period for this event in milliseconds.
+
     - name: process.pgid
       type: long
       description: >

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -3405,6 +3405,16 @@ alias to: event.module
 The name of the metricset that generated the event.
 
 
+--
+
+*`metricset.period`*::
++
+--
+Current data collection period for this event in milliseconds.
+
+
+type: integer
+
 --
 
 *`process.pgid`*::

diff --git a/metricbeat/include/fields/fields.go b/metricbeat/include/fields/fields.go
diff --git a/metricbeat/mb/event.go b/metricbeat/mb/event.go
@@ -44,6 +44,7 @@ type Event struct {
 	Host      string        // Host from which the data was collected.
 	Service   string        // Service type
 	Took      time.Duration // Amount of time it took to collect the event data.
+	Period    time.Duration // Period that is set to retrieve the events
 
 	DisableTimeSeries bool // true if the event doesn't contain timeseries data
 }
@@ -110,17 +111,26 @@ func (e *Event) BeatEvent(module, metricSet string, modifiers ...EventModifier)
 
 // AddMetricSetInfo is an EventModifier that adds information about the
 // MetricSet that generated the event. It will always add the metricset and
-// module names. And it will add the host, namespace, and rtt (round-trip time
-// in microseconds) values if they are non-zero values.
+// module names. And it will add the host, period (in milliseconds), and
+// duration (round-trip time in nanoseconds) values if they are non-zero
+// values.
 //
+//   {
+//     "event": {
+//       "dataset": "apache.status",
+//       "duration": 115,
+//       "module": "apache"
+//     },
+//     "service": {
+//       "address": "127.0.0.1",
+//     },
 //     "metricset": {
-//       "host": "apache",
-//       "module": "apache",
 //       "name": "status",
-//       "rtt": 115
+//       "period": 10000
 //     }
+//   }
+//
 func AddMetricSetInfo(module, metricset string, event *Event) {
-
 	if event.Namespace == "" {
 		event.Namespace = fmt.Sprintf("%s.%s", module, metricset)
 	}
@@ -141,6 +151,9 @@ func AddMetricSetInfo(module, metricset string, event *Event) {
 	if event.Took > 0 {
 		e.Put("event.duration", event.Took/time.Nanosecond)
 	}
+	if event.Period > 0 {
+		e.Put("metricset.period", event.Period/time.Millisecond)
+	}
 
 	if event.RootFields == nil {
 		event.RootFields = e

diff --git a/metricbeat/mb/module/example_test.go b/metricbeat/mb/module/example_test.go
@@ -101,7 +101,8 @@ func ExampleWrapper() {
 	//     }
 	//   },
 	//   "metricset": {
-	//     "name": "eventfetcher"
+	//     "name": "eventfetcher",
+	//     "period": 10000
 	//   },
 	//   "service": {
 	//     "type": "fake"

diff --git a/metricbeat/mb/module/wrapper.go b/metricbeat/mb/module/wrapper.go
@@ -65,6 +65,8 @@ type metricSetWrapper struct {
 	mb.MetricSet
 	module *Wrapper // Parent Module.
 	stats  *stats   // stats for this MetricSet.
+
+	periodic bool // Set to true if this metricset is a periodic fetcher
 }
 
 // stats bundles common metricset stats.
@@ -208,6 +210,9 @@ func (msw *metricSetWrapper) run(done <-chan struct{}, out chan<- beat.Event) {
 // begins a continuous timer scheduled loop to fetch data. To stop the loop the
 // done channel should be closed.
 func (msw *metricSetWrapper) startPeriodicFetching(ctx context.Context, reporter reporter) {
+	// Indicate that it has been started as periodic fetcher
+	msw.periodic = true
+
 	// Fetch immediately.
 	msw.fetch(ctx, reporter)
 
@@ -367,6 +372,9 @@ func (r reporterV2) Event(event mb.Event) bool {
 	if event.Took == 0 && !r.start.IsZero() {
 		event.Took = time.Since(r.start)
 	}
+	if r.msw.periodic {
+		event.Period = r.msw.Module().Config().Period
+	}
 
 	if event.Timestamp.IsZero() {
 		if !r.start.IsZero() {

diff --git a/metricbeat/mb/module/wrapper_test.go b/metricbeat/mb/module/wrapper_test.go
@@ -215,8 +215,7 @@ func TestWrapperOfPushMetricSet(t *testing.T) {
 	<-output
 	close(done)
 
-	// Validate that the channel is closed after receiving the two
-	// initial events.
+	// Validate that the channel is closed after receiving the event.
 	select {
 	case _, ok := <-output:
 		if !ok {
@@ -227,3 +226,47 @@ func TestWrapperOfPushMetricSet(t *testing.T) {
 		}
 	}
 }
+
+func TestPeriodIsAddedToEvent(t *testing.T) {
+	cases := map[string]struct {
+		metricset string
+		hasPeriod bool
+	}{
+		"fetch metricset events should have period": {
+			metricset: eventFetcherName,
+			hasPeriod: true,
+		},
+		"push metricset events should not have period": {
+			metricset: pushMetricSetName,
+			hasPeriod: false,
+		},
+	}
+
+	registry := newTestRegistry(t)
+
+	for title, c := range cases {
+		t.Run(title, func(t *testing.T) {
+			hosts := []string{"alpha"}
+			config := newConfig(t, map[string]interface{}{
+				"module":     moduleName,
+				"metricsets": []string{c.metricset},
+				"hosts":      hosts,
+			})
+
+			m, err := module.NewWrapper(config, registry, module.WithMetricSetInfo())
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			done := make(chan struct{})
+			defer close(done)
+
+			output := m.Start(done)
+
+			event := <-output
+
+			hasPeriod, _ := event.Fields.HasKey("metricset.period")
+			assert.Equal(t, c.hasPeriod, hasPeriod, "has metricset.period in event %+v", event)
+		})
+	}
+}
diff --git a/metricbeat/mb/testing/data_generator.go b/metricbeat/mb/testing/data_generator.go
@@ -189,6 +189,7 @@ func StandardizeEvent(ms mb.MetricSet, e mb.Event, modifiers ...mb.EventModifier
 	e.Timestamp = startTime
 	e.Took = 115 * time.Microsecond
 	e.Host = ms.Host()
+	e.Period = 10 * time.Second
 	if e.Namespace == "" {
 		e.Namespace = ms.Registration().Namespace
 	}

diff --git a/metricbeat/module/apache/status/_meta/data.json b/metricbeat/module/apache/status/_meta/data.json
@@ -57,7 +57,8 @@
         "module": "apache"
     },
     "metricset": {
-        "name": "status"
+        "name": "status",
+        "period": 10000
     },
     "service": {
         "address": "127.0.0.1:55555",

diff --git a/metricbeat/module/apache/status/_meta/testdata/docs.plain-expected.json b/metricbeat/module/apache/status/_meta/testdata/docs.plain-expected.json
@@ -19,7 +19,7 @@
                     "system": 0.2,
                     "user": 0.16
                 },
-                "hostname": "127.0.0.1:34545",
+                "hostname": "127.0.0.1:37579",
                 "load": {
                     "1": 2,
                     "15": 1.91,
@@ -58,7 +58,8 @@
             "module": "apache"
         },
         "metricset": {
-            "name": "status"
+            "name": "status",
+            "period": 10000
         },
         "service": {
             "address": "127.0.0.1:55555",

diff --git a/metricbeat/module/ceph/monitor_health/_meta/data.json b/metricbeat/module/ceph/monitor_health/_meta/data.json
@@ -38,7 +38,8 @@
         "module": "ceph"
     },
     "metricset": {
-        "name": "monitor_health"
+        "name": "monitor_health",
+        "period": 10000
     },
     "service": {
         "address": "127.0.0.1:55555",

diff --git a/metricbeat/module/ceph/monitor_health/_meta/testdata/docs.json-expected.json b/metricbeat/module/ceph/monitor_health/_meta/testdata/docs.json-expected.json
@@ -38,7 +38,8 @@
             "module": "ceph"
         },
         "metricset": {
-            "name": "monitor_health"
+            "name": "monitor_health",
+            "period": 10000
         },
         "service": {
             "address": "127.0.0.1:55555",

diff --git a/metricbeat/module/couchbase/cluster/_meta/data.json b/metricbeat/module/couchbase/cluster/_meta/data.json
@@ -71,7 +71,8 @@
         "module": "couchbase"
     },
     "metricset": {
-        "name": "cluster"
+        "name": "cluster",
+        "period": 10000
     },
     "service": {
         "address": "127.0.0.1:55555",

diff --git a/metricbeat/module/couchbase/cluster/_meta/testdata/4.5.1.json-expected.json b/metricbeat/module/couchbase/cluster/_meta/testdata/4.5.1.json-expected.json
@@ -71,7 +71,8 @@
             "module": "couchbase"
         },
         "metricset": {
-            "name": "cluster"
+            "name": "cluster",
+            "period": 10000
         },
         "service": {
             "address": "127.0.0.1:55555",

diff --git a/metricbeat/module/couchbase/cluster/_meta/testdata/docs.json-expected.json b/metricbeat/module/couchbase/cluster/_meta/testdata/docs.json-expected.json
@@ -71,7 +71,8 @@
             "module": "couchbase"
         },
         "metricset": {
-            "name": "cluster"
+            "name": "cluster",
+            "period": 10000
         },
         "service": {
             "address": "127.0.0.1:55555",

diff --git a/metricbeat/module/couchbase/node/_meta/data.json b/metricbeat/module/couchbase/node/_meta/data.json
@@ -79,7 +79,8 @@
         "module": "couchbase"
     },
     "metricset": {
-        "name": "node"
+        "name": "node",
+        "period": 10000
     },
     "service": {
         "address": "127.0.0.1:55555",

diff --git a/metricbeat/module/couchbase/node/_meta/testdata/4.5.1.json-expected.json b/metricbeat/module/couchbase/node/_meta/testdata/4.5.1.json-expected.json
@@ -79,7 +79,8 @@
             "module": "couchbase"
         },
         "metricset": {
-            "name": "node"
+            "name": "node",
+            "period": 10000
         },
         "service": {
             "address": "127.0.0.1:55555",

diff --git a/metricbeat/module/couchbase/node/_meta/testdata/docs.json-expected.json b/metricbeat/module/couchbase/node/_meta/testdata/docs.json-expected.json
@@ -79,7 +79,8 @@
             "module": "couchbase"
         },
         "metricset": {
-            "name": "node"
+            "name": "node",
+            "period": 10000
         },
         "service": {
             "address": "127.0.0.1:55555",

diff --git a/metricbeat/module/dropwizard/collector/_meta/data.json b/metricbeat/module/dropwizard/collector/_meta/data.json
@@ -43,7 +43,8 @@
         "module": "dropwizard"
     },
     "metricset": {
-        "name": "collector"
+        "name": "collector",
+        "period": 10000
     },
     "service": {
         "address": "127.0.0.1:55555",

diff --git a/metricbeat/module/dropwizard/collector/_meta/testdata/docs.json-expected.json b/metricbeat/module/dropwizard/collector/_meta/testdata/docs.json-expected.json
@@ -43,7 +43,8 @@
             "module": "dropwizard"
         },
         "metricset": {
-            "name": "collector"
+            "name": "collector",
+            "period": 10000
         },
         "service": {
             "address": "127.0.0.1:55555",
@@ -75,7 +76,8 @@
             "module": "dropwizard"
         },
         "metricset": {
-            "name": "collector"
+            "name": "collector",
+            "period": 10000
         },
         "service": {
             "address": "127.0.0.1:55555",

diff --git a/metricbeat/module/etcd/metrics/_meta/testdata/metrics.plain-expected.json b/metricbeat/module/etcd/metrics/_meta/testdata/metrics.plain-expected.json
@@ -94,7 +94,8 @@
             "module": "etcd"
         },
         "metricset": {
-            "name": "metrics"
+            "name": "metrics",
+            "period": 10000
         },
         "service": {
             "address": "127.0.0.1:55555",

diff --git a/metricbeat/module/http/json/_meta/data.json b/metricbeat/module/http/json/_meta/data.json
@@ -11,7 +11,8 @@
         }
     },
     "metricset": {
-        "name": "json"
+        "name": "json",
+        "period": 10000
     },
     "service": {
         "address": "127.0.0.1:55555",

diff --git a/metricbeat/module/http/json/_meta/testdata/docs.json-expected.json b/metricbeat/module/http/json/_meta/testdata/docs.json-expected.json
@@ -11,7 +11,8 @@
             }
         },
         "metricset": {
-            "name": "json"
+            "name": "json",
+            "period": 10000
         },
         "service": {
             "address": "127.0.0.1:55555",

diff --git a/metricbeat/module/kibana/status/_meta/data.json b/metricbeat/module/kibana/status/_meta/data.json
@@ -23,7 +23,8 @@
         }
     },
     "metricset": {
-        "name": "status"
+        "name": "status",
+        "period": 10000
     },
     "service": {
         "address": "127.0.0.1:55555",

diff --git a/metricbeat/module/kibana/status/_meta/testdata/7.0.0.json-expected.json b/metricbeat/module/kibana/status/_meta/testdata/7.0.0.json-expected.json
@@ -23,7 +23,8 @@
             }
         },
         "metricset": {
-            "name": "status"
+            "name": "status",
+            "period": 10000
         },
         "service": {
             "address": "127.0.0.1:55555",

diff --git a/metricbeat/module/kibana/status/_meta/testdata/docs.json-expected.json b/metricbeat/module/kibana/status/_meta/testdata/docs.json-expected.json
@@ -23,7 +23,8 @@
             }
         },
         "metricset": {
-            "name": "status"
+            "name": "status",
+            "period": 10000
         },
         "service": {
             "address": "127.0.0.1:55555",