From 5e514dc999d9cfc776d56f139f9479dd441a08b4 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 11 Aug 2017 17:33:51 -0400 Subject: [PATCH 1/3] Add system uptime metricset This adds a new metricset that reports the system uptime. The metricset is enabled by default with a period of 15m. The metricset reports one field `system.uptime.duration.ms` that is the system's uptime reported in milliseconds. In Kibana the field is formatted as a `duration` and shown in human-readable format (e.g. "8 days"). Closes #4848 --- CHANGELOG.asciidoc | 2 + libbeat/scripts/generate_index_pattern.py | 5 +++ metricbeat/docs/fields.asciidoc | 17 ++++++++ metricbeat/docs/modules/system.asciidoc | 8 ++++ .../docs/modules/system/uptime.asciidoc | 19 +++++++++ metricbeat/include/list.go | 1 + metricbeat/metricbeat.reference.yml | 1 + .../module/system/_meta/config.reference.yml | 1 + metricbeat/module/system/_meta/config.yml | 4 ++ .../module/system/uptime/_meta/data.json | 23 ++++++++++ .../module/system/uptime/_meta/docs.asciidoc | 10 +++++ .../module/system/uptime/_meta/fields.yml | 11 +++++ metricbeat/module/system/uptime/doc.go | 2 + metricbeat/module/system/uptime/metricset.go | 42 +++++++++++++++++++ .../module/system/uptime/metricset_test.go | 28 +++++++++++++ metricbeat/modules.d/system.yml | 4 ++ 16 files changed, 178 insertions(+) create mode 100644 metricbeat/docs/modules/system/uptime.asciidoc create mode 100644 metricbeat/module/system/uptime/_meta/data.json create mode 100644 metricbeat/module/system/uptime/_meta/docs.asciidoc create mode 100644 metricbeat/module/system/uptime/_meta/fields.yml create mode 100644 metricbeat/module/system/uptime/doc.go create mode 100644 metricbeat/module/system/uptime/metricset.go create mode 100644 metricbeat/module/system/uptime/metricset_test.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 3c9b66414b5c..bd39b4d656bd 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -83,6 +83,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta1...master[Check the HEAD di - Add http server metricset to support push metrics via http. {pull}4770[4770] - Make config object public for graphite and http server {pull}4820[4820] +- Add system uptime metricset. {issue}[4848[4848] + *Packetbeat* *Winlogbeat* diff --git a/libbeat/scripts/generate_index_pattern.py b/libbeat/scripts/generate_index_pattern.py index f48068e806f1..7ad6c2be65b1 100644 --- a/libbeat/scripts/generate_index_pattern.py +++ b/libbeat/scripts/generate_index_pattern.py @@ -79,6 +79,11 @@ def field_to_json(fields, desc, path, output, if "format" in desc: fieldFormat["id"] = desc["format"] + if "input_format" in desc: + fieldFormat["params"] = { + "inputFormat": desc["input_format"] + } + if "pattern" in desc: fieldFormat["params"] = { "pattern": desc["pattern"] diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 08191e8c8ac6..4a117b5ab83b 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -10144,6 +10144,23 @@ type: keyword Name of the user running the process. +[float] +== uptime fields + +`uptime` contains the operating system uptime metric. + + + +[float] +=== `system.uptime.duration.ms` + +type: long + +format: duration + +The OS uptime in milliseconds. + + [[exported-fields-vsphere]] == vSphere fields diff --git a/metricbeat/docs/modules/system.asciidoc b/metricbeat/docs/modules/system.asciidoc index d5b9a50f447b..fc8bb49658fa 100644 --- a/metricbeat/docs/modules/system.asciidoc +++ b/metricbeat/docs/modules/system.asciidoc @@ -109,6 +109,10 @@ metricbeat.modules: processors: - drop_event.when.regexp: system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' + +- module: system + period: 15m + metricsets: [uptime] ---- [float] @@ -138,6 +142,8 @@ The following metricsets are available: * <> +* <> + include::system/core.asciidoc[] include::system/cpu.asciidoc[] @@ -160,3 +166,5 @@ include::system/process_summary.asciidoc[] include::system/socket.asciidoc[] +include::system/uptime.asciidoc[] + diff --git a/metricbeat/docs/modules/system/uptime.asciidoc b/metricbeat/docs/modules/system/uptime.asciidoc new file mode 100644 index 000000000000..9087b0be0b4c --- /dev/null +++ b/metricbeat/docs/modules/system/uptime.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-system-uptime]] +include::../../../module/system/uptime/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/system/uptime/_meta/data.json[] +---- diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index fa77791b8290..880a6b196bfe 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -101,6 +101,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/system/process" _ "github.com/elastic/beats/metricbeat/module/system/process_summary" _ "github.com/elastic/beats/metricbeat/module/system/socket" + _ "github.com/elastic/beats/metricbeat/module/system/uptime" _ "github.com/elastic/beats/metricbeat/module/vsphere" _ "github.com/elastic/beats/metricbeat/module/vsphere/datastore" _ "github.com/elastic/beats/metricbeat/module/vsphere/host" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index ef8404cba83a..7c6433789a0b 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -40,6 +40,7 @@ metricbeat.modules: - network # Network IO - process # Per process metrics - process_summary # Process summary + - uptime # System Uptime #- core # Per CPU core usage #- diskio # Disk IO #- socket # Sockets and connection info (linux only) diff --git a/metricbeat/module/system/_meta/config.reference.yml b/metricbeat/module/system/_meta/config.reference.yml index 93928f7c1858..099b2b1f36b6 100644 --- a/metricbeat/module/system/_meta/config.reference.yml +++ b/metricbeat/module/system/_meta/config.reference.yml @@ -8,6 +8,7 @@ - network # Network IO - process # Per process metrics - process_summary # Process summary + - uptime # System Uptime #- core # Per CPU core usage #- diskio # Disk IO #- socket # Sockets and connection info (linux only) diff --git a/metricbeat/module/system/_meta/config.yml b/metricbeat/module/system/_meta/config.yml index ac2e8ce00d9f..2cb0093d08e2 100644 --- a/metricbeat/module/system/_meta/config.yml +++ b/metricbeat/module/system/_meta/config.yml @@ -23,3 +23,7 @@ processors: - drop_event.when.regexp: system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' + +- module: system + period: 15m + metricsets: [uptime] diff --git a/metricbeat/module/system/uptime/_meta/data.json b/metricbeat/module/system/uptime/_meta/data.json new file mode 100644 index 000000000000..519c26f3012e --- /dev/null +++ b/metricbeat/module/system/uptime/_meta/data.json @@ -0,0 +1,23 @@ +{ + "@timestamp": "2016-05-23T08:05:34.853Z", + "@metadata": { + "beat": "noindex", + "type": "doc" + }, + "system": { + "uptime": { + "duration": { + "ms": 695499821 + } + } + }, + "metricset": { + "module": "system", + "name": "uptime", + "rtt": 115 + }, + "beat": { + "name": "host.example.com", + "hostname": "host.example.com" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/uptime/_meta/docs.asciidoc b/metricbeat/module/system/uptime/_meta/docs.asciidoc new file mode 100644 index 000000000000..f738283cb5a0 --- /dev/null +++ b/metricbeat/module/system/uptime/_meta/docs.asciidoc @@ -0,0 +1,10 @@ +=== System Uptime Metricset + +The System `uptime` metricset provides the uptime of the host operating system. + +This metricset is available on: + +- Darwin +- Linux +- OpenBSD +- Windows diff --git a/metricbeat/module/system/uptime/_meta/fields.yml b/metricbeat/module/system/uptime/_meta/fields.yml new file mode 100644 index 000000000000..b641786a1dd0 --- /dev/null +++ b/metricbeat/module/system/uptime/_meta/fields.yml @@ -0,0 +1,11 @@ +- name: uptime + type: group + description: > + `uptime` contains the operating system uptime metric. + fields: + - name: duration.ms + type: long + format: duration + input_format: milliseconds + description: > + The OS uptime in milliseconds. diff --git a/metricbeat/module/system/uptime/doc.go b/metricbeat/module/system/uptime/doc.go new file mode 100644 index 000000000000..289e4a3f97b0 --- /dev/null +++ b/metricbeat/module/system/uptime/doc.go @@ -0,0 +1,2 @@ +// Package uptime reports the system's uptime. +package uptime diff --git a/metricbeat/module/system/uptime/metricset.go b/metricbeat/module/system/uptime/metricset.go new file mode 100644 index 000000000000..67133fb4643d --- /dev/null +++ b/metricbeat/module/system/uptime/metricset.go @@ -0,0 +1,42 @@ +// +build darwin linux openbsd windows + +package uptime + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" + sigar "github.com/elastic/gosigar" +) + +func init() { + if err := mb.Registry.AddMetricSet("system", "uptime", New, parse.EmptyHostParser); err != nil { + panic(err) + } +} + +// MetricSet for fetching an OS uptime metric. +type MetricSet struct { + mb.BaseMetricSet +} + +// New is a mb.MetricSetFactory that returns a new MetricSet. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + return &MetricSet{base}, nil +} + +// Fetch fetches the uptime metric from the OS. +func (m *MetricSet) Fetch() (common.MapStr, error) { + var uptime sigar.Uptime + if err := uptime.Get(); err != nil { + return nil, errors.Wrap(err, "failed to get uptime") + } + + return common.MapStr{ + "duration": common.MapStr{ + "ms": int64(uptime.Length * 1000), + }, + }, nil +} diff --git a/metricbeat/module/system/uptime/metricset_test.go b/metricbeat/module/system/uptime/metricset_test.go new file mode 100644 index 000000000000..524bf871c588 --- /dev/null +++ b/metricbeat/module/system/uptime/metricset_test.go @@ -0,0 +1,28 @@ +// +build darwin linux openbsd windows + +package uptime + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + f := mbtest.NewEventFetcher(t, getConfig()) + + uptime, err := f.Fetch() + if err != nil { + t.Fatal(err) + } + + event := mbtest.CreateFullEvent(f, uptime) + mbtest.WriteEventToDataJSON(t, event) +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "system", + "metricsets": []string{"uptime"}, + } +} diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml index ac2e8ce00d9f..2cb0093d08e2 100644 --- a/metricbeat/modules.d/system.yml +++ b/metricbeat/modules.d/system.yml @@ -23,3 +23,7 @@ processors: - drop_event.when.regexp: system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' + +- module: system + period: 15m + metricsets: [uptime] From 03a9e494738a2efef0de4a96e1474eecc9689fda Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 14 Aug 2017 09:16:35 -0400 Subject: [PATCH 2/3] Use block style list instead of flow style --- metricbeat/docs/modules/system.asciidoc | 3 ++- metricbeat/module/system/_meta/config.yml | 3 ++- metricbeat/modules.d/system.yml | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/metricbeat/docs/modules/system.asciidoc b/metricbeat/docs/modules/system.asciidoc index fc8bb49658fa..1fb694e6c6c7 100644 --- a/metricbeat/docs/modules/system.asciidoc +++ b/metricbeat/docs/modules/system.asciidoc @@ -112,7 +112,8 @@ metricbeat.modules: - module: system period: 15m - metricsets: [uptime] + metricsets: + - uptime ---- [float] diff --git a/metricbeat/module/system/_meta/config.yml b/metricbeat/module/system/_meta/config.yml index 2cb0093d08e2..409ec1b636c9 100644 --- a/metricbeat/module/system/_meta/config.yml +++ b/metricbeat/module/system/_meta/config.yml @@ -26,4 +26,5 @@ - module: system period: 15m - metricsets: [uptime] + metricsets: + - uptime diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml index 2cb0093d08e2..409ec1b636c9 100644 --- a/metricbeat/modules.d/system.yml +++ b/metricbeat/modules.d/system.yml @@ -26,4 +26,5 @@ - module: system period: 15m - metricsets: [uptime] + metricsets: + - uptime From 65314d52bfa1ee9cb7577cf5579f6892ab40faf9 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 14 Aug 2017 09:17:33 -0400 Subject: [PATCH 3/3] Remove newline from CHANGELOG --- CHANGELOG.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index bd39b4d656bd..ae2fa3e5a5fd 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -82,7 +82,6 @@ https://github.com/elastic/beats/compare/v6.0.0-beta1...master[Check the HEAD di - Add graphite protocol metricbeat module. {pull}4734[4734] - Add http server metricset to support push metrics via http. {pull}4770[4770] - Make config object public for graphite and http server {pull}4820[4820] - - Add system uptime metricset. {issue}[4848[4848] *Packetbeat*