From 8758921f2cd086f6cfd4fbae744f48d81d550d25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Fri, 28 Feb 2020 12:34:57 +0100 Subject: [PATCH 01/32] Allow to override already defined metricsets This is useful when we want to have different behaviors between flavours --- metricbeat/mb/registry.go | 30 +++++++++++++++++++++++------- metricbeat/mb/registry_test.go | 8 ++++++++ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/metricbeat/mb/registry.go b/metricbeat/mb/registry.go index 780213d2392a..62ee4f6c1d00 100644 --- a/metricbeat/mb/registry.go +++ b/metricbeat/mb/registry.go @@ -68,6 +68,7 @@ type MetricSetRegistration struct { IsDefault bool HostParser HostParser Namespace string + Replace bool } // MetricSetOption sets an option for a MetricSetFactory that is being @@ -99,6 +100,15 @@ func WithNamespace(namespace string) MetricSetOption { } } +// MustReplace specifies that the MetricSetFactory must be replacing an existing +// metricset with the same name. An error will happen if there is no metricset +// defined with the same params. +func MustReplace() MetricSetOption { + return func(r *MetricSetRegistration) { + r.Replace = true + } +} + // Register contains the factory functions for creating new Modules and new // MetricSets. Registers are thread safe for concurrent usage. type Register struct { @@ -201,22 +211,28 @@ func (r *Register) addMetricSet(module, name string, factory MetricSetFactory, o module = strings.ToLower(module) name = strings.ToLower(name) + // Set the options. + msInfo := MetricSetRegistration{Name: name, Factory: factory} + for _, opt := range options { + opt(&msInfo) + } + if metricsets, ok := r.metricSets[module]; !ok { + if msInfo.Replace { + return fmt.Errorf("metricset '%s/%s' should be replacing an existing metricset, none found", module, name) + } + r.metricSets[module] = map[string]MetricSetRegistration{} } else if _, exists := metricsets[name]; exists { - return fmt.Errorf("metricset '%s/%s' is already registered", module, name) + if !msInfo.Replace { + return fmt.Errorf("metricset '%s/%s' is already registered", module, name) + } } if factory == nil { return fmt.Errorf("metricset '%s/%s' cannot be registered with a nil factory", module, name) } - // Set the options. - msInfo := MetricSetRegistration{Name: name, Factory: factory} - for _, opt := range options { - opt(&msInfo) - } - r.metricSets[module][name] = msInfo r.log.Infof("MetricSet registered: %s/%s", module, name) return nil diff --git a/metricbeat/mb/registry_test.go b/metricbeat/mb/registry_test.go index f4ba90f4dd40..98ed4ce86058 100644 --- a/metricbeat/mb/registry_test.go +++ b/metricbeat/mb/registry_test.go @@ -192,6 +192,14 @@ func TestDefaultMetricSet(t *testing.T) { assert.Contains(t, names, metricSetName) } +func TestMustReplaceMetricSet(t *testing.T) { + registry := NewRegister() + err := registry.addMetricSet(moduleName, metricSetName, fakeMetricSetFactory, MustReplace()) + if assert.Error(t, err) { + assert.Equal(t, "metricset 'mymodule/mymetricset' should be replacing an existing metricset, none found", err.Error()) + } +} + func TestMetricSetQuery(t *testing.T) { registry := NewRegister() err := registry.AddMetricSet(moduleName, metricSetName, fakeMetricSetFactory) From 0de2208850c05fab4d683e9576fb908b829f1f36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 10 Mar 2020 00:23:25 +0100 Subject: [PATCH 02/32] Refactor Prometheus module to accommodate newer uses --- .../module/prometheus/collector/collector.go | 55 +++++++++----- .../prometheus/collector/collector_test.go | 38 +++++----- .../module/prometheus/collector/data.go | 76 ++++++++++++------- .../module/openmetrics/collector/collector.go | 2 +- 4 files changed, 103 insertions(+), 68 deletions(-) diff --git a/metricbeat/module/prometheus/collector/collector.go b/metricbeat/module/prometheus/collector/collector.go index fa78f64294d6..34e545f35b8b 100644 --- a/metricbeat/module/prometheus/collector/collector.go +++ b/metricbeat/module/prometheus/collector/collector.go @@ -35,7 +35,8 @@ const ( ) var ( - hostParser = parse.URLHostParserBuilder{ + // HostParser parses a Prometheus endpoint URL + HostParser = parse.URLHostParserBuilder{ DefaultScheme: defaultScheme, DefaultPath: defaultPath, PathConfigKey: "metrics_path", @@ -43,12 +44,19 @@ var ( ) func init() { - mb.Registry.MustAddMetricSet("prometheus", "collector", MetricSetBuilder("prometheus"), - mb.WithHostParser(hostParser), + mb.Registry.MustAddMetricSet("prometheus", "collector", + MetricSetBuilder("prometheus", DefaultPromEventsGeneratorFactory), + mb.WithHostParser(HostParser), mb.DefaultMetricSet(), ) } +// PromEventsGenerator converts a Prometheus metric family into a PromEvent list +type PromEventsGenerator func(mf *dto.MetricFamily) []PromEvent + +// PromEventsGeneratorFactory creates a PromEventsGenerator when instanciating a metricset +type PromEventsGeneratorFactory func(ms mb.BaseMetricSet) (PromEventsGenerator, error) + // MetricSet for fetching prometheus data type MetricSet struct { mb.BaseMetricSet @@ -56,10 +64,12 @@ type MetricSet struct { includeMetrics []*regexp.Regexp excludeMetrics []*regexp.Regexp namespace string + genPromEvents PromEventsGenerator } -// MetricSetBuilder returns a builder function for a new Prometheus metricset using the given namespace -func MetricSetBuilder(namespace string) func(base mb.BaseMetricSet) (mb.MetricSet, error) { +// MetricSetBuilder returns a builder function for a new Prometheus metricset using +// the given namespace and event generator +func MetricSetBuilder(namespace string, genFactory PromEventsGeneratorFactory) func(base mb.BaseMetricSet) (mb.MetricSet, error) { return func(base mb.BaseMetricSet) (mb.MetricSet, error) { config := defaultConfig if err := base.Module().UnpackConfig(&config); err != nil { @@ -70,10 +80,16 @@ func MetricSetBuilder(namespace string) func(base mb.BaseMetricSet) (mb.MetricSe return nil, err } + genPromEvents, err := genFactory(base) + if err != nil { + return nil, err + } + ms := &MetricSet{ BaseMetricSet: base, prometheus: prometheus, namespace: namespace, + genPromEvents: genPromEvents, } ms.excludeMetrics, err = compilePatternList(config.MetricsFilters.ExcludeMetrics) if err != nil { @@ -106,38 +122,35 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { if m.skipFamily(family) { continue } - promEvents := getPromEventsFromMetricFamily(family) + promEvents := m.genPromEvents(family) for _, promEvent := range promEvents { labelsHash := promEvent.LabelsHash() if _, ok := eventList[labelsHash]; !ok { - eventList[labelsHash] = common.MapStr{ - "metrics": common.MapStr{}, - } + eventList[labelsHash] = common.MapStr{} // Add default instance label if not already there - if exists, _ := promEvent.labels.HasKey("instance"); !exists { - promEvent.labels.Put("instance", m.Host()) + if exists, _ := promEvent.Labels.HasKey("instance"); !exists { + promEvent.Labels.Put("instance", m.Host()) } // Add default job label if not already there - if exists, _ := promEvent.labels.HasKey("job"); !exists { - promEvent.labels.Put("job", m.Module().Name()) + if exists, _ := promEvent.Labels.HasKey("job"); !exists { + promEvent.Labels.Put("job", m.Module().Name()) } // Add labels - if len(promEvent.labels) > 0 { - eventList[labelsHash]["labels"] = promEvent.labels + if len(promEvent.Labels) > 0 { + eventList[labelsHash]["labels"] = promEvent.Labels } } - // Not checking anything here because we create these maps some lines before - metrics := eventList[labelsHash]["metrics"].(common.MapStr) - metrics.Update(promEvent.data) + // Accumulate metrics in the event + eventList[labelsHash].Update(promEvent.Data) } } m.addUpEvent(eventList, 1) - // Converts hash list to slice + // Report events for _, e := range eventList { isOpen := reporter.Event(mb.Event{ RootFields: common.MapStr{m.namespace: e}, @@ -156,7 +169,7 @@ func (m *MetricSet) addUpEvent(eventList map[string]common.MapStr, up int) { return } upPromEvent := PromEvent{ - labels: common.MapStr{ + Labels: common.MapStr{ "instance": m.Host(), "job": "prometheus", }, @@ -165,7 +178,7 @@ func (m *MetricSet) addUpEvent(eventList map[string]common.MapStr, up int) { "metrics": common.MapStr{ "up": up, }, - "labels": upPromEvent.labels, + "labels": upPromEvent.Labels, } } diff --git a/metricbeat/module/prometheus/collector/collector_test.go b/metricbeat/module/prometheus/collector/collector_test.go index 88a361c623e9..105361ada161 100644 --- a/metricbeat/module/prometheus/collector/collector_test.go +++ b/metricbeat/module/prometheus/collector/collector_test.go @@ -63,10 +63,10 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { }, Event: []PromEvent{ { - data: common.MapStr{ + Data: common.MapStr{ "http_request_duration_microseconds": float64(10), }, - labels: labels, + Labels: labels, }, }, }, @@ -85,10 +85,10 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { }, Event: []PromEvent{ { - data: common.MapStr{ + Data: common.MapStr{ "http_request_duration_microseconds": float64(10), }, - labels: common.MapStr{}, + Labels: common.MapStr{}, }, }, }, @@ -114,17 +114,17 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { }, Event: []PromEvent{ { - data: common.MapStr{ + Data: common.MapStr{ "http_request_duration_microseconds_count": uint64(10), "http_request_duration_microseconds_sum": float64(10), }, - labels: common.MapStr{}, + Labels: common.MapStr{}, }, { - data: common.MapStr{ + Data: common.MapStr{ "http_request_duration_microseconds": float64(10), }, - labels: common.MapStr{ + Labels: common.MapStr{ "quantile": "0.99", }, }, @@ -152,17 +152,17 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { }, Event: []PromEvent{ { - data: common.MapStr{ + Data: common.MapStr{ "http_request_duration_microseconds_count": uint64(10), "http_request_duration_microseconds_sum": float64(10), }, - labels: common.MapStr{}, + Labels: common.MapStr{}, }, { - data: common.MapStr{ + Data: common.MapStr{ "http_request_duration_microseconds_bucket": uint64(10), }, - labels: common.MapStr{"le": "0.99"}, + Labels: common.MapStr{"le": "0.99"}, }, }, }, @@ -187,17 +187,17 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { }, Event: []PromEvent{ { - data: common.MapStr{ + Data: common.MapStr{ "http_request_duration_microseconds": float64(10), }, - labels: labels, + Labels: labels, }, }, }, } for _, test := range tests { - event := getPromEventsFromMetricFamily(test.Family) + event := DefaultPromEventsGenerator(test.Family) assert.Equal(t, test.Event, event) } } @@ -320,7 +320,7 @@ func TestSkipMetricFamily(t *testing.T) { metricsToKeep := 0 for _, testFamily := range testFamilies { if !ms.skipFamily(testFamily) { - metricsToKeep += 1 + metricsToKeep++ } } assert.Equal(t, metricsToKeep, len(testFamilies)) @@ -331,7 +331,7 @@ func TestSkipMetricFamily(t *testing.T) { metricsToKeep = 0 for _, testFamily := range testFamilies { if !ms.skipFamily(testFamily) { - metricsToKeep += 1 + metricsToKeep++ } } assert.Equal(t, metricsToKeep, 2) @@ -342,7 +342,7 @@ func TestSkipMetricFamily(t *testing.T) { metricsToKeep = 0 for _, testFamily := range testFamilies { if !ms.skipFamily(testFamily) { - metricsToKeep += 1 + metricsToKeep++ } } assert.Equal(t, len(testFamilies)-2, metricsToKeep) @@ -353,7 +353,7 @@ func TestSkipMetricFamily(t *testing.T) { metricsToKeep = 0 for _, testFamily := range testFamilies { if !ms.skipFamily(testFamily) { - metricsToKeep += 1 + metricsToKeep++ } } assert.Equal(t, 1, metricsToKeep) diff --git a/metricbeat/module/prometheus/collector/data.go b/metricbeat/module/prometheus/collector/data.go index 1651bf3752e9..0cdb74e1bc75 100644 --- a/metricbeat/module/prometheus/collector/data.go +++ b/metricbeat/module/prometheus/collector/data.go @@ -23,22 +23,30 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/helper/prometheus" + "github.com/elastic/beats/v7/metricbeat/mb" dto "github.com/prometheus/client_model/go" ) // PromEvent stores a set of one or more metrics with the same labels type PromEvent struct { - data common.MapStr - labels common.MapStr + Data common.MapStr + Labels common.MapStr } // LabelsHash returns a repeatable string that is unique for the set of labels in this event func (p *PromEvent) LabelsHash() string { - return prometheus.LabelHash(p.labels) + return prometheus.LabelHash(p.Labels) } -func getPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { +// DefaultPromEventsGeneratorFactory returns the default prometheus events generator +func DefaultPromEventsGeneratorFactory(ms mb.BaseMetricSet) (PromEventsGenerator, error) { + return DefaultPromEventsGenerator, nil +} + +// DefaultPromEventsGenerator stores all Prometheus metrics using +// only double field type in Elasticsearch. +func DefaultPromEventsGenerator(mf *dto.MetricFamily) []PromEvent { var events []PromEvent name := *mf.Name @@ -58,10 +66,12 @@ func getPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { if counter != nil { if !math.IsNaN(counter.GetValue()) && !math.IsInf(counter.GetValue(), 0) { events = append(events, PromEvent{ - data: common.MapStr{ - name: counter.GetValue(), + Data: common.MapStr{ + "metrics": common.MapStr{ + name: counter.GetValue(), + }, }, - labels: labels, + Labels: labels, }) } } @@ -70,10 +80,12 @@ func getPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { if gauge != nil { if !math.IsNaN(gauge.GetValue()) && !math.IsInf(gauge.GetValue(), 0) { events = append(events, PromEvent{ - data: common.MapStr{ - name: gauge.GetValue(), + Data: common.MapStr{ + "metrics": common.MapStr{ + name: gauge.GetValue(), + }, }, - labels: labels, + Labels: labels, }) } } @@ -82,11 +94,13 @@ func getPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { if summary != nil { if !math.IsNaN(summary.GetSampleSum()) && !math.IsInf(summary.GetSampleSum(), 0) { events = append(events, PromEvent{ - data: common.MapStr{ - name + "_sum": summary.GetSampleSum(), - name + "_count": summary.GetSampleCount(), + Data: common.MapStr{ + "metrics": common.MapStr{ + name + "_sum": summary.GetSampleSum(), + name + "_count": summary.GetSampleCount(), + }, }, - labels: labels, + Labels: labels, }) } @@ -98,10 +112,12 @@ func getPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { quantileLabels := labels.Clone() quantileLabels["quantile"] = strconv.FormatFloat(quantile.GetQuantile(), 'f', -1, 64) events = append(events, PromEvent{ - data: common.MapStr{ - name: quantile.GetValue(), + Data: common.MapStr{ + "metrics": common.MapStr{ + name: quantile.GetValue(), + }, }, - labels: quantileLabels, + Labels: quantileLabels, }) } } @@ -110,11 +126,13 @@ func getPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { if histogram != nil { if !math.IsNaN(histogram.GetSampleSum()) && !math.IsInf(histogram.GetSampleSum(), 0) { events = append(events, PromEvent{ - data: common.MapStr{ - name + "_sum": histogram.GetSampleSum(), - name + "_count": histogram.GetSampleCount(), + Data: common.MapStr{ + "metrics": common.MapStr{ + name + "_sum": histogram.GetSampleSum(), + name + "_count": histogram.GetSampleCount(), + }, }, - labels: labels, + Labels: labels, }) } @@ -127,10 +145,12 @@ func getPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { bucketLabels["le"] = strconv.FormatFloat(bucket.GetUpperBound(), 'f', -1, 64) events = append(events, PromEvent{ - data: common.MapStr{ - name + "_bucket": bucket.GetCumulativeCount(), + Data: common.MapStr{ + "metrics": common.MapStr{ + name + "_bucket": bucket.GetCumulativeCount(), + }, }, - labels: bucketLabels, + Labels: bucketLabels, }) } } @@ -139,10 +159,12 @@ func getPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { if untyped != nil { if !math.IsNaN(untyped.GetValue()) && !math.IsInf(untyped.GetValue(), 0) { events = append(events, PromEvent{ - data: common.MapStr{ - name: untyped.GetValue(), + Data: common.MapStr{ + "metrics": common.MapStr{ + name: untyped.GetValue(), + }, }, - labels: labels, + Labels: labels, }) } } diff --git a/x-pack/metricbeat/module/openmetrics/collector/collector.go b/x-pack/metricbeat/module/openmetrics/collector/collector.go index f0f7c960f654..ddc3fc70a3e4 100644 --- a/x-pack/metricbeat/module/openmetrics/collector/collector.go +++ b/x-pack/metricbeat/module/openmetrics/collector/collector.go @@ -24,6 +24,6 @@ var ( func init() { mb.Registry.MustAddMetricSet("openmetrics", "collector", - collector.MetricSetBuilder("openmetrics"), + collector.MetricSetBuilder("openmetrics", collector.DefaultPromEventsGenerator), mb.WithHostParser(hostParser)) } From fc726028c5272d5b7c0d58f7fd73380aee4541e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 3 Mar 2020 13:53:34 +0100 Subject: [PATCH 03/32] Add new Prometheus skeleton for basic types usage --- x-pack/metricbeat/include/list.go | 2 + .../module/openmetrics/collector/collector.go | 2 +- .../prometheus/collector/_meta/data.json | 24 +++ .../prometheus/collector/_meta/docs.asciidoc | 77 +++++++++ .../prometheus/collector/_meta/fields.yml | 1 + .../module/prometheus/collector/collector.go | 19 +++ .../module/prometheus/collector/config.go | 20 +++ .../module/prometheus/collector/data.go | 147 ++++++++++++++++++ x-pack/metricbeat/module/prometheus/doc.go | 6 + x-pack/metricbeat/module/prometheus/fields.go | 23 +++ 10 files changed, 320 insertions(+), 1 deletion(-) create mode 100644 x-pack/metricbeat/module/prometheus/collector/_meta/data.json create mode 100644 x-pack/metricbeat/module/prometheus/collector/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/prometheus/collector/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/prometheus/collector/collector.go create mode 100644 x-pack/metricbeat/module/prometheus/collector/config.go create mode 100644 x-pack/metricbeat/module/prometheus/collector/data.go create mode 100644 x-pack/metricbeat/module/prometheus/doc.go create mode 100644 x-pack/metricbeat/module/prometheus/fields.go diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 04f5458895e0..a46368da4082 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -49,6 +49,8 @@ import ( _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/oracle" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/oracle/performance" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/oracle/tablespace" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus/collector" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/redisenterprise" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/sql" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/sql/query" diff --git a/x-pack/metricbeat/module/openmetrics/collector/collector.go b/x-pack/metricbeat/module/openmetrics/collector/collector.go index ddc3fc70a3e4..362260ded074 100644 --- a/x-pack/metricbeat/module/openmetrics/collector/collector.go +++ b/x-pack/metricbeat/module/openmetrics/collector/collector.go @@ -24,6 +24,6 @@ var ( func init() { mb.Registry.MustAddMetricSet("openmetrics", "collector", - collector.MetricSetBuilder("openmetrics", collector.DefaultPromEventsGenerator), + collector.MetricSetBuilder("openmetrics", collector.DefaultPromEventsGeneratorFactory), mb.WithHostParser(hostParser)) } diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/data.json b/x-pack/metricbeat/module/prometheus/collector/_meta/data.json new file mode 100644 index 000000000000..a46b63c74feb --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/data.json @@ -0,0 +1,24 @@ +{ + "@timestamp": "2019-03-01T08:05:34.853Z", + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "job": "prometheus" + }, + "metrics": { + "up": 1 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/docs.asciidoc b/x-pack/metricbeat/module/prometheus/collector/_meta/docs.asciidoc new file mode 100644 index 000000000000..82d58cecab28 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/docs.asciidoc @@ -0,0 +1,77 @@ +The Prometheus `collector` metricset scrapes data from https://prometheus.io/docs/instrumenting/exporters/[prometheus exporters]. + + +[float] +=== Scraping from a Prometheus exporter + +To scrape metrics from a Prometheus exporter, configure the `hosts` field to it. The path +to retrieve the metrics from (`/metrics` by default) can be configured with `metrics_path`. + +[source,yaml] +------------------------------------------------------------------------------------- +- module: prometheus + period: 10s + hosts: ["node:9100"] + metrics_path: /metrics + + #username: "user" + #password: "secret" + + # This can be used for service account based authorization: + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt +------------------------------------------------------------------------------------- + + +[float] +=== Scraping all metrics from a Prometheus server + +This module can scrape all metrics stored in a Prometheus server, by using the +https://prometheus.io/docs/prometheus/latest/federation/[federation API]. By pointing this +config to the Prometheus server: + +[source,yaml] +------------------------------------------------------------------------------------- +metricbeat.modules: +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + metrics_path: '/federate' + query: + 'match[]': '{__name__!=""}' +------------------------------------------------------------------------------------- + +[float] +=== Filtering metrics + +In order to filter out/in metrics one can make use of `metrics_filters.include` `metrics_filters.exclude` settings: + +[source,yaml] +------------------------------------------------------------------------------------- +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + metrics_path: /metrics + metrics_filters: + include: ["node_filesystem_*"] + exclude: ["node_filesystem_device_*"] +------------------------------------------------------------------------------------- + +The configuration above will include only metrics that match `node_filesystem_*` pattern and do not match `node_filesystem_device_*`. + + +To keep only specific metrics, anchor the start and the end of the regexp of each metric: + +- the caret ^ matches the beginning of a text or line, +- the dollar sign $ matches the end of a text. + +[source,yaml] +------------------------------------------------------------------------------------- +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + metrics_path: /metrics + metrics_filters: + include: ["^node_network_net_dev_group$", "^node_network_up$"] +------------------------------------------------------------------------------------- diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/fields.yml b/x-pack/metricbeat/module/prometheus/collector/_meta/fields.yml new file mode 100644 index 000000000000..a927f3fc9f84 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/fields.yml @@ -0,0 +1 @@ +- release: ga diff --git a/x-pack/metricbeat/module/prometheus/collector/collector.go b/x-pack/metricbeat/module/prometheus/collector/collector.go new file mode 100644 index 000000000000..ecae5802a0f5 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/collector.go @@ -0,0 +1,19 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package collector + +import ( + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/prometheus/collector" +) + +func init() { + mb.Registry.MustAddMetricSet("prometheus", "collector", + collector.MetricSetBuilder("prometheus", promEventsGeneratorFactory), + mb.WithHostParser(collector.HostParser), + mb.DefaultMetricSet(), + mb.MustReplace(), + ) +} diff --git a/x-pack/metricbeat/module/prometheus/collector/config.go b/x-pack/metricbeat/module/prometheus/collector/config.go new file mode 100644 index 000000000000..8813d5e3e59c --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/config.go @@ -0,0 +1,20 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package collector + +import "errors" + +type config struct { + UseTypes bool `config:"use_types"` + RateCounters bool `config:"rate_counters"` +} + +func (c *config) Validate() error { + if c.RateCounters && !c.UseTypes { + return errors.New("'rate_counters' can only be enabled when `use_types` is also enabled") + } + + return nil +} diff --git a/x-pack/metricbeat/module/prometheus/collector/data.go b/x-pack/metricbeat/module/prometheus/collector/data.go new file mode 100644 index 000000000000..06b4661b46c6 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/data.go @@ -0,0 +1,147 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package collector + +import ( + "math" + "strconv" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/prometheus/collector" + + dto "github.com/prometheus/client_model/go" +) + +func promEventsGeneratorFactory(base mb.BaseMetricSet) (collector.PromEventsGenerator, error) { + config := config{} + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + if config.UseTypes { + if config.RateCounters { + + } + + return promEventsGenerator, nil + } + + return collector.DefaultPromEventsGenerator, nil +} + +// promEventsGenerator stores all Prometheus metrics using +// only double field type in Elasticsearch. +func promEventsGenerator(mf *dto.MetricFamily) []collector.PromEvent { + var events []collector.PromEvent + + name := *mf.Name + metrics := mf.Metric + for _, metric := range metrics { + labels := common.MapStr{} + + if len(metric.Label) != 0 { + for _, label := range metric.Label { + if label.GetName() != "" && label.GetValue() != "" { + labels[label.GetName()] = label.GetValue() + } + } + } + + counter := metric.GetCounter() + if counter != nil { + if !math.IsNaN(counter.GetValue()) && !math.IsInf(counter.GetValue(), 0) { + events = append(events, collector.PromEvent{ + Data: common.MapStr{ + name + ".counter": counter.GetValue(), + }, + Labels: labels, + }) + } + } + + gauge := metric.GetGauge() + if gauge != nil { + if !math.IsNaN(gauge.GetValue()) && !math.IsInf(gauge.GetValue(), 0) { + events = append(events, collector.PromEvent{ + Data: common.MapStr{ + name: gauge.GetValue(), + }, + Labels: labels, + }) + } + } + + summary := metric.GetSummary() + if summary != nil { + if !math.IsNaN(summary.GetSampleSum()) && !math.IsInf(summary.GetSampleSum(), 0) { + events = append(events, collector.PromEvent{ + Data: common.MapStr{ + name + "_sum": summary.GetSampleSum(), + name + "_count": summary.GetSampleCount(), + }, + Labels: labels, + }) + } + + for _, quantile := range summary.GetQuantile() { + if math.IsNaN(quantile.GetValue()) || math.IsInf(quantile.GetValue(), 0) { + continue + } + + quantileLabels := labels.Clone() + quantileLabels["quantile"] = strconv.FormatFloat(quantile.GetQuantile(), 'f', -1, 64) + events = append(events, collector.PromEvent{ + Data: common.MapStr{ + name: quantile.GetValue(), + }, + Labels: quantileLabels, + }) + } + } + + histogram := metric.GetHistogram() + if histogram != nil { + if !math.IsNaN(histogram.GetSampleSum()) && !math.IsInf(histogram.GetSampleSum(), 0) { + events = append(events, collector.PromEvent{ + Data: common.MapStr{ + name + "_sum": histogram.GetSampleSum(), + name + "_count": histogram.GetSampleCount(), + }, + Labels: labels, + }) + } + + for _, bucket := range histogram.GetBucket() { + if bucket.GetCumulativeCount() == uint64(math.NaN()) || bucket.GetCumulativeCount() == uint64(math.Inf(0)) { + continue + } + + bucketLabels := labels.Clone() + bucketLabels["le"] = strconv.FormatFloat(bucket.GetUpperBound(), 'f', -1, 64) + + events = append(events, collector.PromEvent{ + Data: common.MapStr{ + name + "_bucket": bucket.GetCumulativeCount(), + }, + Labels: bucketLabels, + }) + } + } + + untyped := metric.GetUntyped() + if untyped != nil { + if !math.IsNaN(untyped.GetValue()) && !math.IsInf(untyped.GetValue(), 0) { + events = append(events, collector.PromEvent{ + Data: common.MapStr{ + name: untyped.GetValue(), + }, + Labels: labels, + }) + } + } + } + return events +} diff --git a/x-pack/metricbeat/module/prometheus/doc.go b/x-pack/metricbeat/module/prometheus/doc.go new file mode 100644 index 000000000000..d2a78cdb9b56 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/doc.go @@ -0,0 +1,6 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Package prometheus is a Metricbeat module that contains MetricSets. +package prometheus diff --git a/x-pack/metricbeat/module/prometheus/fields.go b/x-pack/metricbeat/module/prometheus/fields.go new file mode 100644 index 000000000000..1cf2a5b6b39e --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package prometheus + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "prometheus", asset.ModuleFieldsPri, AssetPrometheus); err != nil { + panic(err) + } +} + +// AssetPrometheus returns asset data. +// This is the base64 encoded gzipped contents of module/prometheus. +func AssetPrometheus() string { + return "eJxSgAJdhaLUnNTE4lQrhfRELkAAAAD//zROBVs=" +} From e9b2782a518a0c248f7096ebde73fa3d736b239e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Mon, 16 Mar 2020 12:50:34 +0100 Subject: [PATCH 04/32] Add typed schema to Prometheus module This should make room for using different Elasticsearch types depending on the Prometheus metric --- x-pack/metricbeat/metricbeat.reference.yml | 6 ++ .../module/prometheus/_meta/config.yml | 5 ++ .../module/prometheus/_meta/fields.yml | 31 ++++++++ .../module/prometheus/collector/data.go | 71 +++++++++++-------- x-pack/metricbeat/module/prometheus/fields.go | 2 +- x-pack/metricbeat/modules.d/prometheus.yml | 8 +++ 6 files changed, 94 insertions(+), 29 deletions(-) create mode 100644 x-pack/metricbeat/module/prometheus/_meta/config.yml create mode 100644 x-pack/metricbeat/module/prometheus/_meta/fields.yml create mode 100644 x-pack/metricbeat/modules.d/prometheus.yml diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 00b4daa56136..db8c09345ae3 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -986,6 +986,12 @@ metricbeat.modules: #password: pass #------------------------------ Prometheus Module ------------------------------ +- module: prometheus + metricsets: ['collector'] + period: 10s + hosts: ['localhost:9090'] + # TODO +#------------------------------ Prometheus Module ------------------------------ - module: prometheus period: 10s hosts: ["localhost:9090"] diff --git a/x-pack/metricbeat/module/prometheus/_meta/config.yml b/x-pack/metricbeat/module/prometheus/_meta/config.yml new file mode 100644 index 000000000000..5db1c5d5470f --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/_meta/config.yml @@ -0,0 +1,5 @@ +- module: prometheus + metricsets: ['collector'] + period: 10s + hosts: ['localhost:9090'] + # TODO \ No newline at end of file diff --git a/x-pack/metricbeat/module/prometheus/_meta/fields.yml b/x-pack/metricbeat/module/prometheus/_meta/fields.yml new file mode 100644 index 000000000000..b8fceb18c68e --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/_meta/fields.yml @@ -0,0 +1,31 @@ +- key: prometheus + title: "Prometheus" + description: > + Stats scraped from a Prometheus endpoint. + release: ga + settings: ["ssl", "http"] + fields: + - name: prometheus.*.value + type: object + object_type: double + object_type_mapping_type: "*" + description: > + Prometheus gauge metric + - name: prometheus.*.counter + type: object + object_type: double + object_type_mapping_type: "*" + description: > + Prometheus counter metric + - name: prometheus.*.rate + type: object + object_type: double + object_type_mapping_type: "*" + description: > + Prometheus rated counter metric + - name: prometheus.*.histogram + type: object + object_type: histogram + object_type_mapping_type: "*" + description: > + Prometheus histogram metric diff --git a/x-pack/metricbeat/module/prometheus/collector/data.go b/x-pack/metricbeat/module/prometheus/collector/data.go index 06b4661b46c6..b1ca5a398778 100644 --- a/x-pack/metricbeat/module/prometheus/collector/data.go +++ b/x-pack/metricbeat/module/prometheus/collector/data.go @@ -55,7 +55,9 @@ func promEventsGenerator(mf *dto.MetricFamily) []collector.PromEvent { if !math.IsNaN(counter.GetValue()) && !math.IsInf(counter.GetValue(), 0) { events = append(events, collector.PromEvent{ Data: common.MapStr{ - name + ".counter": counter.GetValue(), + name: common.MapStr{ + "counter": counter.GetValue(), + }, }, Labels: labels, }) @@ -67,7 +69,9 @@ func promEventsGenerator(mf *dto.MetricFamily) []collector.PromEvent { if !math.IsNaN(gauge.GetValue()) && !math.IsInf(gauge.GetValue(), 0) { events = append(events, collector.PromEvent{ Data: common.MapStr{ - name: gauge.GetValue(), + name: common.MapStr{ + "value": gauge.GetValue(), + }, }, Labels: labels, }) @@ -79,8 +83,12 @@ func promEventsGenerator(mf *dto.MetricFamily) []collector.PromEvent { if !math.IsNaN(summary.GetSampleSum()) && !math.IsInf(summary.GetSampleSum(), 0) { events = append(events, collector.PromEvent{ Data: common.MapStr{ - name + "_sum": summary.GetSampleSum(), - name + "_count": summary.GetSampleCount(), + name + "_sum": common.MapStr{ + "counter": summary.GetSampleSum(), + }, + name + "_count": common.MapStr{ + "counter": summary.GetSampleCount(), + }, }, Labels: labels, }) @@ -95,7 +103,9 @@ func promEventsGenerator(mf *dto.MetricFamily) []collector.PromEvent { quantileLabels["quantile"] = strconv.FormatFloat(quantile.GetQuantile(), 'f', -1, 64) events = append(events, collector.PromEvent{ Data: common.MapStr{ - name: quantile.GetValue(), + name: common.MapStr{ + "value": quantile.GetValue(), + }, }, Labels: quantileLabels, }) @@ -104,31 +114,34 @@ func promEventsGenerator(mf *dto.MetricFamily) []collector.PromEvent { histogram := metric.GetHistogram() if histogram != nil { - if !math.IsNaN(histogram.GetSampleSum()) && !math.IsInf(histogram.GetSampleSum(), 0) { - events = append(events, collector.PromEvent{ - Data: common.MapStr{ - name + "_sum": histogram.GetSampleSum(), - name + "_count": histogram.GetSampleCount(), - }, - Labels: labels, - }) - } - - for _, bucket := range histogram.GetBucket() { - if bucket.GetCumulativeCount() == uint64(math.NaN()) || bucket.GetCumulativeCount() == uint64(math.Inf(0)) { - continue + /* + TODO convert histogram to ES type + if !math.IsNaN(histogram.GetSampleSum()) && !math.IsInf(histogram.GetSampleSum(), 0) { + events = append(events, collector.PromEvent{ + Data: common.MapStr{ + name + "_sum.counter": histogram.GetSampleSum(), + name + "_count.counter": histogram.GetSampleCount(), + }, + Labels: labels, + }) } - bucketLabels := labels.Clone() - bucketLabels["le"] = strconv.FormatFloat(bucket.GetUpperBound(), 'f', -1, 64) + for _, bucket := range histogram.GetBucket() { + if bucket.GetCumulativeCount() == uint64(math.NaN()) || bucket.GetCumulativeCount() == uint64(math.Inf(0)) { + continue + } - events = append(events, collector.PromEvent{ - Data: common.MapStr{ - name + "_bucket": bucket.GetCumulativeCount(), - }, - Labels: bucketLabels, - }) - } + bucketLabels := labels.Clone() + bucketLabels["le"] = strconv.FormatFloat(bucket.GetUpperBound(), 'f', -1, 64) + + events = append(events, collector.PromEvent{ + Data: common.MapStr{ + name + "_bucket": bucket.GetCumulativeCount(), + }, + Labels: bucketLabels, + }) + } + */ } untyped := metric.GetUntyped() @@ -136,7 +149,9 @@ func promEventsGenerator(mf *dto.MetricFamily) []collector.PromEvent { if !math.IsNaN(untyped.GetValue()) && !math.IsInf(untyped.GetValue(), 0) { events = append(events, collector.PromEvent{ Data: common.MapStr{ - name: untyped.GetValue(), + name: common.MapStr{ + "value": untyped.GetValue(), + }, }, Labels: labels, }) diff --git a/x-pack/metricbeat/module/prometheus/fields.go b/x-pack/metricbeat/module/prometheus/fields.go index 1cf2a5b6b39e..74ab153081bf 100644 --- a/x-pack/metricbeat/module/prometheus/fields.go +++ b/x-pack/metricbeat/module/prometheus/fields.go @@ -19,5 +19,5 @@ func init() { // AssetPrometheus returns asset data. // This is the base64 encoded gzipped contents of module/prometheus. func AssetPrometheus() string { - return "eJxSgAJdhaLUnNTE4lQrhfRELkAAAAD//zROBVs=" + return "eJzE0ktqMzEMB/C9T/HHy/AlB/DiO0Ohy1KCMlY8bv3C0hRy+5LMkA590CwK8c6SbP1kvMUrnxxar5l15EkMoFETO9iHa9AawLMMPTaNtTj8NwDwqKQCGTo19jj2mkH4OAUuvtVYdGeAzolJ2CGQAYRVYwni8GRFkv0HO6o2+2yAY+TkxV06bFEo89q32+zeKE18SQN6auxQDy886BKaN/s54+t0SPw1s8/UWixhKbMbu9R8M+Z5raYKNAVGZu1x+Bk51Kko9/sxF8Cv0E56x8c8d/c3W8coWkOnfCP4c/3fmK+3rr2zef3L3wMAAP//PRYNew==" } diff --git a/x-pack/metricbeat/modules.d/prometheus.yml b/x-pack/metricbeat/modules.d/prometheus.yml new file mode 100644 index 000000000000..85b3c089e048 --- /dev/null +++ b/x-pack/metricbeat/modules.d/prometheus.yml @@ -0,0 +1,8 @@ +# Module: prometheus +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-prometheus.html + +- module: prometheus + metricsets: ['collector'] + period: 10s + hosts: ['localhost:9100'] + use_types: true \ No newline at end of file From 06bdb1d7772d9b66dc870de93a47ee54034ac0dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 17 Mar 2020 12:12:39 +0100 Subject: [PATCH 05/32] Convert Prometheus histograms to ES histograms --- .../module/prometheus/collector/collector.go | 1 + .../module/prometheus/_meta/config.yml | 8 +- .../module/prometheus/collector/counter.go | 92 +++++++++++++++++++ .../prometheus/collector/counter_test.go | 65 +++++++++++++ .../module/prometheus/collector/data.go | 47 ++++++---- .../module/prometheus/collector/histogram.go | 59 ++++++++++++ 6 files changed, 253 insertions(+), 19 deletions(-) create mode 100644 x-pack/metricbeat/module/prometheus/collector/counter.go create mode 100644 x-pack/metricbeat/module/prometheus/collector/counter_test.go create mode 100644 x-pack/metricbeat/module/prometheus/collector/histogram.go diff --git a/metricbeat/module/prometheus/collector/collector.go b/metricbeat/module/prometheus/collector/collector.go index 34e545f35b8b..95fe083977b1 100644 --- a/metricbeat/module/prometheus/collector/collector.go +++ b/metricbeat/module/prometheus/collector/collector.go @@ -164,6 +164,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { } func (m *MetricSet) addUpEvent(eventList map[string]common.MapStr, up int) { + // TODO add this as family so it follows common code paths metricName := "up" if m.skipFamilyName(metricName) { return diff --git a/x-pack/metricbeat/module/prometheus/_meta/config.yml b/x-pack/metricbeat/module/prometheus/_meta/config.yml index 5db1c5d5470f..d51a2550dca1 100644 --- a/x-pack/metricbeat/module/prometheus/_meta/config.yml +++ b/x-pack/metricbeat/module/prometheus/_meta/config.yml @@ -2,4 +2,10 @@ metricsets: ['collector'] period: 10s hosts: ['localhost:9090'] - # TODO \ No newline at end of file + + # Use Elasticsearch histogram type to store histograms (beta, default: false) + # This will change the default layout and put metric type in the field name + #use_types: true + + # Store counter rates instead of original cumulative counters (experimental, default: false) + #rate_counters: true \ No newline at end of file diff --git a/x-pack/metricbeat/module/prometheus/collector/counter.go b/x-pack/metricbeat/module/prometheus/collector/counter.go new file mode 100644 index 000000000000..b4798044de18 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/counter.go @@ -0,0 +1,92 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package collector + +import ( + "time" + + "github.com/elastic/beats/v7/libbeat/common" +) + +// CounterCache keeps a cache of the last value of all given counters +// and allows to calculate their rate since the last call. +// All methods are thread-unsafe and must not be called concurrently +type CounterCache interface { + // Start the cache cleanup worker. It mus be called once before start using + // the cache + Start() + + // Stop the cache cleanup worker. It mus be called when the cache is disposed + Stop() + + // RateUUint64 returns, for a given counter name, the difference between the given value + // and the value that was given in a previous call. It will return 0 on the first call + RateUint64(counterName string, value uint64) uint64 + + // RateFloat64 returns, for a given counter name, the difference between the given value + // and the value that was given in a previous call. It will return 0.0 on the first call + RateFloat64(counterName string, value float64) float64 +} + +type counterCache struct { + ints *common.Cache + floats *common.Cache + timeout time.Duration +} + +// NewCounterCache initializes and returns a CounterCache. The timeout parameter will be +// used to automatically expire counters that hasn't been updated in a whole timeout period +func NewCounterCache(timeout time.Duration) CounterCache { + return &counterCache{ + ints: common.NewCache(timeout, 0), + floats: common.NewCache(timeout, 0), + timeout: timeout, + } +} + +// RateUint64 returns, for a given counter name, the difference between the given value +// and the value that was given in a previous call. It will return 0 on the first call +func (c *counterCache) RateUint64(counterName string, value uint64) uint64 { + prev := c.ints.PutWithTimeout(counterName, value, c.timeout) + if prev != nil { + if prev.(uint64) > value { + // counter reset + return 0 + } + return value - prev.(uint64) + } + + // first put for this value, return rate of 0 + return 0 +} + +// RateFloat64 returns, for a given counter name, the difference between the given value +// and the value that was given in a previous call. It will return 0.0 on the first call +func (c *counterCache) RateFloat64(counterName string, value float64) float64 { + prev := c.floats.PutWithTimeout(counterName, value, c.timeout) + if prev != nil { + if prev.(float64) > value { + // counter reset + return 0 + } + return value - prev.(float64) + } + + // first put for this value, return rate of 0 + return 0 +} + +// Start the cache cleanup worker. It mus be called once before start using +// the cache +func (c *counterCache) Start() { + c.ints.StartJanitor(c.timeout) + c.floats.StartJanitor(c.timeout) +} + +// Stop the cache cleanup worker. It mus be called when the cache is disposed +func (c *counterCache) Stop() { + c.ints.StopJanitor() + c.floats.StopJanitor() +} diff --git a/x-pack/metricbeat/module/prometheus/collector/counter_test.go b/x-pack/metricbeat/module/prometheus/collector/counter_test.go new file mode 100644 index 000000000000..d3dabe80d853 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/counter_test.go @@ -0,0 +1,65 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package collector + +import ( + "testing" + "time" + + "github.com/elastic/beats/libbeat/common" +) + +func Test_CounterCache(t *testing.T) { + type fields struct { + ints *common.Cache + floats *common.Cache + timeout time.Duration + } + + tests := []struct { + name string + counterCache CounterCache + counterName string + valuesUint64 []uint64 + expectedUin64 []uint64 + valuesFloat64 []float64 + expectedFloat64 []float64 + }{ + { + name: "rates are calculated", + counterCache: NewCounterCache(1 * time.Second), + counterName: "test_counter", + valuesUint64: []uint64{10, 14, 17, 17, 28}, + expectedUin64: []uint64{0, 4, 3, 0, 11}, + valuesFloat64: []float64{1.0, 101.0, 102.0, 102.0, 1034.0}, + expectedFloat64: []float64{0.0, 100.0, 1.0, 0.0, 932.0}, + }, + { + name: "counter reset", + counterCache: NewCounterCache(1 * time.Second), + counterName: "test_counter", + valuesUint64: []uint64{10, 14, 17, 1, 3}, + expectedUin64: []uint64{0, 4, 3, 0, 2}, + valuesFloat64: []float64{1.0, 101.0, 2.0, 13.0}, + expectedFloat64: []float64{0.0, 100.0, 0.0, 11.0}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for i, val := range tt.valuesUint64 { + want := tt.expectedUin64[i] + if got := tt.counterCache.RateUint64(tt.counterName, val); got != want { + t.Errorf("counterCache.RateUint64() = %v, want %v", got, want) + } + } + for i, val := range tt.valuesFloat64 { + want := tt.expectedFloat64[i] + if got := tt.counterCache.RateFloat64(tt.counterName, val); got != want { + t.Errorf("counterCache.RateFloat64() = %v, want %v", got, want) + } + } + }) + } +} diff --git a/x-pack/metricbeat/module/prometheus/collector/data.go b/x-pack/metricbeat/module/prometheus/collector/data.go index b1ca5a398778..8e24b34a45d2 100644 --- a/x-pack/metricbeat/module/prometheus/collector/data.go +++ b/x-pack/metricbeat/module/prometheus/collector/data.go @@ -9,6 +9,7 @@ import ( "strconv" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/prometheus/collector" @@ -22,19 +23,37 @@ func promEventsGeneratorFactory(base mb.BaseMetricSet) (collector.PromEventsGene } if config.UseTypes { + cfgwarn.Beta("Prometheus 'use_types' settings is beta") + + // use a counter cache with a timeout of 5x the period, as a safe value + // to make sure that all counters are available between fetches + counters := NewCounterCache(base.Module().Config().Period * 5) + counters.Start() + if config.RateCounters { + cfgwarn.Beta("Prometheus 'rate_counters' settings is experimental") + } + g := typedGenerator{ + counterCache: counters, + rateCounters: config.RateCounters, } - return promEventsGenerator, nil + // TODO cache cleanup on module stop, probably make this an interface + return g.promEventsGenerator, nil } return collector.DefaultPromEventsGenerator, nil } +type typedGenerator struct { + counterCache CounterCache + rateCounters bool +} + // promEventsGenerator stores all Prometheus metrics using // only double field type in Elasticsearch. -func promEventsGenerator(mf *dto.MetricFamily) []collector.PromEvent { +func (g *typedGenerator) promEventsGenerator(mf *dto.MetricFamily) []collector.PromEvent { var events []collector.PromEvent name := *mf.Name @@ -114,6 +133,14 @@ func promEventsGenerator(mf *dto.MetricFamily) []collector.PromEvent { histogram := metric.GetHistogram() if histogram != nil { + events = append(events, collector.PromEvent{ + Data: common.MapStr{ + name: common.MapStr{ + "histogram": promHistogramToES(g.counterCache, name, labels, histogram), + }, + }, + Labels: labels, + }) /* TODO convert histogram to ES type if !math.IsNaN(histogram.GetSampleSum()) && !math.IsInf(histogram.GetSampleSum(), 0) { @@ -125,22 +152,6 @@ func promEventsGenerator(mf *dto.MetricFamily) []collector.PromEvent { Labels: labels, }) } - - for _, bucket := range histogram.GetBucket() { - if bucket.GetCumulativeCount() == uint64(math.NaN()) || bucket.GetCumulativeCount() == uint64(math.Inf(0)) { - continue - } - - bucketLabels := labels.Clone() - bucketLabels["le"] = strconv.FormatFloat(bucket.GetUpperBound(), 'f', -1, 64) - - events = append(events, collector.PromEvent{ - Data: common.MapStr{ - name + "_bucket": bucket.GetCumulativeCount(), - }, - Labels: bucketLabels, - }) - } */ } diff --git a/x-pack/metricbeat/module/prometheus/collector/histogram.go b/x-pack/metricbeat/module/prometheus/collector/histogram.go new file mode 100644 index 000000000000..ec51f946f89a --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/histogram.go @@ -0,0 +1,59 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package collector + +import ( + "fmt" + "math" + + "github.com/elastic/beats/v7/libbeat/common" + + dto "github.com/prometheus/client_model/go" +) + +// promHistogramToES takes a Prometheus histogram and converts it to +func promHistogramToES(cc CounterCache, name string, labels common.MapStr, histogram *dto.Histogram) common.MapStr { + var values []float64 + var counts []uint64 + + // sort buckets + buckets := histogram.GetBucket() + /* + sort.SliceStable(buckets, func(i, j int) bool { + return buckets[i].GetUpperBound() < buckets[j].GetUpperBound() + }) + */ + + // calculate centroids and rated counts + var lastUpper float64 + var sumCount uint64 + for _, bucket := range buckets { + // Ignore non-numbers + if bucket.GetCumulativeCount() == uint64(math.NaN()) || bucket.GetCumulativeCount() == uint64(math.Inf(0)) { + continue + } + + if bucket.GetUpperBound() == math.Inf(0) { + // Report +Inf bucket as a point + values = append(values, lastUpper) + } else { + // calculate bucket centroid + values = append(values, lastUpper+(bucket.GetUpperBound()-lastUpper)/2.0) + lastUpper = bucket.GetUpperBound() + } + + // take count for this period (rate) + deacumulate + count := cc.RateUint64(name+labels.String()+fmt.Sprintf("%f", bucket.GetUpperBound()), bucket.GetCumulativeCount()) - sumCount + counts = append(counts, count) + sumCount += count + } + + res := common.MapStr{ + "values": values, + "counts": counts, + } + + return res +} From 8820af6e1e574ef4f400c427c67f4c6b8846be91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Wed, 18 Mar 2020 11:35:05 +0100 Subject: [PATCH 06/32] Disable sorting, it's not needed --- .../module/prometheus/collector/histogram.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/x-pack/metricbeat/module/prometheus/collector/histogram.go b/x-pack/metricbeat/module/prometheus/collector/histogram.go index ec51f946f89a..1f4435c69290 100644 --- a/x-pack/metricbeat/module/prometheus/collector/histogram.go +++ b/x-pack/metricbeat/module/prometheus/collector/histogram.go @@ -18,18 +18,10 @@ func promHistogramToES(cc CounterCache, name string, labels common.MapStr, histo var values []float64 var counts []uint64 - // sort buckets - buckets := histogram.GetBucket() - /* - sort.SliceStable(buckets, func(i, j int) bool { - return buckets[i].GetUpperBound() < buckets[j].GetUpperBound() - }) - */ - // calculate centroids and rated counts var lastUpper float64 var sumCount uint64 - for _, bucket := range buckets { + for _, bucket := range histogram.GetBucket() { // Ignore non-numbers if bucket.GetCumulativeCount() == uint64(math.NaN()) || bucket.GetCumulativeCount() == uint64(math.Inf(0)) { continue From 3fbac0b9efe2aab1cda1bb33083e158a11a6000b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Wed, 18 Mar 2020 16:26:36 +0100 Subject: [PATCH 07/32] Handle counter cache start/stop --- .../module/prometheus/collector/collector.go | 29 +++++++++++++++---- .../prometheus/collector/collector_test.go | 3 +- .../module/prometheus/collector/data.go | 9 ++++-- .../module/prometheus/collector/data.go | 18 ++++++++---- .../module/prometheus/collector/histogram.go | 7 +++-- 5 files changed, 49 insertions(+), 17 deletions(-) diff --git a/metricbeat/module/prometheus/collector/collector.go b/metricbeat/module/prometheus/collector/collector.go index 95fe083977b1..e1a9ec2d23d2 100644 --- a/metricbeat/module/prometheus/collector/collector.go +++ b/metricbeat/module/prometheus/collector/collector.go @@ -19,6 +19,7 @@ package collector import ( "regexp" + "sync" "github.com/pkg/errors" dto "github.com/prometheus/client_model/go" @@ -52,7 +53,16 @@ func init() { } // PromEventsGenerator converts a Prometheus metric family into a PromEvent list -type PromEventsGenerator func(mf *dto.MetricFamily) []PromEvent +type PromEventsGenerator interface { + // Start must be called before using the generator + Start() + + // converts a Prometheus metric family into a list of PromEvents + GeneratePromEvents(mf *dto.MetricFamily) []PromEvent + + // Stop must be called when the generator won't be used anymore + Stop() +} // PromEventsGeneratorFactory creates a PromEventsGenerator when instanciating a metricset type PromEventsGeneratorFactory func(ms mb.BaseMetricSet) (PromEventsGenerator, error) @@ -64,7 +74,8 @@ type MetricSet struct { includeMetrics []*regexp.Regexp excludeMetrics []*regexp.Regexp namespace string - genPromEvents PromEventsGenerator + promEventsGen PromEventsGenerator + once sync.Once } // MetricSetBuilder returns a builder function for a new Prometheus metricset using @@ -80,7 +91,7 @@ func MetricSetBuilder(namespace string, genFactory PromEventsGeneratorFactory) f return nil, err } - genPromEvents, err := genFactory(base) + promEventsGen, err := genFactory(base) if err != nil { return nil, err } @@ -89,7 +100,7 @@ func MetricSetBuilder(namespace string, genFactory PromEventsGeneratorFactory) f BaseMetricSet: base, prometheus: prometheus, namespace: namespace, - genPromEvents: genPromEvents, + promEventsGen: promEventsGen, } ms.excludeMetrics, err = compilePatternList(config.MetricsFilters.ExcludeMetrics) if err != nil { @@ -106,6 +117,8 @@ func MetricSetBuilder(namespace string, genFactory PromEventsGeneratorFactory) f // Fetch fetches data and reports it func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + m.once.Do(m.promEventsGen.Start) + families, err := m.prometheus.GetFamilies() eventList := map[string]common.MapStr{} if err != nil { @@ -122,7 +135,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { if m.skipFamily(family) { continue } - promEvents := m.genPromEvents(family) + promEvents := m.promEventsGen.GeneratePromEvents(family) for _, promEvent := range promEvents { labelsHash := promEvent.LabelsHash() @@ -163,6 +176,12 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { return nil } +// Close stops the metricset +func (m *MetricSet) Close() error { + m.promEventsGen.Stop() + return nil +} + func (m *MetricSet) addUpEvent(eventList map[string]common.MapStr, up int) { // TODO add this as family so it follows common code paths metricName := "up" diff --git a/metricbeat/module/prometheus/collector/collector_test.go b/metricbeat/module/prometheus/collector/collector_test.go index 105361ada161..35a231c17f0d 100644 --- a/metricbeat/module/prometheus/collector/collector_test.go +++ b/metricbeat/module/prometheus/collector/collector_test.go @@ -196,8 +196,9 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { }, } + p := promEventGenerator{} for _, test := range tests { - event := DefaultPromEventsGenerator(test.Family) + event := p.GeneratePromEvents(test.Family) assert.Equal(t, test.Event, event) } } diff --git a/metricbeat/module/prometheus/collector/data.go b/metricbeat/module/prometheus/collector/data.go index 0cdb74e1bc75..356e7f4c239a 100644 --- a/metricbeat/module/prometheus/collector/data.go +++ b/metricbeat/module/prometheus/collector/data.go @@ -41,12 +41,17 @@ func (p *PromEvent) LabelsHash() string { // DefaultPromEventsGeneratorFactory returns the default prometheus events generator func DefaultPromEventsGeneratorFactory(ms mb.BaseMetricSet) (PromEventsGenerator, error) { - return DefaultPromEventsGenerator, nil + return &promEventGenerator{}, nil } +type promEventGenerator struct{} + +func (p *promEventGenerator) Start() {} +func (p *promEventGenerator) Stop() {} + // DefaultPromEventsGenerator stores all Prometheus metrics using // only double field type in Elasticsearch. -func DefaultPromEventsGenerator(mf *dto.MetricFamily) []PromEvent { +func (p *promEventGenerator) GeneratePromEvents(mf *dto.MetricFamily) []PromEvent { var events []PromEvent name := *mf.Name diff --git a/x-pack/metricbeat/module/prometheus/collector/data.go b/x-pack/metricbeat/module/prometheus/collector/data.go index 8e24b34a45d2..d14ecf977750 100644 --- a/x-pack/metricbeat/module/prometheus/collector/data.go +++ b/x-pack/metricbeat/module/prometheus/collector/data.go @@ -28,7 +28,6 @@ func promEventsGeneratorFactory(base mb.BaseMetricSet) (collector.PromEventsGene // use a counter cache with a timeout of 5x the period, as a safe value // to make sure that all counters are available between fetches counters := NewCounterCache(base.Module().Config().Period * 5) - counters.Start() if config.RateCounters { cfgwarn.Beta("Prometheus 'rate_counters' settings is experimental") @@ -39,11 +38,10 @@ func promEventsGeneratorFactory(base mb.BaseMetricSet) (collector.PromEventsGene rateCounters: config.RateCounters, } - // TODO cache cleanup on module stop, probably make this an interface - return g.promEventsGenerator, nil + return &g, nil } - return collector.DefaultPromEventsGenerator, nil + return collector.DefaultPromEventsGeneratorFactory(base) } type typedGenerator struct { @@ -51,9 +49,17 @@ type typedGenerator struct { rateCounters bool } -// promEventsGenerator stores all Prometheus metrics using +func (g *typedGenerator) Start() { + g.counterCache.Start() +} + +func (g *typedGenerator) Stop() { + g.counterCache.Stop() +} + +// GeneratePromEvents stores all Prometheus metrics using // only double field type in Elasticsearch. -func (g *typedGenerator) promEventsGenerator(mf *dto.MetricFamily) []collector.PromEvent { +func (g *typedGenerator) GeneratePromEvents(mf *dto.MetricFamily) []collector.PromEvent { var events []collector.PromEvent name := *mf.Name diff --git a/x-pack/metricbeat/module/prometheus/collector/histogram.go b/x-pack/metricbeat/module/prometheus/collector/histogram.go index 1f4435c69290..d4770f59386d 100644 --- a/x-pack/metricbeat/module/prometheus/collector/histogram.go +++ b/x-pack/metricbeat/module/prometheus/collector/histogram.go @@ -19,7 +19,7 @@ func promHistogramToES(cc CounterCache, name string, labels common.MapStr, histo var counts []uint64 // calculate centroids and rated counts - var lastUpper float64 + var lastUpper, prevUpper float64 var sumCount uint64 for _, bucket := range histogram.GetBucket() { // Ignore non-numbers @@ -27,9 +27,10 @@ func promHistogramToES(cc CounterCache, name string, labels common.MapStr, histo continue } + prevUpper = lastUpper if bucket.GetUpperBound() == math.Inf(0) { - // Report +Inf bucket as a point - values = append(values, lastUpper) + // Report +Inf bucket as a point, interpolating its value + values = append(values, lastUpper+(lastUpper-prevUpper)) } else { // calculate bucket centroid values = append(values, lastUpper+(bucket.GetUpperBound()-lastUpper)/2.0) From d5486d56822b8cb1b8caf48717a5a6fa3f5cb93c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Wed, 18 Mar 2020 17:13:25 +0100 Subject: [PATCH 08/32] Remove duplicated docs --- .../prometheus/collector/_meta/docs.asciidoc | 77 ------------------- 1 file changed, 77 deletions(-) delete mode 100644 x-pack/metricbeat/module/prometheus/collector/_meta/docs.asciidoc diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/docs.asciidoc b/x-pack/metricbeat/module/prometheus/collector/_meta/docs.asciidoc deleted file mode 100644 index 82d58cecab28..000000000000 --- a/x-pack/metricbeat/module/prometheus/collector/_meta/docs.asciidoc +++ /dev/null @@ -1,77 +0,0 @@ -The Prometheus `collector` metricset scrapes data from https://prometheus.io/docs/instrumenting/exporters/[prometheus exporters]. - - -[float] -=== Scraping from a Prometheus exporter - -To scrape metrics from a Prometheus exporter, configure the `hosts` field to it. The path -to retrieve the metrics from (`/metrics` by default) can be configured with `metrics_path`. - -[source,yaml] -------------------------------------------------------------------------------------- -- module: prometheus - period: 10s - hosts: ["node:9100"] - metrics_path: /metrics - - #username: "user" - #password: "secret" - - # This can be used for service account based authorization: - #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - #ssl.certificate_authorities: - # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt -------------------------------------------------------------------------------------- - - -[float] -=== Scraping all metrics from a Prometheus server - -This module can scrape all metrics stored in a Prometheus server, by using the -https://prometheus.io/docs/prometheus/latest/federation/[federation API]. By pointing this -config to the Prometheus server: - -[source,yaml] -------------------------------------------------------------------------------------- -metricbeat.modules: -- module: prometheus - period: 10s - hosts: ["localhost:9090"] - metrics_path: '/federate' - query: - 'match[]': '{__name__!=""}' -------------------------------------------------------------------------------------- - -[float] -=== Filtering metrics - -In order to filter out/in metrics one can make use of `metrics_filters.include` `metrics_filters.exclude` settings: - -[source,yaml] -------------------------------------------------------------------------------------- -- module: prometheus - period: 10s - hosts: ["localhost:9090"] - metrics_path: /metrics - metrics_filters: - include: ["node_filesystem_*"] - exclude: ["node_filesystem_device_*"] -------------------------------------------------------------------------------------- - -The configuration above will include only metrics that match `node_filesystem_*` pattern and do not match `node_filesystem_device_*`. - - -To keep only specific metrics, anchor the start and the end of the regexp of each metric: - -- the caret ^ matches the beginning of a text or line, -- the dollar sign $ matches the end of a text. - -[source,yaml] -------------------------------------------------------------------------------------- -- module: prometheus - period: 10s - hosts: ["localhost:9090"] - metrics_path: /metrics - metrics_filters: - include: ["^node_network_net_dev_group$", "^node_network_up$"] -------------------------------------------------------------------------------------- From cf92843b293886b0d51eb716a8e6c1f3deefcdd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Fri, 20 Mar 2020 15:20:07 +0100 Subject: [PATCH 09/32] Do config warnings on start (not config) time --- .../metricbeat/module/prometheus/collector/data.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/metricbeat/module/prometheus/collector/data.go b/x-pack/metricbeat/module/prometheus/collector/data.go index d14ecf977750..cdcd0e4918c1 100644 --- a/x-pack/metricbeat/module/prometheus/collector/data.go +++ b/x-pack/metricbeat/module/prometheus/collector/data.go @@ -23,16 +23,10 @@ func promEventsGeneratorFactory(base mb.BaseMetricSet) (collector.PromEventsGene } if config.UseTypes { - cfgwarn.Beta("Prometheus 'use_types' settings is beta") - // use a counter cache with a timeout of 5x the period, as a safe value // to make sure that all counters are available between fetches counters := NewCounterCache(base.Module().Config().Period * 5) - if config.RateCounters { - cfgwarn.Beta("Prometheus 'rate_counters' settings is experimental") - } - g := typedGenerator{ counterCache: counters, rateCounters: config.RateCounters, @@ -50,6 +44,12 @@ type typedGenerator struct { } func (g *typedGenerator) Start() { + cfgwarn.Beta("Prometheus 'use_types' setting is beta") + + if g.rateCounters { + cfgwarn.Beta("Prometheus 'rate_counters' setting is experimental") + } + g.counterCache.Start() } From 2f84fb17b5bca123f7602741a3cb402764c62b25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Fri, 20 Mar 2020 16:30:59 +0100 Subject: [PATCH 10/32] Rate counters --- .../module/prometheus/collector/data.go | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/x-pack/metricbeat/module/prometheus/collector/data.go b/x-pack/metricbeat/module/prometheus/collector/data.go index cdcd0e4918c1..8882f6d481fa 100644 --- a/x-pack/metricbeat/module/prometheus/collector/data.go +++ b/x-pack/metricbeat/module/prometheus/collector/data.go @@ -80,9 +80,7 @@ func (g *typedGenerator) GeneratePromEvents(mf *dto.MetricFamily) []collector.Pr if !math.IsNaN(counter.GetValue()) && !math.IsInf(counter.GetValue(), 0) { events = append(events, collector.PromEvent{ Data: common.MapStr{ - name: common.MapStr{ - "counter": counter.GetValue(), - }, + name: g.rateCounterFloat64(name, labels, counter.GetValue()), }, Labels: labels, }) @@ -108,12 +106,8 @@ func (g *typedGenerator) GeneratePromEvents(mf *dto.MetricFamily) []collector.Pr if !math.IsNaN(summary.GetSampleSum()) && !math.IsInf(summary.GetSampleSum(), 0) { events = append(events, collector.PromEvent{ Data: common.MapStr{ - name + "_sum": common.MapStr{ - "counter": summary.GetSampleSum(), - }, - name + "_count": common.MapStr{ - "counter": summary.GetSampleCount(), - }, + name + "_sum": g.rateCounterFloat64(name, labels, summary.GetSampleSum()), + name + "_count": g.rateCounterUint64(name, labels, summary.GetSampleCount()), }, Labels: labels, }) @@ -149,15 +143,7 @@ func (g *typedGenerator) GeneratePromEvents(mf *dto.MetricFamily) []collector.Pr }) /* TODO convert histogram to ES type - if !math.IsNaN(histogram.GetSampleSum()) && !math.IsInf(histogram.GetSampleSum(), 0) { - events = append(events, collector.PromEvent{ - Data: common.MapStr{ - name + "_sum.counter": histogram.GetSampleSum(), - name + "_count.counter": histogram.GetSampleCount(), - }, - Labels: labels, - }) - } + Send sum & count? not sure it's worth it */ } @@ -177,3 +163,29 @@ func (g *typedGenerator) GeneratePromEvents(mf *dto.MetricFamily) []collector.Pr } return events } + +// rateCounterUint64 fills a counter value and optionally adds the rate if rate_counters is enabled +func (g *typedGenerator) rateCounterUint64(name string, labels common.MapStr, value uint64) common.MapStr { + d := common.MapStr{ + "counter": value, + } + + if g.rateCounters { + d["rate"] = g.counterCache.RateUint64(name+labels.String(), value) + } + + return d +} + +// rateCounterFloat64 fills a counter value and optionally adds the rate if rate_counters is enabled +func (g *typedGenerator) rateCounterFloat64(name string, labels common.MapStr, value float64) common.MapStr { + d := common.MapStr{ + "counter": value, + } + + if g.rateCounters { + d["rate"] = g.counterCache.RateFloat64(name+labels.String(), value) + } + + return d +} From e76e792512c82e9d986aaf0657613aa7476da4bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Sat, 21 Mar 2020 22:34:07 +0100 Subject: [PATCH 11/32] Unify `up` event codepaths --- .../module/prometheus/collector/collector.go | 61 +++++++++++-------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/metricbeat/module/prometheus/collector/collector.go b/metricbeat/module/prometheus/collector/collector.go index e1a9ec2d23d2..95c94c56ff24 100644 --- a/metricbeat/module/prometheus/collector/collector.go +++ b/metricbeat/module/prometheus/collector/collector.go @@ -42,6 +42,12 @@ var ( DefaultPath: defaultPath, PathConfigKey: "metrics_path", }.Build() + + upMetricName = "up" + upMetricType = dto.MetricType_GAUGE + upMetricInstanceLabel = "instance" + upMetricJobLabel = "job" + upMetricJobValue = "prometheus" ) func init() { @@ -76,6 +82,7 @@ type MetricSet struct { namespace string promEventsGen PromEventsGenerator once sync.Once + host string } // MetricSetBuilder returns a builder function for a new Prometheus metricset using @@ -102,6 +109,7 @@ func MetricSetBuilder(namespace string, genFactory PromEventsGeneratorFactory) f namespace: namespace, promEventsGen: promEventsGen, } + ms.host = ms.Host() ms.excludeMetrics, err = compilePatternList(config.MetricsFilters.ExcludeMetrics) if err != nil { return nil, errors.Wrapf(err, "unable to compile exclude patterns") @@ -122,13 +130,14 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { families, err := m.prometheus.GetFamilies() eventList := map[string]common.MapStr{} if err != nil { - m.addUpEvent(eventList, 0) - for _, evt := range eventList { - reporter.Event(mb.Event{ - RootFields: common.MapStr{m.namespace: evt}, - }) - } - return errors.Wrap(err, "unable to decode response from prometheus endpoint") + // send up event only + families = append(families, m.upMetricFamily(0.0)) + + // set the error to report it after sending the up event + err = errors.Wrap(err, "unable to decode response from prometheus endpoint") + } else { + // add up event to the list + families = append(families, m.upMetricFamily(1.0)) } for _, family := range families { @@ -161,8 +170,6 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { } } - m.addUpEvent(eventList, 1) - // Report events for _, e := range eventList { isOpen := reporter.Event(mb.Event{ @@ -173,7 +180,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { } } - return nil + return err } // Close stops the metricset @@ -182,25 +189,27 @@ func (m *MetricSet) Close() error { return nil } -func (m *MetricSet) addUpEvent(eventList map[string]common.MapStr, up int) { - // TODO add this as family so it follows common code paths - metricName := "up" - if m.skipFamilyName(metricName) { - return +func (m *MetricSet) upMetricFamily(value float64) *dto.MetricFamily { + gauge := dto.Gauge{ + Value: &value, } - upPromEvent := PromEvent{ - Labels: common.MapStr{ - "instance": m.Host(), - "job": "prometheus", - }, + label1 := dto.LabelPair{ + Name: &upMetricInstanceLabel, + Value: &m.host, } - eventList[upPromEvent.LabelsHash()] = common.MapStr{ - "metrics": common.MapStr{ - "up": up, - }, - "labels": upPromEvent.Labels, + label2 := dto.LabelPair{ + Name: &upMetricJobLabel, + Value: &upMetricJobValue, + } + metric := dto.Metric{ + Gauge: &gauge, + Label: []*dto.LabelPair{&label1, &label2}, + } + return &dto.MetricFamily{ + Name: &upMetricName, + Type: &upMetricType, + Metric: []*dto.Metric{&metric}, } - } func (m *MetricSet) skipFamily(family *dto.MetricFamily) bool { From 85e9799efef1a945dc8c8c4e63d3763ec5bfe4ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Sat, 21 Mar 2020 22:45:48 +0100 Subject: [PATCH 12/32] Update config options --- metricbeat/docs/fields.asciidoc | 48 +++++++++++++++++++ x-pack/metricbeat/metricbeat.reference.yml | 22 +++++++-- .../module/prometheus/_meta/config.yml | 14 +++++- .../module/prometheus/collector/data.go | 2 +- x-pack/metricbeat/modules.d/prometheus.yml | 8 ---- 5 files changed, 80 insertions(+), 14 deletions(-) delete mode 100644 x-pack/metricbeat/modules.d/prometheus.yml diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 77d629110286..f39400fd3f45 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -66,6 +66,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -30597,6 +30598,53 @@ type: object Prometheus metric +type: object + +-- + +[[exported-fields-prometheus]] +== Prometheus fields + +Stats scraped from a Prometheus endpoint. + + + +*`prometheus.*.value`*:: ++ +-- +Prometheus gauge metric + + +type: object + +-- + +*`prometheus.*.counter`*:: ++ +-- +Prometheus counter metric + + +type: object + +-- + +*`prometheus.*.rate`*:: ++ +-- +Prometheus rated counter metric + + +type: object + +-- + +*`prometheus.*.histogram`*:: ++ +-- +Prometheus histogram metric - release: ga + + type: object -- diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index db8c09345ae3..19ff72da3c16 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -987,10 +987,26 @@ metricbeat.modules: #------------------------------ Prometheus Module ------------------------------ - module: prometheus - metricsets: ['collector'] period: 10s - hosts: ['localhost:9090'] - # TODO + hosts: ["localhost:9090"] + metrics_path: /metrics + #metrics_filters: + # include: [] + # exclude: [] + #username: "user" + #password: "secret" + + # This can be used for service account based authorization: + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + + # Use Elasticsearch histogram type to store histograms (beta, default: false) + # This will change the default layout and put metric type in the field name + #use_types: true + + # Store counter rates instead of original cumulative counters (experimental, default: false) + #rate_counters: true #------------------------------ Prometheus Module ------------------------------ - module: prometheus period: 10s diff --git a/x-pack/metricbeat/module/prometheus/_meta/config.yml b/x-pack/metricbeat/module/prometheus/_meta/config.yml index d51a2550dca1..89e819e0c218 100644 --- a/x-pack/metricbeat/module/prometheus/_meta/config.yml +++ b/x-pack/metricbeat/module/prometheus/_meta/config.yml @@ -1,7 +1,17 @@ - module: prometheus - metricsets: ['collector'] period: 10s - hosts: ['localhost:9090'] + hosts: ["localhost:9090"] + metrics_path: /metrics + #metrics_filters: + # include: [] + # exclude: [] + #username: "user" + #password: "secret" + + # This can be used for service account based authorization: + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt # Use Elasticsearch histogram type to store histograms (beta, default: false) # This will change the default layout and put metric type in the field name diff --git a/x-pack/metricbeat/module/prometheus/collector/data.go b/x-pack/metricbeat/module/prometheus/collector/data.go index 8882f6d481fa..1f1c59b1f345 100644 --- a/x-pack/metricbeat/module/prometheus/collector/data.go +++ b/x-pack/metricbeat/module/prometheus/collector/data.go @@ -47,7 +47,7 @@ func (g *typedGenerator) Start() { cfgwarn.Beta("Prometheus 'use_types' setting is beta") if g.rateCounters { - cfgwarn.Beta("Prometheus 'rate_counters' setting is experimental") + cfgwarn.Experimental("Prometheus 'rate_counters' setting is experimental") } g.counterCache.Start() diff --git a/x-pack/metricbeat/modules.d/prometheus.yml b/x-pack/metricbeat/modules.d/prometheus.yml deleted file mode 100644 index 85b3c089e048..000000000000 --- a/x-pack/metricbeat/modules.d/prometheus.yml +++ /dev/null @@ -1,8 +0,0 @@ -# Module: prometheus -# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-prometheus.html - -- module: prometheus - metricsets: ['collector'] - period: 10s - hosts: ['localhost:9100'] - use_types: true \ No newline at end of file From a75753ba69abd4d23e685d1532d89bf0841d0571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Sat, 21 Mar 2020 22:50:36 +0100 Subject: [PATCH 13/32] Update x-pack/metricbeat/module/prometheus/collector/histogram.go Co-Authored-By: Chris Mark --- x-pack/metricbeat/module/prometheus/collector/histogram.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/prometheus/collector/histogram.go b/x-pack/metricbeat/module/prometheus/collector/histogram.go index d4770f59386d..63836f2765cc 100644 --- a/x-pack/metricbeat/module/prometheus/collector/histogram.go +++ b/x-pack/metricbeat/module/prometheus/collector/histogram.go @@ -13,7 +13,7 @@ import ( dto "github.com/prometheus/client_model/go" ) -// promHistogramToES takes a Prometheus histogram and converts it to +// promHistogramToES takes a Prometheus histogram and converts it to an ES histogram func promHistogramToES(cc CounterCache, name string, labels common.MapStr, histogram *dto.Histogram) common.MapStr { var values []float64 var counts []uint64 From f05f1cee27f490e302f10d7056c137a35d9c7989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Sat, 21 Mar 2020 23:09:28 +0100 Subject: [PATCH 14/32] Add prometheus.yml.disabled --- .../modules.d/prometheus.yml.disabled | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 x-pack/metricbeat/modules.d/prometheus.yml.disabled diff --git a/x-pack/metricbeat/modules.d/prometheus.yml.disabled b/x-pack/metricbeat/modules.d/prometheus.yml.disabled new file mode 100644 index 000000000000..344142c3ca5a --- /dev/null +++ b/x-pack/metricbeat/modules.d/prometheus.yml.disabled @@ -0,0 +1,24 @@ +# Module: prometheus +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-prometheus.html + +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + metrics_path: /metrics + #metrics_filters: + # include: [] + # exclude: [] + #username: "user" + #password: "secret" + + # This can be used for service account based authorization: + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + + # Use Elasticsearch histogram type to store histograms (beta, default: false) + # This will change the default layout and put metric type in the field name + #use_types: true + + # Store counter rates instead of original cumulative counters (experimental, default: false) + #rate_counters: true \ No newline at end of file From ade341ecbd559b689e8ffad7ee4695a2f4f17e14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Sat, 21 Mar 2020 23:39:15 +0100 Subject: [PATCH 15/32] Fix docs --- metricbeat/docs/fields.asciidoc | 6 +++--- x-pack/metricbeat/metricbeat.reference.yml | 10 ++++------ x-pack/metricbeat/module/prometheus/_meta/fields.yml | 4 ++-- x-pack/metricbeat/module/prometheus/fields.go | 2 +- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 785f633db3f9..bef06098fbc8 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -66,7 +66,7 @@ grouped in the following categories: * <> * <> * <> -* <> +* <> * <> * <> * <> @@ -31624,8 +31624,8 @@ type: object remote write metrics from Prometheus server -[[exported-fields-prometheus]] -== Prometheus fields +[[exported-fields-prometheus-xpack]] +== Prometheus typed metrics fields Stats scraped from a Prometheus endpoint. diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 899d4a9453d2..056ede0b98aa 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -996,11 +996,9 @@ metricbeat.modules: # Password to use when connecting to PostgreSQL. Empty by default. #password: pass -#------------------------------ Prometheus Module ------------------------------ -# Metrics collected from a Prometheus endpoint +#----------------------- Prometheus Typed Metrics Module ----------------------- - module: prometheus period: 10s - metricsets: ["collector"] hosts: ["localhost:9090"] metrics_path: /metrics #metrics_filters: @@ -1014,7 +1012,6 @@ metricbeat.modules: #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt -<<<<<<< HEAD # Use Elasticsearch histogram type to store histograms (beta, default: false) # This will change the default layout and put metric type in the field name #use_types: true @@ -1022,8 +1019,10 @@ metricbeat.modules: # Store counter rates instead of original cumulative counters (experimental, default: false) #rate_counters: true #------------------------------ Prometheus Module ------------------------------ +# Metrics collected from a Prometheus endpoint - module: prometheus period: 10s + metricsets: ["collector"] hosts: ["localhost:9090"] metrics_path: /metrics #metrics_filters: @@ -1036,7 +1035,7 @@ metricbeat.modules: #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt -======= + # Metrics sent by a Prometheus server using remote_write option - module: prometheus metricsets: ["remote_write"] @@ -1046,7 +1045,6 @@ metricbeat.modules: # Secure settings for the server using TLS/SSL: #ssl.certificate: "/etc/pki/server/cert.pem" #ssl.key: "/etc/pki/server/cert.key" ->>>>>>> master #------------------------------- RabbitMQ Module ------------------------------- - module: rabbitmq diff --git a/x-pack/metricbeat/module/prometheus/_meta/fields.yml b/x-pack/metricbeat/module/prometheus/_meta/fields.yml index b8fceb18c68e..797d46d4e287 100644 --- a/x-pack/metricbeat/module/prometheus/_meta/fields.yml +++ b/x-pack/metricbeat/module/prometheus/_meta/fields.yml @@ -1,5 +1,5 @@ -- key: prometheus - title: "Prometheus" +- key: prometheus-xpack + title: "Prometheus typed metrics" description: > Stats scraped from a Prometheus endpoint. release: ga diff --git a/x-pack/metricbeat/module/prometheus/fields.go b/x-pack/metricbeat/module/prometheus/fields.go index 74ab153081bf..63330de9110b 100644 --- a/x-pack/metricbeat/module/prometheus/fields.go +++ b/x-pack/metricbeat/module/prometheus/fields.go @@ -19,5 +19,5 @@ func init() { // AssetPrometheus returns asset data. // This is the base64 encoded gzipped contents of module/prometheus. func AssetPrometheus() string { - return "eJzE0ktqMzEMB/C9T/HHy/AlB/DiO0Ohy1KCMlY8bv3C0hRy+5LMkA590CwK8c6SbP1kvMUrnxxar5l15EkMoFETO9iHa9AawLMMPTaNtTj8NwDwqKQCGTo19jj2mkH4OAUuvtVYdGeAzolJ2CGQAYRVYwni8GRFkv0HO6o2+2yAY+TkxV06bFEo89q32+zeKE18SQN6auxQDy886BKaN/s54+t0SPw1s8/UWixhKbMbu9R8M+Z5raYKNAVGZu1x+Bk51Kko9/sxF8Cv0E56x8c8d/c3W8coWkOnfCP4c/3fmK+3rr2zef3L3wMAAP//PRYNew==" + return "eJzEkktqAzEMhvdzih8vQ5MDeNEzFLosJSi24rjxC0tTmtuXZIYw9EGzKMQ7+/+QPgmvceSTRes1sx54lPVHI3ccAI2a2MI8XSPoqbFHZu3RiRkAz+J6bBprsXgcAOBZSQXiOp3Zfa8ZhEUNLr7VWHQzAJ0Tk7BFoAEQVo0liMWLEUnmAeag2szrAOwjJy/20mGNQpmXzpvV5p3SyJcYF02Luntjp/PTdNlOia/jLvH3ZJuptVjCjJmVmZkfxjyfxVSBxsDzZn6XdHUsyv1+mrPAn6Kd9I7LPHf3N7seomgNnfKNwl/5/3G+Vl36Ts7LX/4ZAAD//xMCFRw=" } From e5adae4ec864a7d9e014bb684d3a12ea90dc3ab4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Sun, 22 Mar 2020 00:07:37 +0100 Subject: [PATCH 16/32] Fix dependency --- x-pack/metricbeat/module/prometheus/collector/counter_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/prometheus/collector/counter_test.go b/x-pack/metricbeat/module/prometheus/collector/counter_test.go index d3dabe80d853..3aed4a8d0175 100644 --- a/x-pack/metricbeat/module/prometheus/collector/counter_test.go +++ b/x-pack/metricbeat/module/prometheus/collector/counter_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common" ) func Test_CounterCache(t *testing.T) { From 70ea28427d6c8d5635a0bd718677ead1451d6d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Mon, 23 Mar 2020 16:34:15 +0100 Subject: [PATCH 17/32] Fix last bucket math --- x-pack/metricbeat/module/prometheus/collector/histogram.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/prometheus/collector/histogram.go b/x-pack/metricbeat/module/prometheus/collector/histogram.go index 63836f2765cc..501f170c3ecc 100644 --- a/x-pack/metricbeat/module/prometheus/collector/histogram.go +++ b/x-pack/metricbeat/module/prometheus/collector/histogram.go @@ -27,13 +27,13 @@ func promHistogramToES(cc CounterCache, name string, labels common.MapStr, histo continue } - prevUpper = lastUpper if bucket.GetUpperBound() == math.Inf(0) { // Report +Inf bucket as a point, interpolating its value values = append(values, lastUpper+(lastUpper-prevUpper)) } else { // calculate bucket centroid values = append(values, lastUpper+(bucket.GetUpperBound()-lastUpper)/2.0) + prevUpper = lastUpper lastUpper = bucket.GetUpperBound() } From 2265921920339ac41c333bafa81a6911c3f3bb32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Mon, 23 Mar 2020 16:44:13 +0100 Subject: [PATCH 18/32] Add a few clarifications --- .../module/prometheus/collector/collector.go | 1 + .../module/prometheus/collector/collector.go | 3 +++ .../module/prometheus/collector/histogram.go | 15 ++++++++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/prometheus/collector/collector.go b/metricbeat/module/prometheus/collector/collector.go index 95c94c56ff24..fdc52f12b61a 100644 --- a/metricbeat/module/prometheus/collector/collector.go +++ b/metricbeat/module/prometheus/collector/collector.go @@ -109,6 +109,7 @@ func MetricSetBuilder(namespace string, genFactory PromEventsGeneratorFactory) f namespace: namespace, promEventsGen: promEventsGen, } + // store host here to use it as a pointer when building `up` metric ms.host = ms.Host() ms.excludeMetrics, err = compilePatternList(config.MetricsFilters.ExcludeMetrics) if err != nil { diff --git a/x-pack/metricbeat/module/prometheus/collector/collector.go b/x-pack/metricbeat/module/prometheus/collector/collector.go index ecae5802a0f5..92f3552eebd2 100644 --- a/x-pack/metricbeat/module/prometheus/collector/collector.go +++ b/x-pack/metricbeat/module/prometheus/collector/collector.go @@ -14,6 +14,9 @@ func init() { collector.MetricSetBuilder("prometheus", promEventsGeneratorFactory), mb.WithHostParser(collector.HostParser), mb.DefaultMetricSet(), + + // must replace ensures that we are replacing the oss implementation with this one + // so we can make use of ES histograms (basic only) when use_types is enabled mb.MustReplace(), ) } diff --git a/x-pack/metricbeat/module/prometheus/collector/histogram.go b/x-pack/metricbeat/module/prometheus/collector/histogram.go index 501f170c3ecc..8a62cbf8b971 100644 --- a/x-pack/metricbeat/module/prometheus/collector/histogram.go +++ b/x-pack/metricbeat/module/prometheus/collector/histogram.go @@ -13,7 +13,20 @@ import ( dto "github.com/prometheus/client_model/go" ) -// promHistogramToES takes a Prometheus histogram and converts it to an ES histogram +// promHistogramToES takes a Prometheus histogram and converts it to an ES histogram: +// +// ES histograms look like this: +// +// "histogram_field" : { +// "values" : [0.1, 0.2, 0.3, 0.4, 0.5], +// "counts" : [3, 7, 23, 12, 6] +// } +// +// This code takes a Prometheus histogram and tries to accomodate it into an ES histogram by: +// - calculating centroids for each bucket (values) +// - undoing counters accumulation for each bucket (counts) +// +// https://www.elastic.co/guide/en/elasticsearch/reference/master/histogram.html func promHistogramToES(cc CounterCache, name string, labels common.MapStr, histogram *dto.Histogram) common.MapStr { var values []float64 var counts []uint64 From 0b0bc0404073c67afe4bf6e58b84e201ee070c55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Mon, 23 Mar 2020 16:56:40 +0100 Subject: [PATCH 19/32] Add tests --- .../module/prometheus/_meta/Dockerfile | 4 +++ .../prometheus/collector/_meta/data.json | 18 +++++++--- .../collector/collector_integration_test.go | 36 +++++++++++++++++++ .../module/prometheus/docker-compose.yml | 11 ++++++ 4 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 x-pack/metricbeat/module/prometheus/_meta/Dockerfile create mode 100644 x-pack/metricbeat/module/prometheus/collector/collector_integration_test.go create mode 100644 x-pack/metricbeat/module/prometheus/docker-compose.yml diff --git a/x-pack/metricbeat/module/prometheus/_meta/Dockerfile b/x-pack/metricbeat/module/prometheus/_meta/Dockerfile new file mode 100644 index 000000000000..7f8d7c77a3e7 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/_meta/Dockerfile @@ -0,0 +1,4 @@ +ARG PROMETHEUS_VERSION +FROM prom/prometheus:v${PROMETHEUS_VERSION} +HEALTHCHECK --interval=1s --retries=90 CMD nc -w 1 localhost 9090 Date: Tue, 24 Mar 2020 10:48:47 +0100 Subject: [PATCH 20/32] restore prometheus image --- x-pack/metricbeat/module/prometheus/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/prometheus/docker-compose.yml b/x-pack/metricbeat/module/prometheus/docker-compose.yml index 7afe1f702a1d..47f7626b291b 100644 --- a/x-pack/metricbeat/module/prometheus/docker-compose.yml +++ b/x-pack/metricbeat/module/prometheus/docker-compose.yml @@ -2,7 +2,7 @@ version: '2.3' services: prometheus: - #image: docker.elastic.co/integrations-ci/beats-prometheus:${PROMETHEUS_VERSION:-2.6.0}-2 + image: docker.elastic.co/integrations-ci/beats-prometheus:${PROMETHEUS_VERSION:-2.6.0}-2 build: context: ./_meta args: From 969c233fa8b59810831407a6d0f1d5a3001fb980 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 24 Mar 2020 10:51:07 +0100 Subject: [PATCH 21/32] rate counters in data.json --- .../module/prometheus/collector/_meta/data.json | 17 ++++++++--------- .../collector/collector_integration_test.go | 9 +++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/data.json b/x-pack/metricbeat/module/prometheus/collector/_meta/data.json index 53458194fa92..f0fed23d976d 100644 --- a/x-pack/metricbeat/module/prometheus/collector/_meta/data.json +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/data.json @@ -12,17 +12,16 @@ "prometheus": { "labels": { "instance": "172.27.0.2:9090", - "job": "prometheus", - "scrape_job": "prometheus" + "interval": "15s", + "job": "prometheus" }, - "prometheus_target_scrape_pool_sync_total": { - "counter": 1 + "prometheus_target_interval_length_seconds_count": { + "counter": 1, + "rate": 0 }, - "prometheus_target_sync_length_seconds_count": { - "counter": 1 - }, - "prometheus_target_sync_length_seconds_sum": { - "counter": 0.000046958 + "prometheus_target_interval_length_seconds_sum": { + "counter": 15.000401344, + "rate": 0 } }, "service": { diff --git a/x-pack/metricbeat/module/prometheus/collector/collector_integration_test.go b/x-pack/metricbeat/module/prometheus/collector/collector_integration_test.go index 9cc3e8e6b5e9..179a97c75ce2 100644 --- a/x-pack/metricbeat/module/prometheus/collector/collector_integration_test.go +++ b/x-pack/metricbeat/module/prometheus/collector/collector_integration_test.go @@ -18,10 +18,11 @@ func TestData(t *testing.T) { service := compose.EnsureUp(t, "prometheus") config := map[string]interface{}{ - "module": "prometheus", - "metricsets": []string{"collector"}, - "hosts": []string{service.Host()}, - "use_types": true, + "module": "prometheus", + "metricsets": []string{"collector"}, + "hosts": []string{service.Host()}, + "use_types": true, + "rate_counters": true, } ms := mbtest.NewReportingMetricSetV2Error(t, config) var err error From 78d1941f31de7865c6e75a6081f4ef46d1cd85b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 24 Mar 2020 10:52:53 +0100 Subject: [PATCH 22/32] Update changelog --- CHANGELOG.next.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d456c72d4f2d..e971e75e9b2b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -254,6 +254,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Release Zookeeper/connection module as GA. {issue}14281[14281] {pull}17043[17043] - Add support for CouchDB v2 {issue}16352[16352] {pull}16455[16455] - Replace vpc metricset into vpn, transitgateway and natgateway metricsets. {pull}16892[16892] +- Use Elasticsearch histogram type to store Prometheus histograms {pull}17061[17061] +- Allow to rate Prometheus counters when scraping them {pull}17061[17061] *Packetbeat* From 174b4fa930904361819aca35aab7854541e15750 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 24 Mar 2020 11:27:27 +0100 Subject: [PATCH 23/32] Update docs --- .../prometheus/collector/_meta/docs.asciidoc | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/metricbeat/module/prometheus/collector/_meta/docs.asciidoc b/metricbeat/module/prometheus/collector/_meta/docs.asciidoc index 82d58cecab28..7270737178b9 100644 --- a/metricbeat/module/prometheus/collector/_meta/docs.asciidoc +++ b/metricbeat/module/prometheus/collector/_meta/docs.asciidoc @@ -24,6 +24,58 @@ to retrieve the metrics from (`/metrics` by default) can be configured with `met ------------------------------------------------------------------------------------- +[float] +[role="xpack"] +=== Histograms and types + +beta[] + +[source,yaml] +------------------------------------------------------------------------------------- +metricbeat.modules: +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + use_types: true + rate_counters: false +------------------------------------------------------------------------------------- + +`use_types` paramater (default: false) enables a different layout for metrics storage, leveraging Elasticsearch +types, including https://www.elastic.co/guide/en/elasticsearch/reference/current/histogram.html[histograms]. + +`rate_counters` paramater (default: false) enables calculating a rate out of Prometheus counters. It will stored +the counter increment since the last collection. This metric should make some aggregations easier and with better +performance. This parameter can only be enabled in combination with `use_types`. + +When `use_types` and `rate_counters` are enabled, metrics will be stored like this: + +[source,json] +---- +{ + "prometheus": { + "labels": { + "instance": "172.27.0.2:9090", + "job": "prometheus" + }, + "prometheus_target_interval_length_seconds_count": { + "counter": 1, + "rate": 0 + }, + "prometheus_target_interval_length_seconds_sum": { + "counter": 15.000401344, + "rate": 0 + } + "prometheus_tsdb_compaction_chunk_range_seconds_bucket": { + "histogram": { + "values": [50, 300, 1000, 4000, 16000], + "counts": [10, 2, 34, 7] + } + } + }, +} +---- + + [float] === Scraping all metrics from a Prometheus server From 5716c4e43bce3195fbdea58512c60bea1e91c05d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 24 Mar 2020 14:21:54 +0100 Subject: [PATCH 24/32] Update metricbeat/module/prometheus/collector/_meta/docs.asciidoc Co-Authored-By: Jaime Soriano Pastor --- metricbeat/module/prometheus/collector/_meta/docs.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/prometheus/collector/_meta/docs.asciidoc b/metricbeat/module/prometheus/collector/_meta/docs.asciidoc index 7270737178b9..da6eb7d29c6b 100644 --- a/metricbeat/module/prometheus/collector/_meta/docs.asciidoc +++ b/metricbeat/module/prometheus/collector/_meta/docs.asciidoc @@ -47,7 +47,7 @@ types, including https://www.elastic.co/guide/en/elasticsearch/reference/current the counter increment since the last collection. This metric should make some aggregations easier and with better performance. This parameter can only be enabled in combination with `use_types`. -When `use_types` and `rate_counters` are enabled, metrics will be stored like this: +When `use_types` and `rate_counters` are enabled, metrics are stored like this: [source,json] ---- From de1973f304e6d72177a3ad7c59dbeee04a3481d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 24 Mar 2020 14:22:39 +0100 Subject: [PATCH 25/32] Update metricbeat/module/prometheus/collector/collector.go Co-Authored-By: Jaime Soriano Pastor --- metricbeat/module/prometheus/collector/collector.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/prometheus/collector/collector.go b/metricbeat/module/prometheus/collector/collector.go index fdc52f12b61a..91cf1b7e40dc 100644 --- a/metricbeat/module/prometheus/collector/collector.go +++ b/metricbeat/module/prometheus/collector/collector.go @@ -153,8 +153,8 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { eventList[labelsHash] = common.MapStr{} // Add default instance label if not already there - if exists, _ := promEvent.Labels.HasKey("instance"); !exists { - promEvent.Labels.Put("instance", m.Host()) + if exists, _ := promEvent.Labels.HasKey(upMetricInstanceLabel); !exists { + promEvent.Labels.Put(upMetricInstanceLabel, m.Host()) } // Add default job label if not already there if exists, _ := promEvent.Labels.HasKey("job"); !exists { From d4bb33383cb4d190f6efc2b607511ae1c3c4b705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 24 Mar 2020 14:23:30 +0100 Subject: [PATCH 26/32] Update x-pack/metricbeat/module/prometheus/collector/data.go Co-Authored-By: Jaime Soriano Pastor --- x-pack/metricbeat/module/prometheus/collector/data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/prometheus/collector/data.go b/x-pack/metricbeat/module/prometheus/collector/data.go index 1f1c59b1f345..8e205b66a2d1 100644 --- a/x-pack/metricbeat/module/prometheus/collector/data.go +++ b/x-pack/metricbeat/module/prometheus/collector/data.go @@ -58,7 +58,7 @@ func (g *typedGenerator) Stop() { } // GeneratePromEvents stores all Prometheus metrics using -// only double field type in Elasticsearch. +// specific Elasticsearch data types. func (g *typedGenerator) GeneratePromEvents(mf *dto.MetricFamily) []collector.PromEvent { var events []collector.PromEvent From 2325fc77abd2d706fb933fe381d5fd30da50bcc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 24 Mar 2020 15:09:20 +0100 Subject: [PATCH 27/32] Update x-pack/metricbeat/module/prometheus/collector/counter.go Co-Authored-By: Jaime Soriano Pastor --- x-pack/metricbeat/module/prometheus/collector/counter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/prometheus/collector/counter.go b/x-pack/metricbeat/module/prometheus/collector/counter.go index b4798044de18..f39e6de17633 100644 --- a/x-pack/metricbeat/module/prometheus/collector/counter.go +++ b/x-pack/metricbeat/module/prometheus/collector/counter.go @@ -21,7 +21,7 @@ type CounterCache interface { // Stop the cache cleanup worker. It mus be called when the cache is disposed Stop() - // RateUUint64 returns, for a given counter name, the difference between the given value + // RateUint64 returns, for a given counter name, the difference between the given value // and the value that was given in a previous call. It will return 0 on the first call RateUint64(counterName string, value uint64) uint64 From 906876fd44fdb20bbc7ec5ba8e94b4872a3720f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 24 Mar 2020 15:27:53 +0100 Subject: [PATCH 28/32] Update metricbeat/module/prometheus/collector/_meta/docs.asciidoc Co-Authored-By: Jaime Soriano Pastor --- metricbeat/module/prometheus/collector/_meta/docs.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/prometheus/collector/_meta/docs.asciidoc b/metricbeat/module/prometheus/collector/_meta/docs.asciidoc index da6eb7d29c6b..e48e57ee2099 100644 --- a/metricbeat/module/prometheus/collector/_meta/docs.asciidoc +++ b/metricbeat/module/prometheus/collector/_meta/docs.asciidoc @@ -43,7 +43,7 @@ metricbeat.modules: `use_types` paramater (default: false) enables a different layout for metrics storage, leveraging Elasticsearch types, including https://www.elastic.co/guide/en/elasticsearch/reference/current/histogram.html[histograms]. -`rate_counters` paramater (default: false) enables calculating a rate out of Prometheus counters. It will stored +`rate_counters` paramater (default: false) enables calculating a rate out of Prometheus counters. When enabled, Metricbeat stores the counter increment since the last collection. This metric should make some aggregations easier and with better performance. This parameter can only be enabled in combination with `use_types`. From 7c89bf31e940a005fd0765b556d6fa0c410b46dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 24 Mar 2020 15:28:11 +0100 Subject: [PATCH 29/32] Reuse prometheus docker-compose --- x-pack/metricbeat/module/prometheus/_meta/Dockerfile | 4 ---- x-pack/metricbeat/module/prometheus/docker-compose.yml | 10 +++------- 2 files changed, 3 insertions(+), 11 deletions(-) delete mode 100644 x-pack/metricbeat/module/prometheus/_meta/Dockerfile diff --git a/x-pack/metricbeat/module/prometheus/_meta/Dockerfile b/x-pack/metricbeat/module/prometheus/_meta/Dockerfile deleted file mode 100644 index 7f8d7c77a3e7..000000000000 --- a/x-pack/metricbeat/module/prometheus/_meta/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -ARG PROMETHEUS_VERSION -FROM prom/prometheus:v${PROMETHEUS_VERSION} -HEALTHCHECK --interval=1s --retries=90 CMD nc -w 1 localhost 9090 Date: Tue, 24 Mar 2020 16:08:20 +0100 Subject: [PATCH 30/32] Add testdata --- metricbeat/mb/testing/testdata.go | 17 +- .../collector/_meta/testdata/config.yml | 8 + .../collector/_meta/testdata/docs.plain | 11 + .../_meta/testdata/docs.plain-expected.json | 251 ++++++++++ .../_meta/testdata/metrics-with-naninf.plain | 38 ++ .../metrics-with-naninf.plain-expected.json | 199 ++++++++ .../testdata/prometheus-2.6.0-partial.plain | 135 ++++++ ...ometheus-2.6.0-partial.plain-expected.json | 444 ++++++++++++++++++ .../prometheus/collector/collector_test.go | 19 + 9 files changed, 1120 insertions(+), 2 deletions(-) create mode 100644 x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml create mode 100644 x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain create mode 100644 x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json create mode 100644 x-pack/metricbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain create mode 100644 x-pack/metricbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain-expected.json create mode 100644 x-pack/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain create mode 100644 x-pack/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain-expected.json create mode 100644 x-pack/metricbeat/module/prometheus/collector/collector_test.go diff --git a/metricbeat/mb/testing/testdata.go b/metricbeat/mb/testing/testdata.go index ca1005e57c37..a0cf85d4b019 100644 --- a/metricbeat/mb/testing/testdata.go +++ b/metricbeat/mb/testing/testdata.go @@ -336,9 +336,22 @@ func documentedFieldCheck(foundKeys common.MapStr, knownKeys map[string]interfac return nil } } - // If a field is defined as object it can also be defined as `status_codes.*` - // So this checks if such a key with the * exists by removing the last part. + // If a field is defined as object it can also have a * somewhere + // So this checks if such a key with the * exists by testing with it splits := strings.Split(foundKey, ".") + found := false + for pos := 1; pos < len(splits)-1; pos++ { + key := strings.Join(splits[0:pos], ".") + ".*." + strings.Join(splits[pos+1:len(splits)], ".") + if _, ok := knownKeys[key]; ok { + found = true + break + } + } + if found { + continue + } + + // last case `status_codes.*`: prefix := strings.Join(splits[0:len(splits)-1], ".") if _, ok := knownKeys[prefix+".*"]; ok { continue diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml new file mode 100644 index 000000000000..875c2a2d6a4d --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml @@ -0,0 +1,8 @@ +type: http +url: "/metrics" +suffix: plain +omit_documented_fields_check: ["prometheus.labels.*"] # this are not mapped by this module but the oss one +remove_fields_from_comparison: ["prometheus.labels.instance"] +module: + use_types: true + rate_counters: true \ No newline at end of file diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain new file mode 100644 index 000000000000..47c3b38aedbe --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain @@ -0,0 +1,11 @@ +# HELP node_network_carrier carrier value of /sys/class/net/. +# TYPE node_network_carrier gauge +node_network_carrier{device="br-0cb306323b90"} 0 +node_network_carrier{device="br-10229e3512d9"} 0 +node_network_carrier{device="br-210476dc4ef8"} 0 +node_network_carrier{device="br-33d819d5f834"} 0 +node_network_carrier{device="br-38425a39f36b"} 0 +node_network_carrier{device="br-38feb0aad6ab"} 0 +node_network_carrier{device="br-3a285aa5e58c"} 0 +node_network_carrier{device="br-425cb4c454a6"} 0 +node_network_carrier{device="br-4e623477470e"} 0 diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json new file mode 100644 index 000000000000..d978670ecc74 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json @@ -0,0 +1,251 @@ +[ + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "device": "br-0cb306323b90", + "instance": "127.0.0.1:38401", + "job": "prometheus" + }, + "node_network_carrier": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "device": "br-210476dc4ef8", + "instance": "127.0.0.1:38401", + "job": "prometheus" + }, + "node_network_carrier": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:38401", + "job": "prometheus" + }, + "up": { + "value": 1 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "device": "br-3a285aa5e58c", + "instance": "127.0.0.1:38401", + "job": "prometheus" + }, + "node_network_carrier": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "device": "br-33d819d5f834", + "instance": "127.0.0.1:38401", + "job": "prometheus" + }, + "node_network_carrier": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "device": "br-4e623477470e", + "instance": "127.0.0.1:38401", + "job": "prometheus" + }, + "node_network_carrier": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "device": "br-10229e3512d9", + "instance": "127.0.0.1:38401", + "job": "prometheus" + }, + "node_network_carrier": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "device": "br-38feb0aad6ab", + "instance": "127.0.0.1:38401", + "job": "prometheus" + }, + "node_network_carrier": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "device": "br-425cb4c454a6", + "instance": "127.0.0.1:38401", + "job": "prometheus" + }, + "node_network_carrier": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "device": "br-38425a39f36b", + "instance": "127.0.0.1:38401", + "job": "prometheus" + }, + "node_network_carrier": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + } +] \ No newline at end of file diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain new file mode 100644 index 000000000000..ea88edb50478 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain @@ -0,0 +1,38 @@ +# HELP kafka_consumer_records_lag_records The latest lag of the partition +# TYPE kafka_consumer_records_lag_records gauge +kafka_consumer_records_lag_records{client_id="consumer1",} NaN +kafka_consumer_records_lag_records{client_id="consumer2",} +Inf +kafka_consumer_records_lag_records{client_id="consumer3",} -Inf +kafka_consumer_records_lag_records{client_id="consumer4",} 5 +# HELP http_failures Total number of http request failures +# TYPE http_failures counter +http_failures{method="GET"} 2 +http_failures{method="POST"} NaN +http_failures{method="DELETE"} +Inf +# HELP go_gc_duration_seconds A summary of the GC invocation durations. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0",} NaN +go_gc_duration_seconds{quantile="0.25",} +Inf +go_gc_duration_seconds{quantile="0.5",} -Inf +go_gc_duration_seconds{quantile="0.75"} 9.8154e-05 +go_gc_duration_seconds{quantile="1",} 0.011689149 +go_gc_duration_seconds_sum 3.451780079 +go_gc_duration_seconds_count 13118 +# HELP http_request_duration_seconds request duration histogram +# TYPE http_request_duration_seconds histogram +http_request_duration_seconds_bucket{le="0.1"} +Inf +http_request_duration_seconds_bucket{le="0.2"} -Inf +http_request_duration_seconds_bucket{le="0.5"} NaN +http_request_duration_seconds_bucket{le="1"} 1 +http_request_duration_seconds_bucket{le="2"} 2 +http_request_duration_seconds_bucket{le="3"} 3 +http_request_duration_seconds_bucket{le="5"} 3 +http_request_duration_seconds_bucket{le="+Inf"} 3 +http_request_duration_seconds_sum 6 +http_request_duration_seconds_count 3 +# HELP net_conntrack_listener_conn_accepted_total Total number of connections opened to the listener of a given name. +# TYPE net_conntrack_listener_conn_accepted_total untyped +net_conntrack_listener_conn_accepted_total{listener_name="http"} 1568652315554 +# HELP net_conntrack_listener_conn_closed_total Total number of connections closed that were made to the listener of a given name. +# TYPE net_conntrack_listener_conn_closed_total untyped +net_conntrack_listener_conn_closed_total{listener_name="http"} NaN diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain-expected.json b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain-expected.json new file mode 100644 index 000000000000..e507e51ba82b --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain-expected.json @@ -0,0 +1,199 @@ +[ + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_gc_duration_seconds_count": { + "counter": 13118, + "rate": 0 + }, + "go_gc_duration_seconds_sum": { + "counter": 3.451780079, + "rate": 0 + }, + "http_request_duration_seconds": { + "histogram": { + "counts": [ + 0, + 0, + 0, + 0, + 0 + ], + "values": [ + 0.5, + 1.5, + 2.5, + 4, + 7 + ] + } + }, + "labels": { + "instance": "127.0.0.1:37675", + "job": "prometheus" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:37675", + "job": "prometheus" + }, + "up": { + "value": 1 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "kafka_consumer_records_lag_records": { + "value": 5 + }, + "labels": { + "client_id": "consumer4", + "instance": "127.0.0.1:37675", + "job": "prometheus" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_gc_duration_seconds": { + "value": 0.011689149 + }, + "labels": { + "instance": "127.0.0.1:37675", + "job": "prometheus", + "quantile": "1" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_gc_duration_seconds": { + "value": 0.000098154 + }, + "labels": { + "instance": "127.0.0.1:37675", + "job": "prometheus", + "quantile": "0.75" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "http_failures": { + "counter": 2, + "rate": 0 + }, + "labels": { + "instance": "127.0.0.1:37675", + "job": "prometheus", + "method": "GET" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:37675", + "job": "prometheus", + "listener_name": "http" + }, + "net_conntrack_listener_conn_accepted_total": { + "value": 1568652315554 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + } +] \ No newline at end of file diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain new file mode 100644 index 000000000000..136196b72c2e --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain @@ -0,0 +1,135 @@ +# HELP go_gc_duration_seconds A summary of the GC invocation durations. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0"} 3.8386e-05 +go_gc_duration_seconds{quantile="0.25"} 4.2803e-05 +go_gc_duration_seconds{quantile="0.5"} 6.0618e-05 +go_gc_duration_seconds{quantile="0.75"} 0.004392391 +go_gc_duration_seconds{quantile="1"} 0.004392391 +go_gc_duration_seconds_sum 0.004534198 +go_gc_duration_seconds_count 4 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines 35 +# HELP go_info Information about the Go environment. +# TYPE go_info gauge +go_info{version="go1.11.3"} 1 +# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. +# TYPE go_memstats_alloc_bytes gauge +go_memstats_alloc_bytes 1.0558112e+07 +# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total 1.408776e+07 +# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. +# TYPE go_memstats_buck_hash_sys_bytes gauge +go_memstats_buck_hash_sys_bytes 1.447018e+06 +# HELP go_memstats_frees_total Total number of frees. +# TYPE go_memstats_frees_total counter +go_memstats_frees_total 15673 +# HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started. +# TYPE go_memstats_gc_cpu_fraction gauge +go_memstats_gc_cpu_fraction 0.0008429952574435172 +# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. +# TYPE go_memstats_gc_sys_bytes gauge +go_memstats_gc_sys_bytes 2.379776e+06 +# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. +# TYPE go_memstats_heap_alloc_bytes gauge +go_memstats_heap_alloc_bytes 1.0558112e+07 +# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. +# TYPE go_memstats_heap_idle_bytes gauge +go_memstats_heap_idle_bytes 5.4042624e+07 +# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. +# TYPE go_memstats_heap_inuse_bytes gauge +go_memstats_heap_inuse_bytes 1.2214272e+07 +# HELP go_memstats_heap_objects Number of allocated objects. +# TYPE go_memstats_heap_objects gauge +go_memstats_heap_objects 61771 +# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. +# TYPE go_memstats_heap_released_bytes gauge +go_memstats_heap_released_bytes 0 +# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. +# TYPE go_memstats_heap_sys_bytes gauge +go_memstats_heap_sys_bytes 6.6256896e+07 +# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. +# TYPE go_memstats_last_gc_time_seconds gauge +go_memstats_last_gc_time_seconds 1.5534303161488917e+09 +# HELP go_memstats_lookups_total Total number of pointer lookups. +# TYPE go_memstats_lookups_total counter +go_memstats_lookups_total 0 +# HELP go_memstats_mallocs_total Total number of mallocs. +# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total 77444 +# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. +# TYPE go_memstats_mcache_inuse_bytes gauge +go_memstats_mcache_inuse_bytes 6912 +# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. +# TYPE go_memstats_mcache_sys_bytes gauge +go_memstats_mcache_sys_bytes 16384 +# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. +# TYPE go_memstats_mspan_inuse_bytes gauge +go_memstats_mspan_inuse_bytes 127984 +# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. +# TYPE go_memstats_mspan_sys_bytes gauge +go_memstats_mspan_sys_bytes 131072 +# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. +# TYPE go_memstats_next_gc_bytes gauge +go_memstats_next_gc_bytes 1.8390112e+07 +# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. +# TYPE go_memstats_other_sys_bytes gauge +go_memstats_other_sys_bytes 1.201294e+06 +# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. +# TYPE go_memstats_stack_inuse_bytes gauge +go_memstats_stack_inuse_bytes 851968 +# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. +# TYPE go_memstats_stack_sys_bytes gauge +go_memstats_stack_sys_bytes 851968 +# HELP go_memstats_sys_bytes Number of bytes obtained from system. +# TYPE go_memstats_sys_bytes gauge +go_memstats_sys_bytes 7.2284408e+07 +# HELP go_threads Number of OS threads created. +# TYPE go_threads gauge +go_threads 14 +# HELP net_conntrack_dialer_conn_attempted_total Total number of connections attempted by the given dialer a given name. +# TYPE net_conntrack_dialer_conn_attempted_total counter +net_conntrack_dialer_conn_attempted_total{dialer_name="alertmanager"} 0 +net_conntrack_dialer_conn_attempted_total{dialer_name="default"} 0 +net_conntrack_dialer_conn_attempted_total{dialer_name="prometheus"} 1 +# HELP net_conntrack_dialer_conn_closed_total Total number of connections closed which originated from the dialer of a given name. +# TYPE net_conntrack_dialer_conn_closed_total counter +net_conntrack_dialer_conn_closed_total{dialer_name="alertmanager"} 0 +net_conntrack_dialer_conn_closed_total{dialer_name="default"} 0 +net_conntrack_dialer_conn_closed_total{dialer_name="prometheus"} 0 +# HELP net_conntrack_dialer_conn_established_total Total number of connections successfully established by the given dialer a given name. +# TYPE net_conntrack_dialer_conn_established_total counter +net_conntrack_dialer_conn_established_total{dialer_name="alertmanager"} 0 +net_conntrack_dialer_conn_established_total{dialer_name="default"} 0 +net_conntrack_dialer_conn_established_total{dialer_name="prometheus"} 1 +# HELP net_conntrack_listener_conn_accepted_total Total number of connections opened to the listener of a given name. +# TYPE net_conntrack_listener_conn_accepted_total counter +net_conntrack_listener_conn_accepted_total{listener_name="http"} 3 +# HELP net_conntrack_listener_conn_closed_total Total number of connections closed that were made to the listener of a given name. +# TYPE net_conntrack_listener_conn_closed_total counter +net_conntrack_listener_conn_closed_total{listener_name="http"} 0 +# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. +# TYPE process_cpu_seconds_total counter +process_cpu_seconds_total 0.14 +# HELP process_max_fds Maximum number of open file descriptors. +# TYPE process_max_fds gauge +process_max_fds 1.048576e+06 +# HELP process_open_fds Number of open file descriptors. +# TYPE process_open_fds gauge +process_open_fds 13 +# HELP process_resident_memory_bytes Resident memory size in bytes. +# TYPE process_resident_memory_bytes gauge +process_resident_memory_bytes 3.5934208e+07 +# HELP process_start_time_seconds Start time of the process since unix epoch in seconds. +# TYPE process_start_time_seconds gauge +process_start_time_seconds 1.5534303054e+09 +# HELP process_virtual_memory_bytes Virtual memory size in bytes. +# TYPE process_virtual_memory_bytes gauge +process_virtual_memory_bytes 1.50646784e+08 +# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes. +# TYPE process_virtual_memory_max_bytes gauge +process_virtual_memory_max_bytes -1 +# HELP prometheus_api_remote_read_queries The current number of remote read queries being executed or waiting. +# TYPE prometheus_api_remote_read_queries gauge +prometheus_api_remote_read_queries 0 diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain-expected.json b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain-expected.json new file mode 100644 index 000000000000..04c3decdc7fb --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain-expected.json @@ -0,0 +1,444 @@ +[ + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_gc_duration_seconds": { + "value": 0.004392391 + }, + "labels": { + "instance": "127.0.0.1:43979", + "job": "prometheus", + "quantile": "1" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "dialer_name": "alertmanager", + "instance": "127.0.0.1:43979", + "job": "prometheus" + }, + "net_conntrack_dialer_conn_attempted_total": { + "counter": 0, + "rate": 0 + }, + "net_conntrack_dialer_conn_closed_total": { + "counter": 0, + "rate": 0 + }, + "net_conntrack_dialer_conn_established_total": { + "counter": 0, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_gc_duration_seconds": { + "value": 0.000038386 + }, + "labels": { + "instance": "127.0.0.1:43979", + "job": "prometheus", + "quantile": "0" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_info": { + "value": 1 + }, + "labels": { + "instance": "127.0.0.1:43979", + "job": "prometheus", + "version": "go1.11.3" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_gc_duration_seconds": { + "value": 0.000060618 + }, + "labels": { + "instance": "127.0.0.1:43979", + "job": "prometheus", + "quantile": "0.5" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:43979", + "job": "prometheus", + "listener_name": "http" + }, + "net_conntrack_listener_conn_accepted_total": { + "counter": 3, + "rate": 0 + }, + "net_conntrack_listener_conn_closed_total": { + "counter": 0, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_gc_duration_seconds": { + "value": 0.000042803 + }, + "labels": { + "instance": "127.0.0.1:43979", + "job": "prometheus", + "quantile": "0.25" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "dialer_name": "prometheus", + "instance": "127.0.0.1:43979", + "job": "prometheus" + }, + "net_conntrack_dialer_conn_attempted_total": { + "counter": 1, + "rate": 0 + }, + "net_conntrack_dialer_conn_closed_total": { + "counter": 0, + "rate": 0 + }, + "net_conntrack_dialer_conn_established_total": { + "counter": 1, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "dialer_name": "default", + "instance": "127.0.0.1:43979", + "job": "prometheus" + }, + "net_conntrack_dialer_conn_attempted_total": { + "counter": 0, + "rate": 0 + }, + "net_conntrack_dialer_conn_closed_total": { + "counter": 0, + "rate": 0 + }, + "net_conntrack_dialer_conn_established_total": { + "counter": 0, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_gc_duration_seconds_count": { + "counter": 4, + "rate": 0 + }, + "go_gc_duration_seconds_sum": { + "counter": 0.004534198, + "rate": 0 + }, + "go_goroutines": { + "value": 35 + }, + "go_memstats_alloc_bytes": { + "value": 10558112 + }, + "go_memstats_alloc_bytes_total": { + "counter": 14087760, + "rate": 0 + }, + "go_memstats_buck_hash_sys_bytes": { + "value": 1447018 + }, + "go_memstats_frees_total": { + "counter": 15673, + "rate": 0 + }, + "go_memstats_gc_cpu_fraction": { + "value": 0.0008429952574435172 + }, + "go_memstats_gc_sys_bytes": { + "value": 2379776 + }, + "go_memstats_heap_alloc_bytes": { + "value": 10558112 + }, + "go_memstats_heap_idle_bytes": { + "value": 54042624 + }, + "go_memstats_heap_inuse_bytes": { + "value": 12214272 + }, + "go_memstats_heap_objects": { + "value": 61771 + }, + "go_memstats_heap_released_bytes": { + "value": 0 + }, + "go_memstats_heap_sys_bytes": { + "value": 66256896 + }, + "go_memstats_last_gc_time_seconds": { + "value": 1553430316.1488917 + }, + "go_memstats_lookups_total": { + "counter": 0, + "rate": 0 + }, + "go_memstats_mallocs_total": { + "counter": 77444, + "rate": 0 + }, + "go_memstats_mcache_inuse_bytes": { + "value": 6912 + }, + "go_memstats_mcache_sys_bytes": { + "value": 16384 + }, + "go_memstats_mspan_inuse_bytes": { + "value": 127984 + }, + "go_memstats_mspan_sys_bytes": { + "value": 131072 + }, + "go_memstats_next_gc_bytes": { + "value": 18390112 + }, + "go_memstats_other_sys_bytes": { + "value": 1201294 + }, + "go_memstats_stack_inuse_bytes": { + "value": 851968 + }, + "go_memstats_stack_sys_bytes": { + "value": 851968 + }, + "go_memstats_sys_bytes": { + "value": 72284408 + }, + "go_threads": { + "value": 14 + }, + "labels": { + "instance": "127.0.0.1:43979", + "job": "prometheus" + }, + "process_cpu_seconds_total": { + "counter": 0.14, + "rate": 0 + }, + "process_max_fds": { + "value": 1048576 + }, + "process_open_fds": { + "value": 13 + }, + "process_resident_memory_bytes": { + "value": 35934208 + }, + "process_start_time_seconds": { + "value": 1553430305.4 + }, + "process_virtual_memory_bytes": { + "value": 150646784 + }, + "process_virtual_memory_max_bytes": { + "value": -1 + }, + "prometheus_api_remote_read_queries": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:43979", + "job": "prometheus" + }, + "up": { + "value": 1 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_gc_duration_seconds": { + "value": 0.004392391 + }, + "labels": { + "instance": "127.0.0.1:43979", + "job": "prometheus", + "quantile": "0.75" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + } +] \ No newline at end of file diff --git a/x-pack/metricbeat/module/prometheus/collector/collector_test.go b/x-pack/metricbeat/module/prometheus/collector/collector_test.go new file mode 100644 index 000000000000..0a3950bdc75d --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/collector_test.go @@ -0,0 +1,19 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !integration + +package collector + +import ( + "testing" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus" +) + +func TestData(t *testing.T) { + mbtest.TestDataFiles(t, "prometheus", "collector") +} From f4470cf62f2be94f43d381c8d21f8fb03e23fcf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 24 Mar 2020 16:53:31 +0100 Subject: [PATCH 31/32] Fix tests --- .../module/prometheus/collector/collector.go | 2 +- .../prometheus/collector/collector_test.go | 32 +++++++++++++------ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/metricbeat/module/prometheus/collector/collector.go b/metricbeat/module/prometheus/collector/collector.go index 91cf1b7e40dc..690672ff8297 100644 --- a/metricbeat/module/prometheus/collector/collector.go +++ b/metricbeat/module/prometheus/collector/collector.go @@ -167,7 +167,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { } // Accumulate metrics in the event - eventList[labelsHash].Update(promEvent.Data) + eventList[labelsHash].DeepUpdate(promEvent.Data) } } diff --git a/metricbeat/module/prometheus/collector/collector_test.go b/metricbeat/module/prometheus/collector/collector_test.go index 35a231c17f0d..94477a0aa2b7 100644 --- a/metricbeat/module/prometheus/collector/collector_test.go +++ b/metricbeat/module/prometheus/collector/collector_test.go @@ -64,7 +64,9 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { Event: []PromEvent{ { Data: common.MapStr{ - "http_request_duration_microseconds": float64(10), + "metrics": common.MapStr{ + "http_request_duration_microseconds": float64(10), + }, }, Labels: labels, }, @@ -86,7 +88,9 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { Event: []PromEvent{ { Data: common.MapStr{ - "http_request_duration_microseconds": float64(10), + "metrics": common.MapStr{ + "http_request_duration_microseconds": float64(10), + }, }, Labels: common.MapStr{}, }, @@ -115,14 +119,18 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { Event: []PromEvent{ { Data: common.MapStr{ - "http_request_duration_microseconds_count": uint64(10), - "http_request_duration_microseconds_sum": float64(10), + "metrics": common.MapStr{ + "http_request_duration_microseconds_count": uint64(10), + "http_request_duration_microseconds_sum": float64(10), + }, }, Labels: common.MapStr{}, }, { Data: common.MapStr{ - "http_request_duration_microseconds": float64(10), + "metrics": common.MapStr{ + "http_request_duration_microseconds": float64(10), + }, }, Labels: common.MapStr{ "quantile": "0.99", @@ -153,14 +161,18 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { Event: []PromEvent{ { Data: common.MapStr{ - "http_request_duration_microseconds_count": uint64(10), - "http_request_duration_microseconds_sum": float64(10), + "metrics": common.MapStr{ + "http_request_duration_microseconds_count": uint64(10), + "http_request_duration_microseconds_sum": float64(10), + }, }, Labels: common.MapStr{}, }, { Data: common.MapStr{ - "http_request_duration_microseconds_bucket": uint64(10), + "metrics": common.MapStr{ + "http_request_duration_microseconds_bucket": uint64(10), + }, }, Labels: common.MapStr{"le": "0.99"}, }, @@ -188,7 +200,9 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { Event: []PromEvent{ { Data: common.MapStr{ - "http_request_duration_microseconds": float64(10), + "metrics": common.MapStr{ + "http_request_duration_microseconds": float64(10), + }, }, Labels: labels, }, From fff3faf2febfefcd2313ea129c79a60408723a12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 24 Mar 2020 18:31:23 +0100 Subject: [PATCH 32/32] Fix test --- .../collector/_meta/testdata/config.yml | 7 +- .../_meta/testdata/docs.plain-expected.json | 44 +- .../metrics-with-naninf.plain-expected.json | 98 ++--- ...ometheus-2.6.0-partial.plain-expected.json | 380 +++++++++--------- 4 files changed, 267 insertions(+), 262 deletions(-) diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml index 875c2a2d6a4d..e7b2a5e67b4d 100644 --- a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml @@ -1,7 +1,12 @@ type: http url: "/metrics" suffix: plain -omit_documented_fields_check: ["prometheus.labels.*"] # this are not mapped by this module but the oss one +omit_documented_fields_check: + # these are not mapped by this module but the oss one + - prometheus.labels.* + # histogram values & counts are not mapped (it's part of the type data) + - '*.histogram.values' + - '*.histogram.counts' remove_fields_from_comparison: ["prometheus.labels.instance"] module: use_types: true diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json index d978670ecc74..6810afce3f63 100644 --- a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json @@ -11,8 +11,8 @@ }, "prometheus": { "labels": { - "device": "br-0cb306323b90", - "instance": "127.0.0.1:38401", + "device": "br-210476dc4ef8", + "instance": "127.0.0.1:41313", "job": "prometheus" }, "node_network_carrier": { @@ -36,8 +36,8 @@ }, "prometheus": { "labels": { - "device": "br-210476dc4ef8", - "instance": "127.0.0.1:38401", + "device": "br-0cb306323b90", + "instance": "127.0.0.1:41313", "job": "prometheus" }, "node_network_carrier": { @@ -61,11 +61,12 @@ }, "prometheus": { "labels": { - "instance": "127.0.0.1:38401", + "device": "br-10229e3512d9", + "instance": "127.0.0.1:41313", "job": "prometheus" }, - "up": { - "value": 1 + "node_network_carrier": { + "value": 0 } }, "service": { @@ -85,8 +86,8 @@ }, "prometheus": { "labels": { - "device": "br-3a285aa5e58c", - "instance": "127.0.0.1:38401", + "device": "br-33d819d5f834", + "instance": "127.0.0.1:41313", "job": "prometheus" }, "node_network_carrier": { @@ -110,12 +111,11 @@ }, "prometheus": { "labels": { - "device": "br-33d819d5f834", - "instance": "127.0.0.1:38401", + "instance": "127.0.0.1:41313", "job": "prometheus" }, - "node_network_carrier": { - "value": 0 + "up": { + "value": 1 } }, "service": { @@ -135,8 +135,8 @@ }, "prometheus": { "labels": { - "device": "br-4e623477470e", - "instance": "127.0.0.1:38401", + "device": "br-38425a39f36b", + "instance": "127.0.0.1:41313", "job": "prometheus" }, "node_network_carrier": { @@ -160,8 +160,8 @@ }, "prometheus": { "labels": { - "device": "br-10229e3512d9", - "instance": "127.0.0.1:38401", + "device": "br-3a285aa5e58c", + "instance": "127.0.0.1:41313", "job": "prometheus" }, "node_network_carrier": { @@ -186,7 +186,7 @@ "prometheus": { "labels": { "device": "br-38feb0aad6ab", - "instance": "127.0.0.1:38401", + "instance": "127.0.0.1:41313", "job": "prometheus" }, "node_network_carrier": { @@ -210,8 +210,8 @@ }, "prometheus": { "labels": { - "device": "br-425cb4c454a6", - "instance": "127.0.0.1:38401", + "device": "br-4e623477470e", + "instance": "127.0.0.1:41313", "job": "prometheus" }, "node_network_carrier": { @@ -235,8 +235,8 @@ }, "prometheus": { "labels": { - "device": "br-38425a39f36b", - "instance": "127.0.0.1:38401", + "device": "br-425cb4c454a6", + "instance": "127.0.0.1:41313", "job": "prometheus" }, "node_network_carrier": { diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain-expected.json b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain-expected.json index e507e51ba82b..03ec694141a1 100644 --- a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain-expected.json +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain-expected.json @@ -10,35 +10,13 @@ "period": 10000 }, "prometheus": { - "go_gc_duration_seconds_count": { - "counter": 13118, - "rate": 0 - }, - "go_gc_duration_seconds_sum": { - "counter": 3.451780079, - "rate": 0 - }, - "http_request_duration_seconds": { - "histogram": { - "counts": [ - 0, - 0, - 0, - 0, - 0 - ], - "values": [ - 0.5, - 1.5, - 2.5, - 4, - 7 - ] - } + "go_gc_duration_seconds": { + "value": 0.000098154 }, "labels": { - "instance": "127.0.0.1:37675", - "job": "prometheus" + "instance": "127.0.0.1:42061", + "job": "prometheus", + "quantile": "0.75" } }, "service": { @@ -57,12 +35,13 @@ "period": 10000 }, "prometheus": { - "labels": { - "instance": "127.0.0.1:37675", - "job": "prometheus" + "go_gc_duration_seconds": { + "value": 0.011689149 }, - "up": { - "value": 1 + "labels": { + "instance": "127.0.0.1:42061", + "job": "prometheus", + "quantile": "1" } }, "service": { @@ -81,13 +60,12 @@ "period": 10000 }, "prometheus": { - "kafka_consumer_records_lag_records": { - "value": 5 - }, "labels": { - "client_id": "consumer4", - "instance": "127.0.0.1:37675", + "instance": "127.0.0.1:42061", "job": "prometheus" + }, + "up": { + "value": 1 } }, "service": { @@ -106,13 +84,35 @@ "period": 10000 }, "prometheus": { - "go_gc_duration_seconds": { - "value": 0.011689149 + "go_gc_duration_seconds_count": { + "counter": 13118, + "rate": 0 + }, + "go_gc_duration_seconds_sum": { + "counter": 3.451780079, + "rate": 0 + }, + "http_request_duration_seconds": { + "histogram": { + "counts": [ + 0, + 0, + 0, + 0, + 0 + ], + "values": [ + 0.5, + 1.5, + 2.5, + 4, + 7 + ] + } }, "labels": { - "instance": "127.0.0.1:37675", - "job": "prometheus", - "quantile": "1" + "instance": "127.0.0.1:42061", + "job": "prometheus" } }, "service": { @@ -131,13 +131,13 @@ "period": 10000 }, "prometheus": { - "go_gc_duration_seconds": { - "value": 0.000098154 + "kafka_consumer_records_lag_records": { + "value": 5 }, "labels": { - "instance": "127.0.0.1:37675", - "job": "prometheus", - "quantile": "0.75" + "client_id": "consumer4", + "instance": "127.0.0.1:42061", + "job": "prometheus" } }, "service": { @@ -161,7 +161,7 @@ "rate": 0 }, "labels": { - "instance": "127.0.0.1:37675", + "instance": "127.0.0.1:42061", "job": "prometheus", "method": "GET" } @@ -183,7 +183,7 @@ }, "prometheus": { "labels": { - "instance": "127.0.0.1:37675", + "instance": "127.0.0.1:42061", "job": "prometheus", "listener_name": "http" }, diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain-expected.json b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain-expected.json index 04c3decdc7fb..5d6410ea0943 100644 --- a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain-expected.json +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain-expected.json @@ -10,13 +10,22 @@ "period": 10000 }, "prometheus": { - "go_gc_duration_seconds": { - "value": 0.004392391 - }, "labels": { - "instance": "127.0.0.1:43979", - "job": "prometheus", - "quantile": "1" + "dialer_name": "prometheus", + "instance": "127.0.0.1:34181", + "job": "prometheus" + }, + "net_conntrack_dialer_conn_attempted_total": { + "counter": 1, + "rate": 0 + }, + "net_conntrack_dialer_conn_closed_total": { + "counter": 0, + "rate": 0 + }, + "net_conntrack_dialer_conn_established_total": { + "counter": 1, + "rate": 0 } }, "service": { @@ -36,19 +45,15 @@ }, "prometheus": { "labels": { - "dialer_name": "alertmanager", - "instance": "127.0.0.1:43979", - "job": "prometheus" - }, - "net_conntrack_dialer_conn_attempted_total": { - "counter": 0, - "rate": 0 + "instance": "127.0.0.1:34181", + "job": "prometheus", + "listener_name": "http" }, - "net_conntrack_dialer_conn_closed_total": { - "counter": 0, + "net_conntrack_listener_conn_accepted_total": { + "counter": 3, "rate": 0 }, - "net_conntrack_dialer_conn_established_total": { + "net_conntrack_listener_conn_closed_total": { "counter": 0, "rate": 0 } @@ -69,13 +74,124 @@ "period": 10000 }, "prometheus": { - "go_gc_duration_seconds": { - "value": 0.000038386 + "go_gc_duration_seconds_count": { + "counter": 4, + "rate": 0 + }, + "go_gc_duration_seconds_sum": { + "counter": 0.004534198, + "rate": 0 + }, + "go_goroutines": { + "value": 35 + }, + "go_memstats_alloc_bytes": { + "value": 10558112 + }, + "go_memstats_alloc_bytes_total": { + "counter": 14087760, + "rate": 0 + }, + "go_memstats_buck_hash_sys_bytes": { + "value": 1447018 + }, + "go_memstats_frees_total": { + "counter": 15673, + "rate": 0 + }, + "go_memstats_gc_cpu_fraction": { + "value": 0.0008429952574435172 + }, + "go_memstats_gc_sys_bytes": { + "value": 2379776 + }, + "go_memstats_heap_alloc_bytes": { + "value": 10558112 + }, + "go_memstats_heap_idle_bytes": { + "value": 54042624 + }, + "go_memstats_heap_inuse_bytes": { + "value": 12214272 + }, + "go_memstats_heap_objects": { + "value": 61771 + }, + "go_memstats_heap_released_bytes": { + "value": 0 + }, + "go_memstats_heap_sys_bytes": { + "value": 66256896 + }, + "go_memstats_last_gc_time_seconds": { + "value": 1553430316.1488917 + }, + "go_memstats_lookups_total": { + "counter": 0, + "rate": 0 + }, + "go_memstats_mallocs_total": { + "counter": 77444, + "rate": 0 + }, + "go_memstats_mcache_inuse_bytes": { + "value": 6912 + }, + "go_memstats_mcache_sys_bytes": { + "value": 16384 + }, + "go_memstats_mspan_inuse_bytes": { + "value": 127984 + }, + "go_memstats_mspan_sys_bytes": { + "value": 131072 + }, + "go_memstats_next_gc_bytes": { + "value": 18390112 + }, + "go_memstats_other_sys_bytes": { + "value": 1201294 + }, + "go_memstats_stack_inuse_bytes": { + "value": 851968 + }, + "go_memstats_stack_sys_bytes": { + "value": 851968 + }, + "go_memstats_sys_bytes": { + "value": 72284408 + }, + "go_threads": { + "value": 14 }, "labels": { - "instance": "127.0.0.1:43979", - "job": "prometheus", - "quantile": "0" + "instance": "127.0.0.1:34181", + "job": "prometheus" + }, + "process_cpu_seconds_total": { + "counter": 0.14, + "rate": 0 + }, + "process_max_fds": { + "value": 1048576 + }, + "process_open_fds": { + "value": 13 + }, + "process_resident_memory_bytes": { + "value": 35934208 + }, + "process_start_time_seconds": { + "value": 1553430305.4 + }, + "process_virtual_memory_bytes": { + "value": 150646784 + }, + "process_virtual_memory_max_bytes": { + "value": -1 + }, + "prometheus_api_remote_read_queries": { + "value": 0 } }, "service": { @@ -94,13 +210,22 @@ "period": 10000 }, "prometheus": { - "go_info": { - "value": 1 - }, "labels": { - "instance": "127.0.0.1:43979", - "job": "prometheus", - "version": "go1.11.3" + "dialer_name": "default", + "instance": "127.0.0.1:34181", + "job": "prometheus" + }, + "net_conntrack_dialer_conn_attempted_total": { + "counter": 0, + "rate": 0 + }, + "net_conntrack_dialer_conn_closed_total": { + "counter": 0, + "rate": 0 + }, + "net_conntrack_dialer_conn_established_total": { + "counter": 0, + "rate": 0 } }, "service": { @@ -119,13 +244,13 @@ "period": 10000 }, "prometheus": { - "go_gc_duration_seconds": { - "value": 0.000060618 + "go_info": { + "value": 1 }, "labels": { - "instance": "127.0.0.1:43979", + "instance": "127.0.0.1:34181", "job": "prometheus", - "quantile": "0.5" + "version": "go1.11.3" } }, "service": { @@ -145,15 +270,19 @@ }, "prometheus": { "labels": { - "instance": "127.0.0.1:43979", - "job": "prometheus", - "listener_name": "http" + "dialer_name": "alertmanager", + "instance": "127.0.0.1:34181", + "job": "prometheus" }, - "net_conntrack_listener_conn_accepted_total": { - "counter": 3, + "net_conntrack_dialer_conn_attempted_total": { + "counter": 0, "rate": 0 }, - "net_conntrack_listener_conn_closed_total": { + "net_conntrack_dialer_conn_closed_total": { + "counter": 0, + "rate": 0 + }, + "net_conntrack_dialer_conn_established_total": { "counter": 0, "rate": 0 } @@ -175,12 +304,12 @@ }, "prometheus": { "go_gc_duration_seconds": { - "value": 0.000042803 + "value": 0.004392391 }, "labels": { - "instance": "127.0.0.1:43979", + "instance": "127.0.0.1:34181", "job": "prometheus", - "quantile": "0.25" + "quantile": "1" } }, "service": { @@ -199,22 +328,13 @@ "period": 10000 }, "prometheus": { - "labels": { - "dialer_name": "prometheus", - "instance": "127.0.0.1:43979", - "job": "prometheus" - }, - "net_conntrack_dialer_conn_attempted_total": { - "counter": 1, - "rate": 0 - }, - "net_conntrack_dialer_conn_closed_total": { - "counter": 0, - "rate": 0 + "go_gc_duration_seconds": { + "value": 0.000038386 }, - "net_conntrack_dialer_conn_established_total": { - "counter": 1, - "rate": 0 + "labels": { + "instance": "127.0.0.1:34181", + "job": "prometheus", + "quantile": "0" } }, "service": { @@ -234,21 +354,11 @@ }, "prometheus": { "labels": { - "dialer_name": "default", - "instance": "127.0.0.1:43979", + "instance": "127.0.0.1:34181", "job": "prometheus" }, - "net_conntrack_dialer_conn_attempted_total": { - "counter": 0, - "rate": 0 - }, - "net_conntrack_dialer_conn_closed_total": { - "counter": 0, - "rate": 0 - }, - "net_conntrack_dialer_conn_established_total": { - "counter": 0, - "rate": 0 + "up": { + "value": 1 } }, "service": { @@ -267,124 +377,13 @@ "period": 10000 }, "prometheus": { - "go_gc_duration_seconds_count": { - "counter": 4, - "rate": 0 - }, - "go_gc_duration_seconds_sum": { - "counter": 0.004534198, - "rate": 0 - }, - "go_goroutines": { - "value": 35 - }, - "go_memstats_alloc_bytes": { - "value": 10558112 - }, - "go_memstats_alloc_bytes_total": { - "counter": 14087760, - "rate": 0 - }, - "go_memstats_buck_hash_sys_bytes": { - "value": 1447018 - }, - "go_memstats_frees_total": { - "counter": 15673, - "rate": 0 - }, - "go_memstats_gc_cpu_fraction": { - "value": 0.0008429952574435172 - }, - "go_memstats_gc_sys_bytes": { - "value": 2379776 - }, - "go_memstats_heap_alloc_bytes": { - "value": 10558112 - }, - "go_memstats_heap_idle_bytes": { - "value": 54042624 - }, - "go_memstats_heap_inuse_bytes": { - "value": 12214272 - }, - "go_memstats_heap_objects": { - "value": 61771 - }, - "go_memstats_heap_released_bytes": { - "value": 0 - }, - "go_memstats_heap_sys_bytes": { - "value": 66256896 - }, - "go_memstats_last_gc_time_seconds": { - "value": 1553430316.1488917 - }, - "go_memstats_lookups_total": { - "counter": 0, - "rate": 0 - }, - "go_memstats_mallocs_total": { - "counter": 77444, - "rate": 0 - }, - "go_memstats_mcache_inuse_bytes": { - "value": 6912 - }, - "go_memstats_mcache_sys_bytes": { - "value": 16384 - }, - "go_memstats_mspan_inuse_bytes": { - "value": 127984 - }, - "go_memstats_mspan_sys_bytes": { - "value": 131072 - }, - "go_memstats_next_gc_bytes": { - "value": 18390112 - }, - "go_memstats_other_sys_bytes": { - "value": 1201294 - }, - "go_memstats_stack_inuse_bytes": { - "value": 851968 - }, - "go_memstats_stack_sys_bytes": { - "value": 851968 - }, - "go_memstats_sys_bytes": { - "value": 72284408 - }, - "go_threads": { - "value": 14 + "go_gc_duration_seconds": { + "value": 0.000060618 }, "labels": { - "instance": "127.0.0.1:43979", - "job": "prometheus" - }, - "process_cpu_seconds_total": { - "counter": 0.14, - "rate": 0 - }, - "process_max_fds": { - "value": 1048576 - }, - "process_open_fds": { - "value": 13 - }, - "process_resident_memory_bytes": { - "value": 35934208 - }, - "process_start_time_seconds": { - "value": 1553430305.4 - }, - "process_virtual_memory_bytes": { - "value": 150646784 - }, - "process_virtual_memory_max_bytes": { - "value": -1 - }, - "prometheus_api_remote_read_queries": { - "value": 0 + "instance": "127.0.0.1:34181", + "job": "prometheus", + "quantile": "0.5" } }, "service": { @@ -403,12 +402,13 @@ "period": 10000 }, "prometheus": { - "labels": { - "instance": "127.0.0.1:43979", - "job": "prometheus" + "go_gc_duration_seconds": { + "value": 0.000042803 }, - "up": { - "value": 1 + "labels": { + "instance": "127.0.0.1:34181", + "job": "prometheus", + "quantile": "0.25" } }, "service": { @@ -431,7 +431,7 @@ "value": 0.004392391 }, "labels": { - "instance": "127.0.0.1:43979", + "instance": "127.0.0.1:34181", "job": "prometheus", "quantile": "0.75" }