From dc152bfdcbd1c6a0ea6d228af888ade10f9b5a27 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 1 Apr 2020 11:08:51 +0300 Subject: [PATCH 1/5] Move dataType field in prometheus labels Signed-off-by: chrismark --- .../module/prometheus/query/_meta/data.json | 4 +-- metricbeat/module/prometheus/query/data.go | 28 ++++++++++++------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/metricbeat/module/prometheus/query/_meta/data.json b/metricbeat/module/prometheus/query/_meta/data.json index 6f03bd4ce7ec..4e06ccf81477 100644 --- a/metricbeat/module/prometheus/query/_meta/data.json +++ b/metricbeat/module/prometheus/query/_meta/data.json @@ -12,12 +12,12 @@ "prometheus": { "labels": { "__name__": "go_threads", + "dataType": "vector", "instance": "localhost:9090", "job": "prometheus" }, "query": { - "dataType": "vector", - "go_threads": 26 + "go_threads": 18 } }, "service": { diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index b89df6b43058..f63332a2e388 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -149,12 +149,12 @@ func getEventsFromMatrix(body []byte, queryName string) ([]mb.Event, error) { if math.IsNaN(val) || math.IsInf(val, 0) { continue } + result.Metric["dataType"] = resultType events = append(events, mb.Event{ Timestamp: getTimestamp(timestamp), ModuleFields: common.MapStr{"labels": result.Metric}, MetricSetFields: common.MapStr{ - "dataType": resultType, - queryName: val, + queryName: val, }, }) } else { @@ -186,12 +186,12 @@ func getEventsFromVector(body []byte, queryName string) ([]mb.Event, error) { if math.IsNaN(val) || math.IsInf(val, 0) { continue } + result.Metric["dataType"] = resultType events = append(events, mb.Event{ Timestamp: getTimestamp(timestamp), ModuleFields: common.MapStr{"labels": result.Metric}, MetricSetFields: common.MapStr{ - "dataType": resultType, - queryName: val, + queryName: val, }, }) } else { @@ -221,9 +221,13 @@ func getEventFromScalarOrString(body []byte, resultType string, queryName string } return mb.Event{ Timestamp: getTimestamp(timestamp), + ModuleFields: common.MapStr{ + "labels": common.MapStr{ + "dataType": resultType, + }, + }, MetricSetFields: common.MapStr{ - "dataType": resultType, - queryName: val, + queryName: val, }, }, nil } else if resultType == "string" { @@ -233,11 +237,15 @@ func getEventFromScalarOrString(body []byte, resultType string, queryName string return mb.Event{}, errors.New(msg) } return mb.Event{ - Timestamp: getTimestamp(timestamp), - ModuleFields: common.MapStr{"labels": common.MapStr{queryName: value}}, + Timestamp: getTimestamp(timestamp), + ModuleFields: common.MapStr{ + "labels": common.MapStr{ + queryName: value, + "dataType": resultType, + }, + }, MetricSetFields: common.MapStr{ - "dataType": resultType, - queryName: 1, + queryName: 1, }, }, nil } From a1ca271f6212e58e4d3c23ebde031ee3ac4e9e06 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 1 Apr 2020 18:36:14 +0300 Subject: [PATCH 2/5] Remove dataType field completely Signed-off-by: chrismark --- metricbeat/module/prometheus/query/_meta/data.json | 3 +-- metricbeat/module/prometheus/query/data.go | 12 +----------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/metricbeat/module/prometheus/query/_meta/data.json b/metricbeat/module/prometheus/query/_meta/data.json index 4e06ccf81477..5f1d69b909a2 100644 --- a/metricbeat/module/prometheus/query/_meta/data.json +++ b/metricbeat/module/prometheus/query/_meta/data.json @@ -12,7 +12,6 @@ "prometheus": { "labels": { "__name__": "go_threads", - "dataType": "vector", "instance": "localhost:9090", "job": "prometheus" }, @@ -21,7 +20,7 @@ } }, "service": { - "address": "localhost:32768", + "address": "localhost:32769", "type": "prometheus" } } \ No newline at end of file diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index f63332a2e388..bc3528856e99 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -129,7 +129,6 @@ func parseResponse(body []byte, pathConfig QueryConfig) ([]mb.Event, error) { func getEventsFromMatrix(body []byte, queryName string) ([]mb.Event, error) { events := []mb.Event{} - resultType := "matrix" convertedMap, err := convertJSONToRangeVectorResponse(body) if err != nil { return events, err @@ -149,7 +148,6 @@ func getEventsFromMatrix(body []byte, queryName string) ([]mb.Event, error) { if math.IsNaN(val) || math.IsInf(val, 0) { continue } - result.Metric["dataType"] = resultType events = append(events, mb.Event{ Timestamp: getTimestamp(timestamp), ModuleFields: common.MapStr{"labels": result.Metric}, @@ -167,7 +165,6 @@ func getEventsFromMatrix(body []byte, queryName string) ([]mb.Event, error) { func getEventsFromVector(body []byte, queryName string) ([]mb.Event, error) { events := []mb.Event{} - resultType := "vector" convertedMap, err := convertJSONToInstantVectorResponse(body) if err != nil { return events, err @@ -186,7 +183,6 @@ func getEventsFromVector(body []byte, queryName string) ([]mb.Event, error) { if math.IsNaN(val) || math.IsInf(val, 0) { continue } - result.Metric["dataType"] = resultType events = append(events, mb.Event{ Timestamp: getTimestamp(timestamp), ModuleFields: common.MapStr{"labels": result.Metric}, @@ -221,11 +217,6 @@ func getEventFromScalarOrString(body []byte, resultType string, queryName string } return mb.Event{ Timestamp: getTimestamp(timestamp), - ModuleFields: common.MapStr{ - "labels": common.MapStr{ - "dataType": resultType, - }, - }, MetricSetFields: common.MapStr{ queryName: val, }, @@ -240,8 +231,7 @@ func getEventFromScalarOrString(body []byte, resultType string, queryName string Timestamp: getTimestamp(timestamp), ModuleFields: common.MapStr{ "labels": common.MapStr{ - queryName: value, - "dataType": resultType, + queryName: value, }, }, MetricSetFields: common.MapStr{ From 6d9efd7b5dc76f26a66be78ea1e6b3b2c9b9ff8d Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Wed, 1 Apr 2020 21:02:13 +0300 Subject: [PATCH 3/5] Remove resultType from structs --- metricbeat/module/prometheus/query/data.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index bc3528856e99..e47f2b092aa8 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -63,7 +63,6 @@ type InstantVectorResponse struct { Data instantVectorData `json:"data"` } type instantVectorData struct { - ResultType string `json:"resultType"` Results []instantVectorResult `json:"result"` } type instantVectorResult struct { @@ -85,7 +84,6 @@ type RangeVectorResponse struct { Data rangeVectorData `json:"data"` } type rangeVectorData struct { - ResultType string `json:"resultType"` Results []rangeVectorResult `json:"result"` } type rangeVectorResult struct { From be64adf75b3ed47d31e92c7fd4bc1c96e695f38a Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Wed, 1 Apr 2020 21:07:05 +0300 Subject: [PATCH 4/5] Remove ResultType from leftover struct --- metricbeat/module/prometheus/query/data.go | 1 - 1 file changed, 1 deletion(-) diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index e47f2b092aa8..1b7fbb3bc47a 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -45,7 +45,6 @@ type ArrayResponse struct { Data arrayData `json:"data"` } type arrayData struct { - ResultType string `json:"resultType"` Results []interface{} `json:"result"` } From 19118e2fe89385af1313ff08388171329c2488cc Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 1 Apr 2020 23:39:30 +0300 Subject: [PATCH 5/5] fmt Signed-off-by: chrismark --- metricbeat/module/prometheus/query/data.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index 1b7fbb3bc47a..b0d3d4d1bb01 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -45,7 +45,7 @@ type ArrayResponse struct { Data arrayData `json:"data"` } type arrayData struct { - Results []interface{} `json:"result"` + Results []interface{} `json:"result"` } // InstantVectorResponse is for "vector" type from Prometheus Query API Request @@ -62,7 +62,7 @@ type InstantVectorResponse struct { Data instantVectorData `json:"data"` } type instantVectorData struct { - Results []instantVectorResult `json:"result"` + Results []instantVectorResult `json:"result"` } type instantVectorResult struct { Metric map[string]string `json:"metric"` @@ -83,7 +83,7 @@ type RangeVectorResponse struct { Data rangeVectorData `json:"data"` } type rangeVectorData struct { - Results []rangeVectorResult `json:"result"` + Results []rangeVectorResult `json:"result"` } type rangeVectorResult struct { Metric map[string]string `json:"metric"`