From 010a393d392bbb9be4203e1635b3813aa679b723 Mon Sep 17 00:00:00 2001 From: stefans-elastic Date: Tue, 3 Dec 2024 20:28:42 +0200 Subject: [PATCH] [xpack][metricbeats][gcp] don't skip first value in histogram calculations (#41822) * [xpack][metricbeats][gcp] don't skip first value in histogram calculations * update changelog * update PR id in changelog * update data_topic.json * update data_topic.json * fix integer overflow --------- Co-authored-by: Ishleen Kaur <102962586+ishleenk17@users.noreply.github.com> (cherry picked from commit a1acad54fa99ee884cb3576ea74f494dd1a63da6) --- CHANGELOG.next.asciidoc | 1 + x-pack/metricbeat/module/gcp/distribution.go | 14 +-- .../module/gcp/distribution_test.go | 12 +- .../module/gcp/pubsub/_meta/data_topic.json | 111 ++++++++++++++++-- 4 files changed, 112 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 461e1c26db42..b9b3881e7a9a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -212,6 +212,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix Kubernetes metadata sometimes not being present after startup {pull}41216[41216] - Do not report non-existant 0 values for RSS metrics in docker/memory {pull}41449[41449] - Log Cisco Meraki `getDevicePerformanceScores` errors without stopping metrics collection. {pull}41622[41622] +- Don't skip first bucket value in GCP metrics metricset for distribution type metrics {pull}41822[41822] *Osquerybeat* diff --git a/x-pack/metricbeat/module/gcp/distribution.go b/x-pack/metricbeat/module/gcp/distribution.go index aae21e1d58e5..f77b3afc7112 100644 --- a/x-pack/metricbeat/module/gcp/distribution.go +++ b/x-pack/metricbeat/module/gcp/distribution.go @@ -53,7 +53,7 @@ func calcLinearUpperBound(bucket *distribution.Distribution_BucketOptions_Linear return bucket.Offset + (bucket.Width * float64(i)) } -func createHistogram(values []float64, counts []uint64) mapstr.M { +func createHistogram(values []float64, counts []int64) mapstr.M { return mapstr.M{ "values": values, "counts": counts, @@ -62,11 +62,11 @@ func createHistogram(values []float64, counts []uint64) mapstr.M { func DistributionHistogramToES(d *distribution.Distribution) mapstr.M { if !containsHistogram(d) { - return createHistogram([]float64{}, []uint64{}) + return createHistogram([]float64{}, []int64{}) } values := make([]float64, 0, len(d.BucketCounts)) - counts := make([]uint64, 0, len(d.BucketCounts)) + counts := make([]int64, 0, len(d.BucketCounts)) switch { case d.BucketOptions.GetExplicitBuckets() != nil: @@ -79,19 +79,17 @@ func DistributionHistogramToES(d *distribution.Distribution) mapstr.M { bucket := d.BucketOptions.GetExponentialBuckets() for i := range d.BucketCounts { - values = append(values, calcExponentialUpperBound(bucket, i+1)) + values = append(values, calcExponentialUpperBound(bucket, i)) } case d.BucketOptions.GetLinearBuckets() != nil: bucket := d.BucketOptions.GetLinearBuckets() for i := range d.BucketCounts { - values = append(values, calcLinearUpperBound(bucket, i+1)) + values = append(values, calcLinearUpperBound(bucket, i)) } } - for i := range d.BucketCounts { - counts = append(counts, uint64(d.BucketCounts[i])) - } + counts = append(counts, d.BucketCounts...) return createHistogram(values, counts) } diff --git a/x-pack/metricbeat/module/gcp/distribution_test.go b/x-pack/metricbeat/module/gcp/distribution_test.go index e2f582653beb..3838a9e9dd3a 100644 --- a/x-pack/metricbeat/module/gcp/distribution_test.go +++ b/x-pack/metricbeat/module/gcp/distribution_test.go @@ -40,7 +40,7 @@ func TestDistributionHistogramToES(t *testing.T) { }, }, expected: mapstr.M{ - "counts": []uint64{0, 0, 0, 6, 1, 1}, + "counts": []int64{0, 0, 0, 6, 1, 1}, "values": []float64{0, 1, 2, 5, 10, 20}, }, }, @@ -63,8 +63,8 @@ func TestDistributionHistogramToES(t *testing.T) { }, }, expected: mapstr.M{ - "counts": []uint64{0, 0, 3, 1}, - "values": []float64{6, 12, 24, 48}, + "counts": []int64{0, 0, 3, 1}, + "values": []float64{3, 6, 12, 24}, }, }, }, @@ -86,8 +86,8 @@ func TestDistributionHistogramToES(t *testing.T) { }, }, expected: mapstr.M{ - "counts": []uint64{0, 1, 2, 0}, - "values": []float64{20, 35, 50, 65}, + "counts": []int64{0, 1, 2, 0}, + "values": []float64{5, 20, 35, 50}, }, }, }, @@ -107,7 +107,7 @@ func TestDistributionHistogramToES(t *testing.T) { }, }, expected: mapstr.M{ - "counts": []uint64{}, + "counts": []int64{}, "values": []float64{}, }, }, diff --git a/x-pack/metricbeat/module/gcp/pubsub/_meta/data_topic.json b/x-pack/metricbeat/module/gcp/pubsub/_meta/data_topic.json index 1453f9f379c7..cb6585606dfd 100644 --- a/x-pack/metricbeat/module/gcp/pubsub/_meta/data_topic.json +++ b/x-pack/metricbeat/module/gcp/pubsub/_meta/data_topic.json @@ -12,25 +12,112 @@ "module": "gcp" }, "gcp": { - "labels": { - "resource": { - "topic_id": "test-ks" - } - }, "pubsub": { "topic": { "message_sizes": { - "bucket_options": { - "Options": { - "ExponentialBuckets": { - "num_finite_buckets": 16, - "growth_factor": 4, - "scale": 1 - } + "bytes": { + "histogram": { + "values": [ + 1, + 1.2, + 1.44, + 1.728, + 2.0736, + 2.48832, + 2.9859839999999997, + 3.5831807999999996, + 4.299816959999999, + 5.159780351999999, + 6.191736422399999, + 7.430083706879999, + 8.916100448255998, + 10.699320537907198, + 12.839184645488636, + 15.407021574586365, + 18.488425889503635, + 22.18611106740436, + 26.623333280885234, + 31.94799993706228, + 38.33759992447474, + 46.00511990936968, + 55.20614389124361, + 66.24737266949234, + 79.4968472033908, + 95.39621664406897, + 114.47545997288276, + 137.3705519674593, + 164.84466236095116, + 197.8135948331414, + 237.37631379976966, + 284.8515765597236, + 341.82189187166824, + 410.18627024600187, + 492.22352429520225, + 590.6682291542427, + 708.8018749850912, + 850.5622499821095, + 1020.6746999785313, + 1224.8096399742376, + 1469.771567969085, + 1763.725881562902, + 2116.4710578754825, + 2539.7652694505787 + ], + "counts": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 10, + 3, + 2 + ] } } } } + }, + "labels": { + "resource": { + "topic_id": "test-topic" + } } }, "metricset": {