From bbed3ae55602e83f57c62de85b57a3593aa49efa Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 10:58:42 -0600 Subject: [PATCH] [8.15](backport #41022) [AWS] Use namespace for GetListMetrics when exists (#41175) * [AWS] Use namespace for GetListMetrics when exists (#41022) * Use namespace for GetListMetrics when exists (cherry picked from commit 36327a4b2841f20e4685c308469df356f04415a6) --------- Co-authored-by: kaiyan-sheng --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/modules/aws.asciidoc | 3 ++- .../metricbeat/module/aws/_meta/docs.asciidoc | 3 ++- .../module/aws/cloudwatch/cloudwatch.go | 20 +++++++++++++++---- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3b9774c3c3b4..8e9dd4d0a3af 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -124,6 +124,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix Azure Monitor 429 error by causing metricbeat to retry the request again. {pull}38294[38294] - Fix fields not being parsed correctly in postgresql/database {issue}25301[25301] {pull}37720[37720] - rabbitmq/queue - Change the mapping type of `rabbitmq.queue.consumers.utilisation.pct` to `scaled_float` from `long` because the values fall within the range of `[0.0, 1.0]`. Previously, conversion to integer resulted in reporting either `0` or `1`. +- Use namespace for GetListMetrics when exists in AWS {pull}41022[41022] *Osquerybeat* diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 20b6854a834e..0ee7f601052f 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -329,7 +329,8 @@ GetMetricData max page size: 100, based on https://docs.aws.amazon.com/AmazonClo | IAM ListAccountAliases | 1 | Once on startup | STS GetCallerIdentity | 1 | Once on startup | EC2 DescribeRegions| 1 | Once on startup -| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per collection period +| CloudWatch ListMetrics without specifying namespace in configuration | Total number of results / ListMetrics max page size | Per region per collection period +| CloudWatch ListMetrics with specific namespaces in configuration | Total number of results / ListMetrics max page size * number of unique namespaces | Per region per collection period | CloudWatch GetMetricData | Total number of results / GetMetricData max page size | Per region per namespace per collection period |=== `billing`, `ebs`, `elb`, `sns`, `usage` and `lambda` are the same as `cloudwatch` metricset. diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index 223f90e0e24b..0b224bf5630c 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -317,7 +317,8 @@ GetMetricData max page size: 100, based on https://docs.aws.amazon.com/AmazonClo | IAM ListAccountAliases | 1 | Once on startup | STS GetCallerIdentity | 1 | Once on startup | EC2 DescribeRegions| 1 | Once on startup -| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per collection period +| CloudWatch ListMetrics without specifying namespace in configuration | Total number of results / ListMetrics max page size | Per region per collection period +| CloudWatch ListMetrics with specific namespaces in configuration | Total number of results / ListMetrics max page size * number of unique namespaces | Per region per collection period | CloudWatch GetMetricData | Total number of results / GetMetricData max page size | Per region per namespace per collection period |=== `billing`, `ebs`, `elb`, `sns`, `usage` and `lambda` are the same as `cloudwatch` metricset. diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index dde2463ea852..ed043e8c38f1 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -179,10 +179,22 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { continue } - // retrieve all the details for all the metrics available in the current region - listMetricsOutput, err := aws.GetListMetricsOutput("*", regionName, m.Period, m.IncludeLinkedAccounts, m.OwningAccount, m.MonitoringAccountID, svcCloudwatch) - if err != nil { - m.Logger().Errorf("Error while retrieving the list of metrics for region %s: %w", regionName, err) + // retrieve all the details for all the metrics available in the current region when no namespace is specified + // otherwise only retrieve metrics from the specific namespaces from the config + var listMetricsOutput []aws.MetricWithID + if len(namespaceDetailTotal) == 0 { + listMetricsOutput, err = aws.GetListMetricsOutput("*", regionName, m.Period, m.IncludeLinkedAccounts, m.OwningAccount, m.MonitoringAccountID, svcCloudwatch) + if err != nil { + m.Logger().Errorf("Error while retrieving the list of metrics for region %s and namespace %s: %w", regionName, "*", err) + } + } else { + for namespace := range namespaceDetailTotal { + listMetricsOutputPerNamespace, err := aws.GetListMetricsOutput(namespace, regionName, m.Period, m.IncludeLinkedAccounts, m.OwningAccount, m.MonitoringAccountID, svcCloudwatch) + if err != nil { + m.Logger().Errorf("Error while retrieving the list of metrics for region %s and namespace %s: %w", regionName, namespace, err) + } + listMetricsOutput = append(listMetricsOutput, listMetricsOutputPerNamespace...) + } } if len(listMetricsOutput) == 0 {