diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a6e57edb1fe6..7df6ec6b8841 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -535,6 +535,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for v1 consumer API in Cloud Foundry module, use it by default. {pull}19268[19268] - Add support for named ports in autodiscover. {pull}19398[19398] - Add param `aws_partition` to support aws-cn, aws-us-gov regions. {issue}18850[18850] {pull}19423[19423] +- Add support for wildcard `*` in dimension value of AWS CloudWatch metrics config. {issue}18050[18050] {pull}19660[19660] - The `elasticsearch/index` metricset now collects metrics for hidden indices as well. {issue}18639[18639] {pull}18703[18703] *Packetbeat* diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc index 38d393570fc8..faa47f248935 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc @@ -20,7 +20,8 @@ iam:ListAccountAliases For example, AWS/EC2, AWS/S3. If wildcard * is given for namespace, metrics from all namespaces will be collected automatically. * *name*: The name of the metric to filter against. For example, CPUUtilization for EC2 instance. -* *dimensions*: The dimensions to filter against. For example, InstanceId=i-123. +* *dimensions*: The dimensions to filter against. For example, InstanceId=i-123. Dimension value +could be wildcard `*` to match any value. * *tags.resource_type_filter*: The constraints on the resources that you want returned. The format of each resource type is service[:resourceType]. For example, specifying a resource type of ec2 returns all Amazon EC2 resources @@ -158,3 +159,26 @@ metric(average) from EC2 instance i-456. value: i-456 statistic: ["Average"] ---- + + +With the configuration below, user can filter out only `LoadBalacer` and `TargetGroup` dimension +metircs with the metric name `UnHealthyHostCount`, `LoadBalacer` and `TargetGroup` value could +be any. + +[source,yaml] +---- +- module: aws + period: 300s + metricsets: + - cloudwatch + metrics: + - namespace: AWS/ApplicationELB + statistic: ['Maximum'] + name: ['UnHealthyHostCount'] + dimensions: + - name: LoadBalancer + value: "*" + - name: TargetGroup + value: "*" + tags.resource_type_filter: elasticloadbalancing +---- diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index aac756190f2b..57aa99913b81 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -6,7 +6,6 @@ package cloudwatch import ( "reflect" - "sort" "strconv" "strings" "time" @@ -26,15 +25,16 @@ import ( ) var ( - metricsetName = "cloudwatch" - metricNameIdx = 0 - namespaceIdx = 1 - statisticIdx = 2 - identifierNameIdx = 3 - identifierValueIdx = 4 - defaultStatistics = []string{"Average", "Maximum", "Minimum", "Sum", "SampleCount"} - labelSeparator = "|" - dimensionSeparator = "," + metricsetName = "cloudwatch" + metricNameIdx = 0 + namespaceIdx = 1 + statisticIdx = 2 + identifierNameIdx = 3 + identifierValueIdx = 4 + defaultStatistics = []string{"Average", "Maximum", "Minimum", "Sum", "SampleCount"} + labelSeparator = "|" + dimensionSeparator = "," + dimensionValueWildcard = "*" ) // init registers the MetricSet with the central registry as soon as the program @@ -252,8 +252,21 @@ func filterListMetricsOutput(listMetricsOutput []cloudwatch.Metric, namespaceDet statistic: configPerNamespace.statistics, tags: configPerNamespace.tags, }) + } else if configPerNamespace.names != nil && configPerNamespace.dimensions != nil { + if exists, _ := aws.StringInSlice(*listMetric.MetricName, configPerNamespace.names); !exists { + continue + } + if !compareAWSDimensions(listMetric.Dimensions, configPerNamespace.dimensions) { + continue + } + filteredMetricWithStatsTotal = append(filteredMetricWithStatsTotal, + metricsWithStatistics{ + cloudwatchMetric: listMetric, + statistic: configPerNamespace.statistics, + tags: configPerNamespace.tags, + }) } else { - // if no metric name or dimensions given, then keep all listMetricsOutput + // if no metric name and no dimensions given, then keep all listMetricsOutput filteredMetricWithStatsTotal = append(filteredMetricWithStatsTotal, metricsWithStatistics{ cloudwatchMetric: listMetric, @@ -320,8 +333,10 @@ func (m *MetricSet) readCloudwatchConfig() (listMetricWithDetail, map[string][]n Value: &value, }) } - - if config.MetricName != nil && config.Dimensions != nil { + // if any Dimension value contains wildcard, then compare dimensions with + // listMetrics result in filterListMetricsOutput + if config.MetricName != nil && config.Dimensions != nil && + !configDimensionValueContainsWildcard(config.Dimensions) { namespace := config.Namespace for i := range config.MetricName { metricsWithStats := metricsWithStatistics{ @@ -589,22 +604,37 @@ func reportEvents(eventsWithIdentifier map[string]mb.Event, report mb.ReporterV2 return nil } +func configDimensionValueContainsWildcard(dim []Dimension) bool { + for i := range dim { + if dim[i].Value == dimensionValueWildcard { + return true + } + } + return false +} + func compareAWSDimensions(dim1 []cloudwatch.Dimension, dim2 []cloudwatch.Dimension) bool { if len(dim1) != len(dim2) { return false } - var dim1String []string - var dim2String []string - for i := range dim1 { - dim1String = append(dim1String, dim1[i].String()) - } + + var dim1NameToValue = make(map[string]string, len(dim1)) + var dim2NameToValue = make(map[string]string, len(dim1)) + for i := range dim2 { - dim2String = append(dim2String, dim2[i].String()) + dim1NameToValue[*dim1[i].Name] = *dim1[i].Value + dim2NameToValue[*dim2[i].Name] = *dim2[i].Value } - - sort.Strings(dim1String) - sort.Strings(dim2String) - return reflect.DeepEqual(dim1String, dim2String) + for name, v1 := range dim1NameToValue { + v2, exists := dim2NameToValue[name] + if exists && v2 == dimensionValueWildcard { + // wildcard can represent any value, so we set the + // dimension name with value in CloudWatch ListMetircs result, + // then the compare result is true + dim2NameToValue[name] = v1 + } + } + return reflect.DeepEqual(dim1NameToValue, dim2NameToValue) } func insertTags(events map[string]mb.Event, identifier string, resourceTagMap map[string][]resourcegroupstaggingapi.Tag) { diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 93a0c42492cc..a141bd342ed0 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -834,6 +834,60 @@ func TestCompareAWSDimensions(t *testing.T) { []cloudwatch.Dimension{}, false, }, + { + "compare with wildcard dimension value, one same name dimension", + []cloudwatch.Dimension{ + {Name: awssdk.String("ID1"), Value: awssdk.String("111")}, + }, + []cloudwatch.Dimension{ + {Name: awssdk.String("ID1"), Value: awssdk.String(dimensionValueWildcard)}, + }, + true, + }, + { + "compare with wildcard dimension value, one different name dimension", + []cloudwatch.Dimension{ + {Name: awssdk.String("IDx"), Value: awssdk.String("111")}, + }, + []cloudwatch.Dimension{ + {Name: awssdk.String("ID1"), Value: awssdk.String(dimensionValueWildcard)}, + }, + false, + }, + { + "compare with wildcard dimension value, two same name dimensions", + []cloudwatch.Dimension{ + {Name: awssdk.String("ID1"), Value: awssdk.String("111")}, + {Name: awssdk.String("ID2"), Value: awssdk.String("222")}, + }, + []cloudwatch.Dimension{ + {Name: awssdk.String("ID1"), Value: awssdk.String("111")}, + {Name: awssdk.String("ID2"), Value: awssdk.String(dimensionValueWildcard)}, + }, + true, + }, + { + "compare with wildcard dimension value, different length, case1", + []cloudwatch.Dimension{ + {Name: awssdk.String("ID1"), Value: awssdk.String("111")}, + {Name: awssdk.String("ID2"), Value: awssdk.String("222")}, + }, + []cloudwatch.Dimension{ + {Name: awssdk.String("ID2"), Value: awssdk.String(dimensionValueWildcard)}, + }, + false, + }, + { + "compare with wildcard dimension value, different length, case2", + []cloudwatch.Dimension{ + {Name: awssdk.String("ID1"), Value: awssdk.String("111")}, + }, + []cloudwatch.Dimension{ + {Name: awssdk.String("ID1"), Value: awssdk.String("111")}, + {Name: awssdk.String("ID2"), Value: awssdk.String(dimensionValueWildcard)}, + }, + false, + }, } for _, c := range cases { @@ -1471,3 +1525,47 @@ func TestInsertTags(t *testing.T) { }) } } + +func TestConfigDimensionValueContainsWildcard(t *testing.T) { + cases := []struct { + title string + dimensions []Dimension + expectedResult bool + }{ + { + "test dimensions without wolidcard value", + []Dimension{ + { + Name: "InstanceId", + Value: "i-111111", + }, + { + Name: "InstanceId", + Value: "i-2222", + }, + }, + false, + }, + { + "test dimensions without wolidcard value", + []Dimension{ + { + Name: "InstanceId", + Value: "i-111111", + }, + { + Name: "InstanceId", + Value: dimensionValueWildcard, + }, + }, + true, + }, + } + + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + result := configDimensionValueContainsWildcard(c.dimensions) + assert.Equal(t, c.expectedResult, result) + }) + } +}