From 387bae06588fb0eb9288d20fb6ddacb7c9a6c251 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 28 Jan 2020 07:23:31 -0700 Subject: [PATCH] [Metricbeat] Add dedot to aws ec2 metricset and cloudwatch metricset (#15844) * Add dedot to aws ec2 metricset and cloudwatch metricset * Remove dedot config parameter --- CHANGELOG.next.asciidoc | 2 +- .../module/aws/cloudwatch/cloudwatch.go | 6 ++++-- x-pack/metricbeat/module/aws/ec2/ec2.go | 3 ++- x-pack/metricbeat/module/aws/ec2/ec2_test.go | 19 ++++++++++++++++++- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5f64b868049d..709e18625eef 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -67,9 +67,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Metricbeat* +- Add dedot for tags in ec2 metricset and cloudwatch metricset. {issue}15843[15843] {pull}15844[15844] - Use RFC3339 format for timestamps collected using the SQL module. {pull}15847[15847] - *Packetbeat* diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index e06e58ece652..31b3bfc1a3ec 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -17,6 +17,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface" "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/x-pack/metricbeat/module/aws" ) @@ -514,10 +515,11 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatchiface.ClientAPI, svcRes events[identifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID) } events[identifierValue] = insertRootFields(events[identifierValue], output.Values[timestampIdx], labels) + + // By default, replace dot "." using under bar "_" for tag keys and values for _, tag := range tags { - events[identifierValue].RootFields.Put("aws.tags."+*tag.Key, *tag.Value) + events[identifierValue].RootFields.Put("aws.tags."+common.DeDot(*tag.Key), common.DeDot(*tag.Value)) } - } } } diff --git a/x-pack/metricbeat/module/aws/ec2/ec2.go b/x-pack/metricbeat/module/aws/ec2/ec2.go index 3a31d32fe2e2..41770c495ab1 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2.go @@ -201,8 +201,9 @@ func (m *MetricSet) createCloudWatchEvents(getMetricDataResults []cloudwatch.Met } } + // By default, replace dot "." using under bar "_" for tag keys and values for _, tag := range tags { - events[instanceID].ModuleFields.Put("tags."+*tag.Key, *tag.Value) + events[instanceID].ModuleFields.Put("tags."+common.DeDot(*tag.Key), common.DeDot(*tag.Value)) } machineType, err := instanceOutput[instanceID].InstanceType.MarshalValue() diff --git a/x-pack/metricbeat/module/aws/ec2/ec2_test.go b/x-pack/metricbeat/module/aws/ec2/ec2_test.go index e5f99138f9b4..6e06acd77361 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2_test.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2_test.go @@ -73,6 +73,17 @@ func (m *MockEC2Client) DescribeInstancesRequest(input *ec2.DescribeInstancesInp privateDNSName := "ip-5-6-7-8.us-west-1.compute.internal" privateIP := "5.6.7.8" + tags := []ec2.Tag{ + { + Key: awssdk.String("app.kubernetes.io/name"), + Value: awssdk.String("foo"), + }, + { + Key: awssdk.String("helm.sh/chart"), + Value: awssdk.String("foo-chart"), + }, + } + instance := ec2.Instance{ InstanceId: awssdk.String(instanceID), InstanceType: ec2.InstanceTypeT2Medium, @@ -95,6 +106,7 @@ func (m *MockEC2Client) DescribeInstancesRequest(input *ec2.DescribeInstancesInp PublicIpAddress: &publicIP, PrivateDnsName: &privateDNSName, PrivateIpAddress: &privateIP, + Tags: tags, } httpReq, _ := http.NewRequest("", "", nil) @@ -126,7 +138,7 @@ func TestGetInstanceIDs(t *testing.T) { assert.Equal(t, awssdk.String("us-west-1a"), instancesOutputs[instanceID].Placement.AvailabilityZone) } -func TestCreateCloudWatchEvents(t *testing.T) { +func TestCreateCloudWatchEventsDedotTags(t *testing.T) { expectedEvent := mb.Event{ RootFields: common.MapStr{ "cloud": common.MapStr{ @@ -156,6 +168,10 @@ func TestCreateCloudWatchEvents(t *testing.T) { "ip": "5.6.7.8", }, }, + "tags": common.MapStr{ + "app_kubernetes_io/name": "foo", + "helm_sh/chart": "foo-chart", + }, }, } svcEC2Mock := &MockEC2Client{} @@ -203,6 +219,7 @@ func TestCreateCloudWatchEvents(t *testing.T) { assert.Equal(t, expectedEvent.RootFields, events[instanceID].RootFields) assert.Equal(t, expectedEvent.MetricSetFields["cpu"], events[instanceID].MetricSetFields["cpu"]) assert.Equal(t, expectedEvent.MetricSetFields["instance"], events[instanceID].MetricSetFields["instance"]) + assert.Equal(t, expectedEvent.MetricSetFields["tags"], events[instanceID].ModuleFields["tags"]) } func TestConstructMetricQueries(t *testing.T) {