From 6bb97a30eaa2466759b0dcef8d5024d2dedf3dce Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 26 Feb 2020 07:24:03 -0700 Subject: [PATCH 01/13] add ListTagsForResourceRequest for rds metricset --- x-pack/metricbeat/module/aws/rds/data.go | 10 +++++----- x-pack/metricbeat/module/aws/rds/rds.go | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/x-pack/metricbeat/module/aws/rds/data.go b/x-pack/metricbeat/module/aws/rds/data.go index fcebee061d0..d914ed79f3a 100644 --- a/x-pack/metricbeat/module/aws/rds/data.go +++ b/x-pack/metricbeat/module/aws/rds/data.go @@ -71,11 +71,11 @@ var ( }, "login_failures": c.Float("LoginFailures"), - "db_instance.identifier": c.Str("DBInstanceIdentifier"), - "db_instance.db_cluster_identifier": c.Str("DBClusterIdentifier"), - "db_instance.class": c.Str("DatabaseClass"), - "db_instance.role": c.Str("Role"), - "db_instance.engine_name": c.Str("EngineName"), + //"db_instance.identifier": c.Str("DBInstanceIdentifier"), + //"db_instance.db_cluster_identifier": c.Str("DBClusterIdentifier"), + //"db_instance.class": c.Str("DatabaseClass"), + //"db_instance.role": c.Str("Role"), + //"db_instance.engine_name": c.Str("EngineName"), "aurora_bin_log_replica_lag": c.Float("AuroraBinlogReplicaLag"), diff --git a/x-pack/metricbeat/module/aws/rds/rds.go b/x-pack/metricbeat/module/aws/rds/rds.go index 34e00571b02..3924141edea 100644 --- a/x-pack/metricbeat/module/aws/rds/rds.go +++ b/x-pack/metricbeat/module/aws/rds/rds.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "github.com/elastic/beats/libbeat/common" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/rdsiface" @@ -49,6 +51,7 @@ type DBDetails struct { dbAvailabilityZone string dbIdentifier string dbStatus string + tags []aws.Tag } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -172,6 +175,20 @@ func getDBInstancesPerRegion(svc rdsiface.ClientAPI) ([]string, map[string]DBDet dbDetails.dbAvailabilityZone = *dbInstance.AvailabilityZone } + // Get tags for each RDS instance + listTagsInput := rds.ListTagsForResourceInput{ + ResourceName: dbInstance.DBInstanceArn, + } + reqListTags := svc.ListTagsForResourceRequest(&listTagsInput) + outputListTags, err := reqListTags.Send(context.TODO()) + if err != nil { + return nil, nil, errors.Wrap(err, "Error ListTagsForResourceRequest") + } + + for _, tag := range outputListTags.TagList { + dbDetails.tags = append(dbDetails.tags, aws.Tag{Key: *tag.Key, Value: *tag.Value}) + } + dbDetailsMap[*dbInstance.DBInstanceIdentifier] = dbDetails } return dbInstanceIDs, dbDetailsMap, nil @@ -268,6 +285,11 @@ func createCloudWatchEvents(getMetricDataResults []cloudwatch.MetricDataResult, events[dbIdentifier].MetricSetFields.Put("db_instance.class", dbInstanceMap[dbIdentifier].dbClass) events[dbIdentifier].MetricSetFields.Put("db_instance.identifier", dbInstanceMap[dbIdentifier].dbIdentifier) events[dbIdentifier].MetricSetFields.Put("db_instance.status", dbInstanceMap[dbIdentifier].dbStatus) + + // By default, replace dot "." using under bar "_" for tag keys and values + for _, tag := range dbInstanceMap[dbIdentifier].tags { + events[dbIdentifier].MetricSetFields.Put("db_instance.tags."+common.DeDot(tag.Key), common.DeDot(tag.Value)) + } } } metricSetFieldResults[dimValues][labels[i]] = fmt.Sprint(labels[(i + 1)]) From 6097bd501c7098a31a4422bd7723388437c8d078 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 26 Feb 2020 11:40:29 -0700 Subject: [PATCH 02/13] Regenerate data.json for tags --- CHANGELOG.next.asciidoc | 1 + .../metricbeat/module/aws/rds/_meta/data.json | 52 +++++++++++-------- x-pack/metricbeat/module/aws/rds/data.go | 10 ++-- x-pack/metricbeat/module/aws/rds/rds.go | 2 +- 4 files changed, 38 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c38a716d5f8..9440074e7f8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -181,6 +181,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add filtering option for prometheus collector. {pull}16420[16420] - Add metricsets based on Ceph Manager Daemon to the `ceph` module. {issue}7723[7723] {pull}16254[16254] - Release `statsd` module as GA. {pull}16447[16447] {issue}14280[14280] +- Add collecting tags for rds metricset in aws module. {pull}16605[16605] {issue}16358[16358] *Packetbeat* diff --git a/x-pack/metricbeat/module/aws/rds/_meta/data.json b/x-pack/metricbeat/module/aws/rds/_meta/data.json index 70fa4d531b0..ed1c988651a 100644 --- a/x-pack/metricbeat/module/aws/rds/_meta/data.json +++ b/x-pack/metricbeat/module/aws/rds/_meta/data.json @@ -3,46 +3,55 @@ "aws": { "rds": { "aurora_bin_log_replica_lag": 0, - "aurora_replica.lag.ms": 19.683, - "aurora_replica.lag_max.ms": 19.651500701904297, - "aurora_replica.lag_min.ms": 19.651500701904297, + "aurora_replica.lag_max.ms": 9.630999565124512, + "aurora_replica.lag_min.ms": 9.630999565124512, "cache_hit_ratio.buffer": 100, "cache_hit_ratio.result_set": 0, "cpu": { "total": { - "pct": 0.035 + "pct": 0.04 } }, "database_connections": 0, - "db_instance.class": "db.r5.large", + "db_instance": { + "arn": "arn:aws:rds:us-east-1:428152502467:db:database-1-instance-1", + "class": "db.r5.large", + "identifier": "database-1-instance-1", + "status": "available", + "tags": { + "created-by": "ks", + "dept": "engr" + } + }, + "db_instance.identifier": "database-1-instance-1", "deadlocks": 0, "disk_usage": { "bin_log.bytes": 0 }, - "engine_uptime.sec": 278598.25, - "free_local_storage.bytes": 33112842240, - "freeable_memory.bytes": 4879514624, + "engine_uptime.sec": 2240297, + "free_local_storage.bytes": 31996100608, + "freeable_memory.bytes": 4659970048, "latency": { - "commit": 2.9322999999999997, + "commit": 6.052133333333333, "ddl": 0, "delete": 0, - "dml": 0.08325833333333334, - "insert": 0.08325833333333334, - "select": 0.20890321021571023, + "dml": 0.1599, + "insert": 0.1599, + "select": 0.1762108108108108, "update": 0 }, "login_failures": 0, - "queries": 7.807070323085375, + "queries": 8.868736038408963, "throughput": { - "commit": 0.2500020925349523, + "commit": 0.5002668089647812, "ddl": 0, "delete": 0, - "dml": 0.2500020925349523, - "insert": 0.2500020925349523, - "network": 1.7498192019524124, - "network_receive": 0.8749096009762062, - "network_transmit": 0.8749096009762062, - "select": 2.8751975752392616, + "dml": 0.5002668089647812, + "insert": 0.5002668089647812, + "network": 1.3996267661956812, + "network_receive": 0.6998133830978406, + "network_transmit": 0.6998133830978406, + "select": 3.0849786552828173, "update": 0 }, "transactions": { @@ -56,8 +65,9 @@ "id": "428152502467", "name": "elastic-beats" }, + "availability_zone": "us-east-1b", "provider": "aws", - "region": "eu-west-1" + "region": "us-east-1" }, "event": { "dataset": "aws.rds", diff --git a/x-pack/metricbeat/module/aws/rds/data.go b/x-pack/metricbeat/module/aws/rds/data.go index d914ed79f3a..fcebee061d0 100644 --- a/x-pack/metricbeat/module/aws/rds/data.go +++ b/x-pack/metricbeat/module/aws/rds/data.go @@ -71,11 +71,11 @@ var ( }, "login_failures": c.Float("LoginFailures"), - //"db_instance.identifier": c.Str("DBInstanceIdentifier"), - //"db_instance.db_cluster_identifier": c.Str("DBClusterIdentifier"), - //"db_instance.class": c.Str("DatabaseClass"), - //"db_instance.role": c.Str("Role"), - //"db_instance.engine_name": c.Str("EngineName"), + "db_instance.identifier": c.Str("DBInstanceIdentifier"), + "db_instance.db_cluster_identifier": c.Str("DBClusterIdentifier"), + "db_instance.class": c.Str("DatabaseClass"), + "db_instance.role": c.Str("Role"), + "db_instance.engine_name": c.Str("EngineName"), "aurora_bin_log_replica_lag": c.Float("AuroraBinlogReplicaLag"), diff --git a/x-pack/metricbeat/module/aws/rds/rds.go b/x-pack/metricbeat/module/aws/rds/rds.go index 3924141edea..681f99b1348 100644 --- a/x-pack/metricbeat/module/aws/rds/rds.go +++ b/x-pack/metricbeat/module/aws/rds/rds.go @@ -279,7 +279,7 @@ func createCloudWatchEvents(getMetricDataResults []cloudwatch.MetricDataResult, for i := 1; i < len(labels); i += 2 { if labels[i] == "DBInstanceIdentifier" { dbIdentifier := labels[i+1] - if _, found := events[dbIdentifier]; !found { + if _, found := events[dbIdentifier]; found { events[dbIdentifier].RootFields.Put("cloud.availability_zone", dbInstanceMap[dbIdentifier].dbAvailabilityZone) events[dbIdentifier].MetricSetFields.Put("db_instance.arn", dbInstanceMap[dbIdentifier].dbArn) events[dbIdentifier].MetricSetFields.Put("db_instance.class", dbInstanceMap[dbIdentifier].dbClass) From 7682a3dd3fbaf05b226cd1c44501b81f854a6ebf Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 26 Feb 2020 12:25:05 -0700 Subject: [PATCH 03/13] add unit test --- x-pack/metricbeat/module/aws/rds/rds_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/x-pack/metricbeat/module/aws/rds/rds_test.go b/x-pack/metricbeat/module/aws/rds/rds_test.go index 57366f5ae24..2f855e9f7ee 100644 --- a/x-pack/metricbeat/module/aws/rds/rds_test.go +++ b/x-pack/metricbeat/module/aws/rds/rds_test.go @@ -11,6 +11,8 @@ import ( "testing" "time" + "github.com/elastic/beats/x-pack/metricbeat/module/aws" + awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/rdsiface" @@ -73,6 +75,20 @@ func (m *MockRDSClient) DescribeDBInstancesRequest(input *rds.DescribeDBInstance } } +func (m *MockRDSClient) ListTagsForResourceRequest(input *rds.ListTagsForResourceInput) rds.ListTagsForResourceRequest { + httpReq, _ := http.NewRequest("", "", nil) + return rds.ListTagsForResourceRequest{ + Request: &awssdk.Request{ + Data: &rds.ListTagsForResourceOutput{ + TagList: []rds.Tag{ + {Key: awssdk.String("dept"), Value: awssdk.String("engr")}, + }, + }, + HTTPRequest: httpReq, + }, + } +} + func TestConstructLabel(t *testing.T) { cases := []struct { dimensions []cloudwatch.Dimension @@ -126,6 +142,7 @@ func TestGetDBInstancesPerRegion(t *testing.T) { dbAvailabilityZone: availabilityZone, dbIdentifier: dbInstanceIdentifier, dbStatus: dbInstanceStatus, + tags: []aws.Tag{{Key: "dept", Value: "engr"}}, } assert.Equal(t, dbInstanceMap, dbDetailsMap[dbInstanceIDs[0]]) } From 80fc9c7eb808905ee36baa9be3e34c111c3ee61b Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 27 Feb 2020 10:33:47 -0700 Subject: [PATCH 04/13] dedot tags by default and add test for dedot --- x-pack/metricbeat/module/aws/rds/rds.go | 10 +++++++--- x-pack/metricbeat/module/aws/rds/rds_test.go | 8 ++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/x-pack/metricbeat/module/aws/rds/rds.go b/x-pack/metricbeat/module/aws/rds/rds.go index 681f99b1348..5209f4feb51 100644 --- a/x-pack/metricbeat/module/aws/rds/rds.go +++ b/x-pack/metricbeat/module/aws/rds/rds.go @@ -186,7 +186,12 @@ func getDBInstancesPerRegion(svc rdsiface.ClientAPI) ([]string, map[string]DBDet } for _, tag := range outputListTags.TagList { - dbDetails.tags = append(dbDetails.tags, aws.Tag{Key: *tag.Key, Value: *tag.Value}) + // By default, replace dot "." using under bar "_" for tag keys and values + dbDetails.tags = append(dbDetails.tags, + aws.Tag{ + Key: common.DeDot(*tag.Key), + Value: common.DeDot(*tag.Value), + }) } dbDetailsMap[*dbInstance.DBInstanceIdentifier] = dbDetails @@ -286,9 +291,8 @@ func createCloudWatchEvents(getMetricDataResults []cloudwatch.MetricDataResult, events[dbIdentifier].MetricSetFields.Put("db_instance.identifier", dbInstanceMap[dbIdentifier].dbIdentifier) events[dbIdentifier].MetricSetFields.Put("db_instance.status", dbInstanceMap[dbIdentifier].dbStatus) - // By default, replace dot "." using under bar "_" for tag keys and values for _, tag := range dbInstanceMap[dbIdentifier].tags { - events[dbIdentifier].MetricSetFields.Put("db_instance.tags."+common.DeDot(tag.Key), common.DeDot(tag.Value)) + events[dbIdentifier].MetricSetFields.Put("db_instance.tags."+tag.Key, tag.Value) } } } diff --git a/x-pack/metricbeat/module/aws/rds/rds_test.go b/x-pack/metricbeat/module/aws/rds/rds_test.go index 2f855e9f7ee..a4de3846688 100644 --- a/x-pack/metricbeat/module/aws/rds/rds_test.go +++ b/x-pack/metricbeat/module/aws/rds/rds_test.go @@ -81,7 +81,8 @@ func (m *MockRDSClient) ListTagsForResourceRequest(input *rds.ListTagsForResourc Request: &awssdk.Request{ Data: &rds.ListTagsForResourceOutput{ TagList: []rds.Tag{ - {Key: awssdk.String("dept"), Value: awssdk.String("engr")}, + {Key: awssdk.String("dept.name"), Value: awssdk.String("eng.software")}, + {Key: awssdk.String("created-by"), Value: awssdk.String("foo")}, }, }, HTTPRequest: httpReq, @@ -142,7 +143,10 @@ func TestGetDBInstancesPerRegion(t *testing.T) { dbAvailabilityZone: availabilityZone, dbIdentifier: dbInstanceIdentifier, dbStatus: dbInstanceStatus, - tags: []aws.Tag{{Key: "dept", Value: "engr"}}, + tags: []aws.Tag{ + {Key: "dept_name", Value: "eng_software"}, + {Key: "created-by", Value: "foo"}, + }, } assert.Equal(t, dbInstanceMap, dbDetailsMap[dbInstanceIDs[0]]) } From 1462e693cc457574b29f9c82c8651aab8aaa6ab0 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 27 Feb 2020 14:18:12 -0700 Subject: [PATCH 05/13] update doc for rds --- metricbeat/docs/modules/aws.asciidoc | 2 +- x-pack/metricbeat/module/aws/_meta/docs.asciidoc | 2 +- x-pack/metricbeat/module/aws/rds/_meta/docs.asciidoc | 4 +++- x-pack/metricbeat/module/aws/rds/rds.go | 8 +++++--- x-pack/metricbeat/module/aws/rds/rds_test.go | 10 +++++----- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 6f157672faf..866276818c3 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -38,7 +38,7 @@ Currently, we have `billing`, `cloudwatch`, `dynamodb`, `ebs`, `ec2`, `elb`, `lambda`, `rds`, `s3_daily_storage`, `s3_request`, `sns`, `sqs` and `usage` metricset in `aws` module. -Collecting `tags` for `ec2`, `cloudwatch`, and metricset created based on +Collecting `tags` for `ec2`, `rds`, `cloudwatch`, and metricset created based on `cloudwatch` using light module is supported. * *tags.*: Tag key value pairs from aws resources. A tag is a label that user assigns to an AWS resource. diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index 942910f0e1a..accca1c9300 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -30,7 +30,7 @@ Currently, we have `billing`, `cloudwatch`, `dynamodb`, `ebs`, `ec2`, `elb`, `lambda`, `rds`, `s3_daily_storage`, `s3_request`, `sns`, `sqs` and `usage` metricset in `aws` module. -Collecting `tags` for `ec2`, `cloudwatch`, and metricset created based on +Collecting `tags` for `ec2`, `rds`, `cloudwatch`, and metricset created based on `cloudwatch` using light module is supported. * *tags.*: Tag key value pairs from aws resources. A tag is a label that user assigns to an AWS resource. diff --git a/x-pack/metricbeat/module/aws/rds/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/rds/_meta/docs.asciidoc index 9081d279969..51ce9a8bf23 100644 --- a/x-pack/metricbeat/module/aws/rds/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/rds/_meta/docs.asciidoc @@ -5,7 +5,8 @@ with Amazon RDS, users can monitor network throughput, I/O for read, write, and/ metadata operations, client connections, and burst credit balances for their DB instances. Amazon RDS sends metrics and dimensions to Amazon CloudWatch every minute. Amazon Aurora provides a variety of Amazon CloudWatch metrics that users can -use to monitor health and performance of their Aurora DB cluster. +use to monitor health and performance of their Aurora DB cluster. This metricset +by default collects all tags from AWS RDS. [float] === AWS Permissions @@ -14,6 +15,7 @@ Some specific AWS permissions are required for IAM user to collect AWS RDS metri cloudwatch:GetMetricData ec2:DescribeRegions rds:DescribeDBInstances +rds:ListTagsForResource sts:GetCallerIdentity iam:ListAccountAliases ---- diff --git a/x-pack/metricbeat/module/aws/rds/rds.go b/x-pack/metricbeat/module/aws/rds/rds.go index 5209f4feb51..399d6884eae 100644 --- a/x-pack/metricbeat/module/aws/rds/rds.go +++ b/x-pack/metricbeat/module/aws/rds/rds.go @@ -90,7 +90,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { m.Endpoint, "rds", regionName, awsConfig)) // Get DBInstance IDs per region - dbInstanceIDs, dbDetailsMap, err := getDBInstancesPerRegion(svc) + dbInstanceIDs, dbDetailsMap, err := m.getDBInstancesPerRegion(svc) if err != nil { err = errors.Wrap(err, "getDBInstancesPerRegion failed, skipping region "+regionName) m.Logger().Errorf(err.Error()) @@ -152,7 +152,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return nil } -func getDBInstancesPerRegion(svc rdsiface.ClientAPI) ([]string, map[string]DBDetails, error) { +func (m *MetricSet) getDBInstancesPerRegion(svc rdsiface.ClientAPI) ([]string, map[string]DBDetails, error) { describeInstanceInput := &rds.DescribeDBInstancesInput{} req := svc.DescribeDBInstancesRequest(describeInstanceInput) output, err := req.Send(context.TODO()) @@ -182,7 +182,9 @@ func getDBInstancesPerRegion(svc rdsiface.ClientAPI) ([]string, map[string]DBDet reqListTags := svc.ListTagsForResourceRequest(&listTagsInput) outputListTags, err := reqListTags.Send(context.TODO()) if err != nil { - return nil, nil, errors.Wrap(err, "Error ListTagsForResourceRequest") + m.Logger().Warn("ListTagsForResourceRequest failed, rds:ListTagsForResource permission is required for getting tags.") + dbDetailsMap[*dbInstance.DBInstanceIdentifier] = dbDetails + return dbInstanceIDs, dbDetailsMap, nil } for _, tag := range outputListTags.TagList { diff --git a/x-pack/metricbeat/module/aws/rds/rds_test.go b/x-pack/metricbeat/module/aws/rds/rds_test.go index a4de3846688..8e3b9bc55be 100644 --- a/x-pack/metricbeat/module/aws/rds/rds_test.go +++ b/x-pack/metricbeat/module/aws/rds/rds_test.go @@ -11,14 +11,13 @@ import ( "testing" "time" - "github.com/elastic/beats/x-pack/metricbeat/module/aws" - awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/rdsiface" - - "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/x-pack/metricbeat/module/aws" ) // MockRDSClient struct is used for unit tests. @@ -129,7 +128,8 @@ func TestConstructLabel(t *testing.T) { func TestGetDBInstancesPerRegion(t *testing.T) { mockSvc := &MockRDSClient{} - dbInstanceIDs, dbDetailsMap, err := getDBInstancesPerRegion(mockSvc) + m := MetricSet{} + dbInstanceIDs, dbDetailsMap, err := m.getDBInstancesPerRegion(mockSvc) if err != nil { t.FailNow() } From 384a9c1a4f7f526bff5529bd21e98309a28932df Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 28 Feb 2020 12:55:31 -0700 Subject: [PATCH 06/13] add checks for DBInstanceIdentifier, DBInstanceArn, DBInstanceClass and DBInstanceStatus --- x-pack/metricbeat/module/aws/rds/rds.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/x-pack/metricbeat/module/aws/rds/rds.go b/x-pack/metricbeat/module/aws/rds/rds.go index 399d6884eae..d9627004ebf 100644 --- a/x-pack/metricbeat/module/aws/rds/rds.go +++ b/x-pack/metricbeat/module/aws/rds/rds.go @@ -163,12 +163,22 @@ func (m *MetricSet) getDBInstancesPerRegion(svc rdsiface.ClientAPI) ([]string, m var dbInstanceIDs []string dbDetailsMap := map[string]DBDetails{} for _, dbInstance := range output.DBInstances { - dbInstanceIDs = append(dbInstanceIDs, *dbInstance.DBInstanceIdentifier) - dbDetails := DBDetails{ - dbArn: *dbInstance.DBInstanceArn, - dbClass: *dbInstance.DBInstanceClass, - dbIdentifier: *dbInstance.DBInstanceIdentifier, - dbStatus: *dbInstance.DBInstanceStatus, + dbDetails := DBDetails{} + if dbInstance.DBInstanceIdentifier != nil { + dbDetails.dbIdentifier = *dbInstance.DBInstanceIdentifier + dbInstanceIDs = append(dbInstanceIDs, *dbInstance.DBInstanceIdentifier) + } + + if dbInstance.DBInstanceArn != nil { + dbDetails.dbArn = *dbInstance.DBInstanceArn + } + + if dbInstance.DBInstanceClass != nil { + dbDetails.dbClass = *dbInstance.DBInstanceClass + } + + if dbInstance.DBInstanceClass != nil { + dbDetails.dbStatus = *dbInstance.DBInstanceStatus } if dbInstance.AvailabilityZone != nil { From d5f461bea903bd4be368fee7d32fb0df869d3728 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 2 Mar 2020 20:16:02 -0700 Subject: [PATCH 07/13] Add tags_filter for rds metricset --- x-pack/metricbeat/module/aws/aws.go | 16 ++- x-pack/metricbeat/module/aws/ec2/ec2.go | 14 +- x-pack/metricbeat/module/aws/ec2/ec2_test.go | 140 ++++++++++++++++++- x-pack/metricbeat/module/aws/rds/rds.go | 38 ++++- x-pack/metricbeat/module/aws/rds/rds_test.go | 86 +++++++++++- 5 files changed, 271 insertions(+), 23 deletions(-) diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index e1ef1f7c100..fc46b43b939 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -12,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/ec2iface" "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi" "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/pkg/errors" @@ -23,9 +24,10 @@ import ( // Config defines all required and optional parameters for aws metricsets type Config struct { - Period time.Duration `config:"period" validate:"nonzero,required"` - Regions []string `config:"regions"` - AWSConfig awscommon.ConfigAWS `config:",inline"` + Period time.Duration `config:"period" validate:"nonzero,required"` + Regions []string `config:"regions"` + AWSConfig awscommon.ConfigAWS `config:",inline"` + TagsFilter []Tag `config:"tags_filter"` } // MetricSet is the base metricset for all aws metricsets @@ -37,6 +39,7 @@ type MetricSet struct { AwsConfig *awssdk.Config AccountName string AccountID string + TagsFilter []Tag } // Tag holds a configuration specific for ec2 and cloudwatch metricset. @@ -84,6 +87,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { BaseMetricSet: base, Period: config.Period, AwsConfig: &awsConfig, + TagsFilter: config.TagsFilter, } // Get IAM account name @@ -194,6 +198,12 @@ func CheckTagFiltersExist(tagsFilter []Tag, tags interface{}) bool { tagKeys = append(tagKeys, *tag.Key) tagValues = append(tagValues, *tag.Value) } + case []rds.Tag: + tagsRDS := tags.([]rds.Tag) + for _, tag := range tagsRDS { + tagKeys = append(tagKeys, *tag.Key) + tagValues = append(tagValues, *tag.Value) + } } for _, tagFilter := range tagsFilter { diff --git a/x-pack/metricbeat/module/aws/ec2/ec2.go b/x-pack/metricbeat/module/aws/ec2/ec2.go index 023788e905e..a0a61ca7260 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2.go @@ -45,7 +45,6 @@ func init() { // interface methods except for Fetch. type MetricSet struct { *aws.MetricSet - TagsFilter []aws.Tag `config:"tags_filter"` } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -56,15 +55,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, errors.Wrap(err, "error creating aws metricset") } - config := struct { - Tags []aws.Tag `config:"tags_filter"` - }{} - - err = base.Module().UnpackConfig(&config) - if err != nil { - return nil, errors.Wrap(err, "error unpack raw module config using UnpackConfig") - } - // Check if period is set to be multiple of 60s or 300s remainder300 := int(metricSet.Period.Seconds()) % 300 remainder60 := int(metricSet.Period.Seconds()) % 60 @@ -76,8 +66,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { } return &MetricSet{ - MetricSet: metricSet, - TagsFilter: config.Tags, + MetricSet: metricSet, }, nil } @@ -201,6 +190,7 @@ func (m *MetricSet) createCloudWatchEvents(getMetricDataResults []cloudwatch.Met // If tag filter doesn't exist in tagKeys/tagValues, // then do not report this event/instance. if exists := aws.CheckTagFiltersExist(m.TagsFilter, tags); !exists { + delete(events, instanceID) continue } } diff --git a/x-pack/metricbeat/module/aws/ec2/ec2_test.go b/x-pack/metricbeat/module/aws/ec2/ec2_test.go index 6e06acd7736..e2c0b93f69f 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2_test.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2_test.go @@ -211,7 +211,6 @@ func TestCreateCloudWatchEventsDedotTags(t *testing.T) { metricSet := MetricSet{ &aws.MetricSet{}, - nil, } events, err := metricSet.createCloudWatchEvents(getMetricDataOutput, instancesOutputs, "us-west-1") assert.NoError(t, err) @@ -222,6 +221,145 @@ func TestCreateCloudWatchEventsDedotTags(t *testing.T) { assert.Equal(t, expectedEvent.MetricSetFields["tags"], events[instanceID].ModuleFields["tags"]) } +func TestCreateCloudWatchEventsWithTagsFilter(t *testing.T) { + expectedEvent := mb.Event{ + RootFields: common.MapStr{ + "cloud": common.MapStr{ + "region": regionName, + "provider": "aws", + "instance": common.MapStr{"id": "i-123"}, + "machine": common.MapStr{"type": "t2.medium"}, + "availability_zone": "us-west-1a", + }, + }, + MetricSetFields: common.MapStr{ + "cpu": common.MapStr{ + "total": common.MapStr{"pct": 0.25}, + }, + "instance": common.MapStr{ + "image": common.MapStr{"id": "image-123"}, + "core": common.MapStr{"count": int64(1)}, + "threads_per_core": int64(1), + "state": common.MapStr{"code": int64(16), "name": "running"}, + "monitoring": common.MapStr{"state": "disabled"}, + "public": common.MapStr{ + "dns_name": "ec2-1-2-3-4.us-west-1.compute.amazonaws.com", + "ip": "1.2.3.4", + }, + "private": common.MapStr{ + "dns_name": "ip-5-6-7-8.us-west-1.compute.internal", + "ip": "5.6.7.8", + }, + }, + "tags": common.MapStr{ + "app_kubernetes_io/name": "foo", + "helm_sh/chart": "foo-chart", + }, + }, + } + + svcEC2Mock := &MockEC2Client{} + instanceIDs, instancesOutputs, err := getInstancesPerRegion(svcEC2Mock) + assert.NoError(t, err) + assert.Equal(t, 1, len(instanceIDs)) + instanceID := instanceIDs[0] + assert.Equal(t, instanceID, instanceID) + timestamp := time.Now() + + getMetricDataOutput := []cloudwatch.MetricDataResult{ + { + Id: &id1, + Label: &label1, + Values: []float64{0.25}, + Timestamps: []time.Time{timestamp}, + }, + { + Id: &id2, + Label: &label2, + Values: []float64{0.0}, + Timestamps: []time.Time{timestamp}, + }, + { + Id: &id3, + Label: &label3, + Values: []float64{0.0}, + Timestamps: []time.Time{timestamp}, + }, + { + Id: &id4, + Label: &label4, + Values: []float64{0.0}, + Timestamps: []time.Time{timestamp}, + }, + } + + metricSet := MetricSet{ + &aws.MetricSet{ + TagsFilter: []aws.Tag{{ + Key: "app.kubernetes.io/name", + Value: "foo", + }}, + }, + } + events, err := metricSet.createCloudWatchEvents(getMetricDataOutput, instancesOutputs, "us-west-1") + + assert.NoError(t, err) + assert.Equal(t, 1, len(events)) + assert.Equal(t, expectedEvent.RootFields, events[instanceID].RootFields) + assert.Equal(t, expectedEvent.MetricSetFields["cpu"], events[instanceID].MetricSetFields["cpu"]) + assert.Equal(t, expectedEvent.MetricSetFields["instance"], events[instanceID].MetricSetFields["instance"]) + assert.Equal(t, expectedEvent.MetricSetFields["tags"], events[instanceID].ModuleFields["tags"]) +} + +func TestCreateCloudWatchEventsWithNotMatchingTagsFilter(t *testing.T) { + svcEC2Mock := &MockEC2Client{} + instanceIDs, instancesOutputs, err := getInstancesPerRegion(svcEC2Mock) + assert.NoError(t, err) + assert.Equal(t, 1, len(instanceIDs)) + instanceID := instanceIDs[0] + assert.Equal(t, instanceID, instanceID) + timestamp := time.Now() + + getMetricDataOutput := []cloudwatch.MetricDataResult{ + { + Id: &id1, + Label: &label1, + Values: []float64{0.25}, + Timestamps: []time.Time{timestamp}, + }, + { + Id: &id2, + Label: &label2, + Values: []float64{0.0}, + Timestamps: []time.Time{timestamp}, + }, + { + Id: &id3, + Label: &label3, + Values: []float64{0.0}, + Timestamps: []time.Time{timestamp}, + }, + { + Id: &id4, + Label: &label4, + Values: []float64{0.0}, + Timestamps: []time.Time{timestamp}, + }, + } + + metricSet := MetricSet{ + &aws.MetricSet{ + TagsFilter: []aws.Tag{{ + Key: "app_kubernetes_io/name", + Value: "not_foo", + }}, + }, + } + events, err := metricSet.createCloudWatchEvents(getMetricDataOutput, instancesOutputs, "us-west-1") + assert.NoError(t, err) + assert.Equal(t, 0, len(events)) +} + func TestConstructMetricQueries(t *testing.T) { name := "InstanceId" dim := cloudwatch.Dimension{ diff --git a/x-pack/metricbeat/module/aws/rds/rds.go b/x-pack/metricbeat/module/aws/rds/rds.go index d9627004ebf..f9446ab42fb 100644 --- a/x-pack/metricbeat/module/aws/rds/rds.go +++ b/x-pack/metricbeat/module/aws/rds/rds.go @@ -132,7 +132,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } // Create Cloudwatch Events for RDS - events, err := createCloudWatchEvents(metricDataOutput, regionName, dbDetailsMap, m.AccountName, m.AccountID) + events, err := m.createCloudWatchEvents(metricDataOutput, regionName, dbDetailsMap) if err != nil { m.Logger().Error(err.Error()) report.Error(err) @@ -197,6 +197,16 @@ func (m *MetricSet) getDBInstancesPerRegion(svc rdsiface.ClientAPI) ([]string, m return dbInstanceIDs, dbDetailsMap, nil } + if m.TagsFilter != nil { + // Check with each tag filter + // If tag filter doesn't exist in tagKeys/tagValues, + // then remove this dbInstance entry from dbDetailsMap. + if exists := aws.CheckTagFiltersExist(m.TagsFilter, outputListTags.TagList); !exists { + delete(dbDetailsMap, *dbInstance.DBInstanceIdentifier) + continue + } + } + for _, tag := range outputListTags.TagList { // By default, replace dot "." using under bar "_" for tag keys and values dbDetails.tags = append(dbDetails.tags, @@ -205,7 +215,6 @@ func (m *MetricSet) getDBInstancesPerRegion(svc rdsiface.ClientAPI) ([]string, m Value: common.DeDot(*tag.Value), }) } - dbDetailsMap[*dbInstance.DBInstanceIdentifier] = dbDetails } return dbInstanceIDs, dbDetailsMap, nil @@ -257,7 +266,7 @@ func constructLabel(metricDimensions []cloudwatch.Dimension, metricName string) return label } -func createCloudWatchEvents(getMetricDataResults []cloudwatch.MetricDataResult, regionName string, dbInstanceMap map[string]DBDetails, accountName string, accountID string) (map[string]mb.Event, error) { +func (m *MetricSet) createCloudWatchEvents(getMetricDataResults []cloudwatch.MetricDataResult, regionName string, dbInstanceMap map[string]DBDetails) (map[string]mb.Event, error) { // Initialize events and metricSetFieldResults per dbInstance events := map[string]mb.Event{} metricSetFieldResults := map[string]map[string]interface{}{} @@ -279,7 +288,7 @@ func createCloudWatchEvents(getMetricDataResults []cloudwatch.MetricDataResult, } if _, ok := events[dimValues]; !ok { - events[dimValues] = aws.InitEvent(regionName, accountName, accountID) + events[dimValues] = aws.InitEvent(regionName, m.AccountName, m.AccountID) } if _, ok := metricSetFieldResults[dimValues]; !ok { @@ -297,6 +306,10 @@ func createCloudWatchEvents(getMetricDataResults []cloudwatch.MetricDataResult, if labels[i] == "DBInstanceIdentifier" { dbIdentifier := labels[i+1] if _, found := events[dbIdentifier]; found { + if _, found := dbInstanceMap[dbIdentifier]; !found { + delete(metricSetFieldResults, dimValues) + continue + } events[dbIdentifier].RootFields.Put("cloud.availability_zone", dbInstanceMap[dbIdentifier].dbAvailabilityZone) events[dbIdentifier].MetricSetFields.Put("db_instance.arn", dbInstanceMap[dbIdentifier].dbArn) events[dbIdentifier].MetricSetFields.Put("db_instance.class", dbInstanceMap[dbIdentifier].dbClass) @@ -304,12 +317,27 @@ func createCloudWatchEvents(getMetricDataResults []cloudwatch.MetricDataResult, events[dbIdentifier].MetricSetFields.Put("db_instance.status", dbInstanceMap[dbIdentifier].dbStatus) for _, tag := range dbInstanceMap[dbIdentifier].tags { - events[dbIdentifier].MetricSetFields.Put("db_instance.tags."+tag.Key, tag.Value) + events[dbIdentifier].ModuleFields.Put("tags."+tag.Key, tag.Value) } } } metricSetFieldResults[dimValues][labels[i]] = fmt.Sprint(labels[(i + 1)]) } + + // if tags_filter is given, then only return metrics with DBInstanceIdentifier as dimension + if m.TagsFilter != nil { + if len(labels) == 1 { + delete(events, dimValues) + delete(metricSetFieldResults, dimValues) + } + + for i := 1; i < len(labels); i += 2 { + if labels[i] != "DBInstanceIdentifier" && i == len(labels)-2 { + delete(events, dimValues) + delete(metricSetFieldResults, dimValues) + } + } + } } } } diff --git a/x-pack/metricbeat/module/aws/rds/rds_test.go b/x-pack/metricbeat/module/aws/rds/rds_test.go index 8e3b9bc55be..59ffeb5493c 100644 --- a/x-pack/metricbeat/module/aws/rds/rds_test.go +++ b/x-pack/metricbeat/module/aws/rds/rds_test.go @@ -128,8 +128,72 @@ func TestConstructLabel(t *testing.T) { func TestGetDBInstancesPerRegion(t *testing.T) { mockSvc := &MockRDSClient{} - m := MetricSet{} - dbInstanceIDs, dbDetailsMap, err := m.getDBInstancesPerRegion(mockSvc) + metricSet := MetricSet{ + &aws.MetricSet{}, + } + dbInstanceIDs, dbDetailsMap, err := metricSet.getDBInstancesPerRegion(mockSvc) + if err != nil { + t.FailNow() + } + + assert.Equal(t, 1, len(dbInstanceIDs)) + assert.Equal(t, 1, len(dbDetailsMap)) + + dbInstanceMap := DBDetails{ + dbArn: dbInstanceArn, + dbClass: dbInstanceClass, + dbAvailabilityZone: availabilityZone, + dbIdentifier: dbInstanceIdentifier, + dbStatus: dbInstanceStatus, + tags: []aws.Tag{ + {Key: "dept_name", Value: "eng_software"}, + {Key: "created-by", Value: "foo"}, + }, + } + assert.Equal(t, dbInstanceMap, dbDetailsMap[dbInstanceIDs[0]]) +} + +func TestGetDBInstancesPerRegionWithTagsFilter(t *testing.T) { + mockSvc := &MockRDSClient{} + metricSet := MetricSet{ + &aws.MetricSet{ + TagsFilter: []aws.Tag{ + {Key: "created-by", Value: "foo"}, + }, + }, + } + dbInstanceIDs, dbDetailsMap, err := metricSet.getDBInstancesPerRegion(mockSvc) + if err != nil { + t.FailNow() + } + + assert.Equal(t, 1, len(dbInstanceIDs)) + assert.Equal(t, 1, len(dbDetailsMap)) + + dbInstanceMap := DBDetails{ + dbArn: dbInstanceArn, + dbClass: dbInstanceClass, + dbAvailabilityZone: availabilityZone, + dbIdentifier: dbInstanceIdentifier, + dbStatus: dbInstanceStatus, + tags: []aws.Tag{ + {Key: "dept_name", Value: "eng_software"}, + {Key: "created-by", Value: "foo"}, + }, + } + assert.Equal(t, dbInstanceMap, dbDetailsMap[dbInstanceIDs[0]]) +} + +func TestGetDBInstancesPerRegionWithDotInTag(t *testing.T) { + mockSvc := &MockRDSClient{} + metricSet := MetricSet{ + &aws.MetricSet{ + TagsFilter: []aws.Tag{ + {Key: "dept.name", Value: "eng.software"}, + }, + }, + } + dbInstanceIDs, dbDetailsMap, err := metricSet.getDBInstancesPerRegion(mockSvc) if err != nil { t.FailNow() } @@ -151,6 +215,24 @@ func TestGetDBInstancesPerRegion(t *testing.T) { assert.Equal(t, dbInstanceMap, dbDetailsMap[dbInstanceIDs[0]]) } +func TestGetDBInstancesPerRegionWithNoMatchingTagsFilter(t *testing.T) { + mockSvc := &MockRDSClient{} + metricSet := MetricSet{ + &aws.MetricSet{ + TagsFilter: []aws.Tag{ + {Key: "dept.name", Value: "accounting"}, + }, + }, + } + dbInstanceIDs, dbDetailsMap, err := metricSet.getDBInstancesPerRegion(mockSvc) + if err != nil { + t.FailNow() + } + + assert.Equal(t, 1, len(dbInstanceIDs)) + assert.Equal(t, 0, len(dbDetailsMap)) +} + func TestConstructMetricQueries(t *testing.T) { dim1 := cloudwatch.Dimension{ Name: &dimName1, From 034bd64c66be6136aef51d2bd3fe0a379bfdc9d7 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 2 Mar 2020 21:33:18 -0700 Subject: [PATCH 08/13] update documentation for tags_filter --- CHANGELOG.next.asciidoc | 2 +- metricbeat/docs/modules/aws.asciidoc | 3 ++ x-pack/metricbeat/metricbeat.reference.yml | 6 +++ .../module/aws/_meta/config.reference.yml | 3 ++ .../metricbeat/module/aws/rds/_meta/data.json | 44 +++++++++---------- 5 files changed, 35 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b5ac2f5fd3b..72fbad2dfe8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -195,7 +195,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add filtering option for prometheus collector. {pull}16420[16420] - Add metricsets based on Ceph Manager Daemon to the `ceph` module. {issue}7723[7723] {pull}16254[16254] - Release `statsd` module as GA. {pull}16447[16447] {issue}14280[14280] -- Add collecting tags for rds metricset in aws module. {pull}16605[16605] {issue}16358[16358] +- Add collecting tags and tags_filter for rds metricset in aws module. {pull}16605[16605] {issue}16358[16358] - Add `cloudfoundry` module to send events from Cloud Foundry. {pull}16671[16671] - Add `redisenterprise` module. {pull}16482[16482] {issue}15269[15269] diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index d3fa79b2a42..73d10f37ba2 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -295,6 +295,9 @@ metricbeat.modules: - module: aws period: 60s credential_profile_name: test-mb + tags_filter: + - key: "dept" + value: "eng" metricsets: - rds ---- diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index e763bfdaa65..72c30bbfc28 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -216,6 +216,9 @@ metricbeat.modules: - module: aws period: 60s credential_profile_name: test-mb + tags_filter: + - key: "dept" + value: "eng" metricsets: - rds @@ -304,6 +307,9 @@ metricbeat.modules: - value enabled: true api_address: '${CLOUDFOUNDRY_API_ADDRESS:""}' + doppler_address: '${CLOUDFOUNDRY_DOPPLER_ADDRESS:""}' + uaa_address: '${CLOUDFOUNDRY_UAA_ADDRESS:""}' + rlp_address: '${CLOUDFOUNDRY_RLP_ADDRESS:""}' client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}' client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}' diff --git a/x-pack/metricbeat/module/aws/_meta/config.reference.yml b/x-pack/metricbeat/module/aws/_meta/config.reference.yml index d7e7586e4a5..d6439e56b5e 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.reference.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.reference.yml @@ -35,5 +35,8 @@ - module: aws period: 60s credential_profile_name: test-mb + tags_filter: + - key: "dept" + value: "eng" metricsets: - rds diff --git a/x-pack/metricbeat/module/aws/rds/_meta/data.json b/x-pack/metricbeat/module/aws/rds/_meta/data.json index ed1c988651a..35d89c69fb7 100644 --- a/x-pack/metricbeat/module/aws/rds/_meta/data.json +++ b/x-pack/metricbeat/module/aws/rds/_meta/data.json @@ -3,8 +3,8 @@ "aws": { "rds": { "aurora_bin_log_replica_lag": 0, - "aurora_replica.lag_max.ms": 9.630999565124512, - "aurora_replica.lag_min.ms": 9.630999565124512, + "aurora_replica.lag_max.ms": 19.108999252319336, + "aurora_replica.lag_min.ms": 19.108999252319336, "cache_hit_ratio.buffer": 100, "cache_hit_ratio.result_set": 0, "cpu": { @@ -17,47 +17,47 @@ "arn": "arn:aws:rds:us-east-1:428152502467:db:database-1-instance-1", "class": "db.r5.large", "identifier": "database-1-instance-1", - "status": "available", - "tags": { - "created-by": "ks", - "dept": "engr" - } + "status": "available" }, "db_instance.identifier": "database-1-instance-1", "deadlocks": 0, "disk_usage": { "bin_log.bytes": 0 }, - "engine_uptime.sec": 2240297, - "free_local_storage.bytes": 31996100608, - "freeable_memory.bytes": 4659970048, + "engine_uptime.sec": 2704277, + "free_local_storage.bytes": 31745863680, + "freeable_memory.bytes": 4634234880, "latency": { - "commit": 6.052133333333333, + "commit": 5.270933333333333, "ddl": 0, "delete": 0, - "dml": 0.1599, - "insert": 0.1599, - "select": 0.1762108108108108, + "dml": 0.1624, + "insert": 0.1624, + "select": 0.17333862433862435, "update": 0 }, "login_failures": 0, - "queries": 8.868736038408963, + "queries": 9.11833836203304, "throughput": { - "commit": 0.5002668089647812, + "commit": 0.5000916834753039, "ddl": 0, "delete": 0, - "dml": 0.5002668089647812, - "insert": 0.5002668089647812, - "network": 1.3996267661956812, - "network_receive": 0.6998133830978406, - "network_transmit": 0.6998133830978406, - "select": 3.0849786552828173, + "dml": 0.5000916834753039, + "insert": 0.5000916834753039, + "network": 1.4, + "network_receive": 0.7, + "network_transmit": 0.7, + "select": 3.150577605894414, "update": 0 }, "transactions": { "active": 0, "blocked": 0 } + }, + "tags": { + "created-by": "ks", + "dept": "engr" } }, "cloud": { From 8a110dc84bce00a49011d18ef7a341f2c7de4d08 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 4 Mar 2020 10:37:51 -0700 Subject: [PATCH 09/13] rerun make update --- go.mod | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/go.mod b/go.mod index de1bb8b07f9..5b050f213fa 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 + github.com/elastic/beats v7.6.0+incompatible github.com/elastic/ecs v1.4.0 github.com/elastic/go-libaudit v0.4.0 github.com/elastic/go-licenser v0.2.1 diff --git a/go.sum b/go.sum index d95286d6c56..c4f38904244 100644 --- a/go.sum +++ b/go.sum @@ -209,6 +209,8 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 h1:DW6WrARxK5J+o8uAKCiACi5wy9EK1UzrsCpGBPsKHAA= github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= +github.com/elastic/beats v7.6.0+incompatible h1:vUcDbZ/qsvox6pC/t/tSdSItlt1/pKbsKaHEVS4GWss= +github.com/elastic/beats v7.6.0+incompatible/go.mod h1:7cX7zGsOwJ01FLkZs9Tg5nBdnQi6XB3hYAyWekpKgeY= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/ecs v1.4.0 h1:BGIUwWJhThRO2IQxzm7ekV9TMUGwZoYyevT5/1xmMf0= From 2a5705a790a9b55d0126ab291e108d810e8133d1 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 4 Mar 2020 11:19:49 -0700 Subject: [PATCH 10/13] Fix import library with v7 --- x-pack/metricbeat/module/aws/rds/rds.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/metricbeat/module/aws/rds/rds.go b/x-pack/metricbeat/module/aws/rds/rds.go index 1621845d81f..6c1ca1c8b19 100644 --- a/x-pack/metricbeat/module/aws/rds/rds.go +++ b/x-pack/metricbeat/module/aws/rds/rds.go @@ -11,13 +11,12 @@ import ( "strings" "time" - "github.com/elastic/beats/libbeat/common" - "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/rdsiface" "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws" From 0657fc870e71d155c8dad91356ee80af2c7f4d19 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 4 Mar 2020 15:17:42 -0700 Subject: [PATCH 11/13] update import library in rds_test.go --- x-pack/metricbeat/module/aws/rds/rds_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/aws/rds/rds_test.go b/x-pack/metricbeat/module/aws/rds/rds_test.go index 59ffeb5493c..016ffda4434 100644 --- a/x-pack/metricbeat/module/aws/rds/rds_test.go +++ b/x-pack/metricbeat/module/aws/rds/rds_test.go @@ -17,7 +17,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/rds/rdsiface" "github.com/stretchr/testify/assert" - "github.com/elastic/beats/x-pack/metricbeat/module/aws" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws" ) // MockRDSClient struct is used for unit tests. From 40dfb9c4f39292b294c0e88ca458de9b503a876e Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 5 Mar 2020 08:23:47 -0700 Subject: [PATCH 12/13] rerun go mod tidy --- go.mod | 1 - go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/go.mod b/go.mod index 5b050f213fa..de1bb8b07f9 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,6 @@ require ( github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 - github.com/elastic/beats v7.6.0+incompatible github.com/elastic/ecs v1.4.0 github.com/elastic/go-libaudit v0.4.0 github.com/elastic/go-licenser v0.2.1 diff --git a/go.sum b/go.sum index c4f38904244..d95286d6c56 100644 --- a/go.sum +++ b/go.sum @@ -209,8 +209,6 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 h1:DW6WrARxK5J+o8uAKCiACi5wy9EK1UzrsCpGBPsKHAA= github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= -github.com/elastic/beats v7.6.0+incompatible h1:vUcDbZ/qsvox6pC/t/tSdSItlt1/pKbsKaHEVS4GWss= -github.com/elastic/beats v7.6.0+incompatible/go.mod h1:7cX7zGsOwJ01FLkZs9Tg5nBdnQi6XB3hYAyWekpKgeY= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/ecs v1.4.0 h1:BGIUwWJhThRO2IQxzm7ekV9TMUGwZoYyevT5/1xmMf0= From 51c3d762f4b5663bb24f122127d5d003f774d798 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 9 Mar 2020 13:59:03 -0600 Subject: [PATCH 13/13] add comment in ec2.go --- x-pack/metricbeat/module/aws/ec2/ec2.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/metricbeat/module/aws/ec2/ec2.go b/x-pack/metricbeat/module/aws/ec2/ec2.go index 4aefc2d9bd9..50030477412 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2.go @@ -190,6 +190,8 @@ func (m *MetricSet) createCloudWatchEvents(getMetricDataResults []cloudwatch.Met // If tag filter doesn't exist in tagKeys/tagValues, // then do not report this event/instance. if exists := aws.CheckTagFiltersExist(m.TagsFilter, tags); !exists { + // if tag filter doesn't exist, remove this event initial + // entry to avoid report an empty event. delete(events, instanceID) continue }