From c940b007e1f09c26764d0078cddf0450c079a78b Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 18 Nov 2019 09:15:55 -0700 Subject: [PATCH] Cherry-pick #14123 to 7.x: [Metricbeat] Add collecting Application ELB and Network ELB into elb metricset (#14577) * [Metricbeat] Add collecting Application ELB and Network ELB into elb metricset (#14123) * Add alb and nlb into elb metricset * Generate different data.json file for different load balancers (cherry picked from commit 61b5102bc009f33dfc12f824585bb05ecbcd8dd0) * Update changelog --- CHANGELOG.next.asciidoc | 1 + .../metricbeat/module/aws/elb/_meta/data.json | 21 ++++--- .../module/aws/elb/_meta/data_alb.json | 39 +++++++++++++ .../module/aws/elb/_meta/data_nlb.json | 54 ++++++++++++++++++ .../module/aws/elb/_meta/docs.asciidoc | 57 +++++++++++++++++-- .../module/aws/elb/elb_integration_test.go | 26 ++++++++- x-pack/metricbeat/module/aws/elb/manifest.yml | 28 +++++++++ 7 files changed, 213 insertions(+), 13 deletions(-) create mode 100644 x-pack/metricbeat/module/aws/elb/_meta/data_alb.json create mode 100644 x-pack/metricbeat/module/aws/elb/_meta/data_nlb.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e69b6545942e..bd92d58e53cb 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -473,6 +473,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Release CoreDNS module as GA. {pull}14308[14308] - Release CouchDB module as GA. {pull}14300[14300] - Add `elasticsearch/enrich` metricset. {pull}14243[14243] {issue}14221[14221] +- Add support for Application ELB and Network ELB. {pull}14123[14123] {issue}13538[13538] {issue}13539[13539] - Release aws ebs metricset as GA. {pull}14312[14312] {issue}14060[14060] - Add metrics to envoyproxy server metricset and support for envoy proxy 1.12. {pull}14416[14416] {issue}13642[13642] diff --git a/x-pack/metricbeat/module/aws/elb/_meta/data.json b/x-pack/metricbeat/module/aws/elb/_meta/data.json index eaec2f483c01..eb96bb571ad8 100644 --- a/x-pack/metricbeat/module/aws/elb/_meta/data.json +++ b/x-pack/metricbeat/module/aws/elb/_meta/data.json @@ -3,20 +3,27 @@ "aws": { "cloudwatch": { "dimensions": { - "AvailabilityZone": "us-east-1c" + "LoadBalancerName": "tf-lb-20171218185946539900000004" }, "namespace": "AWS/ELB" }, "metrics": { - "BackendConnectionErrors": { - "sum": 10 + "EstimatedALBActiveConnectionCount": { + "avg": 2.6 }, - "HealthyHostCount": { - "max": 2 + "EstimatedALBConsumedLCUs": { + "avg": 0.000015000000000000002 }, - "UnHealthyHostCount": { - "max": 1 + "EstimatedALBNewConnectionCount": { + "avg": 0 + }, + "EstimatedProcessedBytes": { + "avg": 237.6 } + }, + "tags": { + "Name": "logstash", + "owner": "logstash" } }, "cloud": { diff --git a/x-pack/metricbeat/module/aws/elb/_meta/data_alb.json b/x-pack/metricbeat/module/aws/elb/_meta/data_alb.json new file mode 100644 index 000000000000..0027f4254cf9 --- /dev/null +++ b/x-pack/metricbeat/module/aws/elb/_meta/data_alb.json @@ -0,0 +1,39 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "aws": { + "cloudwatch": { + "dimensions": { + "LoadBalancer": "app/filebeat-aws-elb-test-lb/aaa75bb026cc4c54" + }, + "namespace": "AWS/ApplicationELB" + }, + "metrics": { + "ConsumedLCUs": { + "avg": 0 + }, + "RequestCount": { + "sum": 0 + } + } + }, + "cloud": { + "account": { + "id": "627959692251", + "name": "elastic-test" + }, + "provider": "aws", + "region": "ap-southeast-1" + }, + "event": { + "dataset": "aws.elb", + "duration": 115000, + "module": "aws" + }, + "metricset": { + "name": "elb", + "period": 10000 + }, + "service": { + "type": "aws" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/elb/_meta/data_nlb.json b/x-pack/metricbeat/module/aws/elb/_meta/data_nlb.json new file mode 100644 index 000000000000..114851ebd5d2 --- /dev/null +++ b/x-pack/metricbeat/module/aws/elb/_meta/data_nlb.json @@ -0,0 +1,54 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "aws": { + "cloudwatch": { + "dimensions": { + "LoadBalancer": "net/filebeat-aws-elb-test-tcp-lb/d194184ba08f017f" + }, + "namespace": "AWS/NetworkELB" + }, + "metrics": { + "ActiveFlowCount": { + "avg": 0 + }, + "ConsumedLCUs": { + "avg": 0 + }, + "NewFlowCount": { + "sum": 0 + }, + "ProcessedBytes": { + "sum": 336 + }, + "TCP_Client_Reset_Count": { + "sum": 0 + }, + "TCP_ELB_Reset_Count": { + "sum": 0 + }, + "TCP_Target_Reset_Count": { + "sum": 0 + } + } + }, + "cloud": { + "account": { + "id": "627959692251", + "name": "elastic-test" + }, + "provider": "aws", + "region": "ap-southeast-1" + }, + "event": { + "dataset": "aws.elb", + "duration": 115000, + "module": "aws" + }, + "metricset": { + "name": "elb", + "period": 10000 + }, + "service": { + "type": "aws" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/elb/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/elb/_meta/docs.asciidoc index 2d27a8f8b1e9..3975a252e887 100644 --- a/x-pack/metricbeat/module/aws/elb/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/elb/_meta/docs.asciidoc @@ -17,7 +17,7 @@ iam:ListAccountAliases [float] === Dashboard -The aws elb metricset comes with a predefined dashboard. For example: +The aws elb metricset comes with a predefined dashboard for classic ELB. For example: image::./images/metricbeat-aws-elb-overview.png[] @@ -39,10 +39,12 @@ image::./images/metricbeat-aws-elb-overview.png[] [float] === Metrics -Elastic Load Balancing publishes data points to Amazon CloudWatch for your load -balancers and back-end instances. Please see more details for each metric in -https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-cloudwatch-metrics.html[elb-cloudwatch-metric]. +elb metricset collects Cloudwatch metrics from https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-cloudwatch-metrics.html[classic ELB], +https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-cloudwatch-metrics.html[application ELB] and +https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-cloudwatch-metrics.html[network ELB]. +[float] +==== Metrics for Classic ELB |=== |Metric Name|Statistic Method |BackendConnectionErrors | Sum @@ -63,3 +65,50 @@ https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-cloudwatch-m |EstimatedALBNewConnectionCount | Average |EstimatedProcessedBytes | Average |=== + +[float] +==== Metrics for Application ELB +|=== +|Metric Name|Statistic Method +|ActiveConnectionCount | Sum +|ClientTLSNegotiationErrorCount | Sum +|ConsumedLCUs | Average +|HTTP_Fixed_Response_Count | Sum +|HTTP_Redirect_Count | Sum +|HTTP_Redirect_Url_Limit_Exceeded_Count | Sum +|HTTPCode_ELB_3XX_Count | Sum +|HTTPCode_ELB_4XX_Count | Sum +|HTTPCode_ELB_5XX_Count | Sum +|HTTPCode_ELB_500_Count | Sum +|HTTPCode_ELB_502_Count | Sum +|HTTPCode_ELB_503_Count | Sum +|HTTPCode_ELB_504_Count | Sum +|IPv6ProcessedBytes | Sum +|IPv6RequestCount | Sum +|NewConnectionCount | Sum +|ProcessedBytes | Sum +|RejectedConnectionCount | Sum +|RequestCount | Sum +|RuleEvaluations | Sum +|=== + +[float] +==== Metrics for Network ELB +|=== +|Metric Name|Statistic Method +|ActiveFlowCount | Average +|ActiveFlowCount_TLS | Average +|ClientTLSNegotiationErrorCount | Sum +|ConsumedLCUs | Average +|HealthyHostCount | Maximum +|NewFlowCount | Sum +|NewFlowCount_TLS | Sum +|ProcessedBytes | Sum +|ProcessedBytes_TLS | Sum +|TargetTLSNegotiationErrorCount | Sum +|TCP_Client_Reset_Count | Sum +|TCP_ELB_Reset_Count | Sum +|TCP_Target_Reset_Count | Sum +|UnHealthyHostCount | Average +|EstimatedALBConsumedLCUs | Maximum +|=== diff --git a/x-pack/metricbeat/module/aws/elb/elb_integration_test.go b/x-pack/metricbeat/module/aws/elb/elb_integration_test.go index a7e6b2a4189c..38ea98d1cb22 100644 --- a/x-pack/metricbeat/module/aws/elb/elb_integration_test.go +++ b/x-pack/metricbeat/module/aws/elb/elb_integration_test.go @@ -7,18 +7,40 @@ package elb import ( + "fmt" "testing" + "github.com/elastic/beats/libbeat/common" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/x-pack/metricbeat/module/aws/mtest" ) func TestData(t *testing.T) { + namespaceIs := func(namespace string) func(e common.MapStr) bool { + return func(e common.MapStr) bool { + v, err := e.GetValue("aws.cloudwatch.namespace") + return err == nil && v == namespace + } + } + + dataFiles := []struct { + namespace string + path string + }{ + {"AWS/ELB", "./_meta/data.json"}, + {"AWS/ApplicationELB", "./_meta/data_alb.json"}, + {"AWS/NetworkELB", "./_meta/data_nlb.json"}, + } + config, info := mtest.GetConfigForTest("elb", "300s") if info != "" { t.Skip("Skipping TestData: " + info) } - metricSet := mbtest.NewFetcher(t, config) - metricSet.WriteEvents(t, "/") + for _, df := range dataFiles { + metricSet := mbtest.NewFetcher(t, config) + t.Run(fmt.Sprintf("namespace: %s", df.namespace), func(t *testing.T) { + metricSet.WriteEventsCond(t, df.path, namespaceIs(df.namespace)) + }) + } } diff --git a/x-pack/metricbeat/module/aws/elb/manifest.yml b/x-pack/metricbeat/module/aws/elb/manifest.yml index 319ea3f90615..211a42c63bdd 100644 --- a/x-pack/metricbeat/module/aws/elb/manifest.yml +++ b/x-pack/metricbeat/module/aws/elb/manifest.yml @@ -16,3 +16,31 @@ input: statistic: ["Average"] name: ["Latency", "EstimatedALBActiveConnectionCount", "EstimatedALBConsumedLCUs", "EstimatedALBNewConnectionCount", "EstimatedProcessedBytes"] tags.resource_type_filter: elasticloadbalancing + - namespace: AWS/ApplicationELB + statistic: ["Sum"] + name: ["ActiveConnectionCount", "ClientTLSNegotiationErrorCount", "HTTP_Fixed_Response_Count", + "HTTP_Redirect_Count", "HTTP_Redirect_Url_Limit_Exceeded_Count", + "HTTPCode_ELB_3XX_Count", "HTTPCode_ELB_4XX_Count", "HTTPCode_ELB_5XX_Count", + "HTTPCode_ELB_500_Count", "HTTPCode_ELB_502_Count", "HTTPCode_ELB_503_Count", + "HTTPCode_ELB_504_Count", "IPv6ProcessedBytes", "IPv6RequestCount", + "NewConnectionCount", "ProcessedBytes", "RejectedConnectionCount", + "RequestCount", "RuleEvaluations"] + tags.resource_type_filter: elasticloadbalancing + - namespace: AWS/ApplicationELB + statistic: ["Average"] + name: ["ConsumedLCUs"] + tags.resource_type_filter: elasticloadbalancing + - namespace: AWS/NetworkELB + statistic: ["Average"] + name: ["ActiveFlowCount", "ActiveFlowCount_TLS", "ConsumedLCUs"] + tags.resource_type_filter: elasticloadbalancing + - namespace: AWS/NetworkELB + statistic: ["Sum"] + name: ["ClientTLSNegotiationErrorCount", "NewFlowCount", "NewFlowCount_TLS", + "ProcessedBytes", "ProcessedBytes_TLS", "TargetTLSNegotiationErrorCount", + "TCP_Client_Reset_Count", "TCP_ELB_Reset_Count", "TCP_Target_Reset_Count"] + tags.resource_type_filter: elasticloadbalancing + - namespace: AWS/NetworkELB + statistic: ["Maximum"] + name: ["HealthyHostCount", "UnHealthyHostCount"] + tags.resource_type_filter: elasticloadbalancing