Remove metricIDNameMap and use ListMetrics to construct metric queries (

elastic#10627) * Remove metricNameIDMap and use ListMetrics to construct metric queries * Add utils.go to put common functions for all metricsets * Remove fields that dont need to apply schema * Remove s.Optional in schema and use s.FailOnRequired instead * Regenerate data,json * Move GetMetricDataResults into utils.go * Replace with report.error * Separating TestData from TestFetch * Move two functions only used by integration tests into a separate file * Move aws_testing.go under mtest
cwurm · Feb 19, 2019 · 589fd32 · 589fd32
1 parent 59061d4
commit 589fd32
Show file tree

Hide file tree

Showing 9 changed files with 611 additions and 413 deletions.
diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go
@@ -81,6 +81,11 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 	}
 
 	// Calculate duration based on period
+	if config.Period == "" {
+		err = errors.New("period is not set in AWS module config")
+		return nil, err
+	}
+
 	durationString, periodSec, err := convertPeriodToDuration(config.Period)
 	if err != nil {
 		return nil, err

diff --git a/x-pack/metricbeat/module/aws/ec2/_meta/data.json b/x-pack/metricbeat/module/aws/ec2/_meta/data.json
@@ -7,12 +7,12 @@
     "aws": {
         "ec2": {
             "cpu": {
-                "credit_balance": 169.089216,
-                "credit_usage": 0.002926,
+                "credit_balance": 576,
+                "credit_usage": 0.144723,
                 "surplus_credit_balance": 0,
                 "surplus_credits_charged": 0,
                 "total": {
-                    "pct": 0.033333333333322
+                    "pct": 1.366194313233248
                 }
             },
             "diskio": {
@@ -27,21 +27,21 @@
             },
             "instance": {
                 "core": {
-                    "count": 1
+                    "count": 2
                 },
                 "image": {
-                    "id": "ami-01e24be29428c15b2"
+                    "id": "ami-f920cd94"
                 },
                 "monitoring": {
                     "state": "disabled"
                 },
                 "private": {
-                    "dns_name": "ip-172-31-26-12.us-west-2.compute.internal",
-                    "ip": "172.31.26.12"
+                    "dns_name": "ip-10-0-0-148.ec2.internal",
+                    "ip": "10.0.0.148"
                 },
                 "public": {
-                    "dns_name": "ec2-34-217-213-210.us-west-2.compute.amazonaws.com",
-                    "ip": "34.217.213.210"
+                    "dns_name": "ec2-54-226-109-162.compute-1.amazonaws.com",
+                    "ip": "54.226.109.162"
                 },
                 "state": {
                     "code": 16,
@@ -51,12 +51,12 @@
             },
             "network": {
                 "in": {
-                    "bytes": 296.6,
-                    "packets": 3
+                    "bytes": 737000.4,
+                    "packets": 1361.2
                 },
                 "out": {
-                    "bytes": 269,
-                    "packets": 3.6
+                    "bytes": 227871.2,
+                    "packets": 1411.2
                 }
             },
             "status": {
@@ -67,15 +67,15 @@
         }
     },
     "cloud": {
-        "availability_zone": "us-west-2a",
+        "availability_zone": "us-east-1b",
         "instance": {
-            "id": "i-077bdaf7e5d81bba3"
+            "id": "i-77f84332"
         },
         "machine": {
-            "type": "t2.micro"
+            "type": "t2.medium"
         },
         "provider": "ec2",
-        "region": "us-west-2"
+        "region": "us-east-1"
     },
     "event": {
         "dataset": "aws.ec2",

diff --git a/x-pack/metricbeat/module/aws/ec2/data.go b/x-pack/metricbeat/module/aws/ec2/data.go
@@ -5,7 +5,6 @@
 package ec2
 
 import (
-	"github.com/elastic/beats/libbeat/common"
 	s "github.com/elastic/beats/libbeat/common/schema"
 	c "github.com/elastic/beats/libbeat/common/schema/mapstrstr"
 )
@@ -14,85 +13,37 @@ var (
 	schemaMetricSetFields = s.Schema{
 		"cpu": s.Object{
 			"total": s.Object{
-				"pct": c.Float("cpu.total.pct", s.Optional),
+				"pct": c.Float("CPUUtilization"),
 			},
-			"credit_usage":            c.Float("cpu.credit_usage", s.Optional),
-			"credit_balance":          c.Float("cpu.credit_balance", s.Optional),
-			"surplus_credit_balance":  c.Float("cpu.surplus_credit_balance", s.Optional),
-			"surplus_credits_charged": c.Float("cpu.surplus_credits_charged", s.Optional),
+			"credit_usage":            c.Float("CPUCreditUsage"),
+			"credit_balance":          c.Float("CPUCreditBalance"),
+			"surplus_credit_balance":  c.Float("CPUSurplusCreditBalance"),
+			"surplus_credits_charged": c.Float("CPUSurplusCreditsCharged"),
 		},
 		"diskio": s.Object{
 			"read": s.Object{
-				"bytes": c.Float("diskio.read.bytes", s.Optional),
-				"count": c.Float("diskio.read.count", s.Optional),
+				"bytes": c.Float("DiskReadBytes"),
+				"count": c.Float("DiskReadOps"),
 			},
 			"write": s.Object{
-				"bytes": c.Float("diskio.write.bytes", s.Optional),
-				"count": c.Float("diskio.write.count", s.Optional),
+				"bytes": c.Float("DiskWriteBytes"),
+				"count": c.Float("DiskWriteOps"),
 			},
 		},
 		"network": s.Object{
 			"in": s.Object{
-				"bytes":   c.Float("network.in.bytes", s.Optional),
-				"packets": c.Float("network.in.packets", s.Optional),
+				"bytes":   c.Float("NetworkIn"),
+				"packets": c.Float("NetworkPacketsIn"),
 			},
 			"out": s.Object{
-				"bytes":   c.Float("network.out.bytes", s.Optional),
-				"packets": c.Float("network.out.packets", s.Optional),
+				"bytes":   c.Float("NetworkOut"),
+				"packets": c.Float("NetworkPacketsOut"),
 			},
 		},
 		"status": s.Object{
-			"check_failed":          c.Int("status.check_failed", s.Optional),
-			"check_failed_instance": c.Int("status.check_failed_instance", s.Optional),
-			"check_failed_system":   c.Int("status.check_failed_system", s.Optional),
-		},
-		"instance": s.Object{
-			"image": s.Object{
-				"id": c.Str("instance.image.id", s.Optional),
-			},
-			"state": s.Object{
-				"name": c.Str("instance.state.name", s.Optional),
-				"code": c.Int("instance.state.code", s.Optional),
-			},
-			"monitoring": s.Object{
-				"state": c.Str("instance.monitoring.state", s.Optional),
-			},
-			"core": s.Object{
-				"count": c.Int("instance.core.count", s.Optional),
-			},
-			"threads_per_core": c.Int("instance.threads_per_core", s.Optional),
-			"public": s.Object{
-				"ip":       c.Str("instance.public.ip", s.Optional),
-				"dns_name": c.Str("instance.public.dns_name", s.Optional),
-			},
-			"private": s.Object{
-				"ip":       c.Str("instance.private.ip", s.Optional),
-				"dns_name": c.Str("instance.private.dns_name", s.Optional),
-			},
-		},
-	}
-)
-
-var (
-	schemaRootFields = s.Schema{
-		"service": s.Object{
-			"name": c.Str("service.name", s.Optional),
-		},
-		"cloud": s.Object{
-			"provider":          c.Str("cloud.provider", s.Optional),
-			"availability_zone": c.Str("cloud.availability_zone", s.Optional),
-			"region":            c.Str("cloud.region", s.Optional),
-			"instance": s.Object{
-				"id":   c.Str("cloud.instance.id", s.Optional),
-				"name": c.Str("cloud.instance.name", s.Optional),
-			},
-			"machine": s.Object{
-				"type": c.Str("cloud.machine.type", s.Optional),
-			},
+			"check_failed":          c.Int("StatusCheckFailed"),
+			"check_failed_instance": c.Int("StatusCheckFailed_Instance"),
+			"check_failed_system":   c.Int("StatusCheckFailed_System"),
 		},
 	}
 )
-
-func eventMapping(input map[string]interface{}, schema s.Schema) (common.MapStr, error) {
-	return schema.Apply(input)
-}