Cherry-pick #16234 to 7.x: Add database_account azure metricset (#16662)

* Add database_account azure metricset (#16234)

* add metricset

* work on database_Account

* fix

* work on tests

* mage fmt update

* work on manifest

* update json

* move timegrain

* uppercase

(cherry picked from commit 6816bb1)

* changelog

CHANGELOG.next.asciidoc

@@ -259,6 +259,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Reuse connections in SQL module. {pull}16001[16001]
 - Improve the `logstash` module (when `xpack.enabled` is set to `true`) to use the override `cluster_uuid` returned by Logstash APIs. {issue}15772[15772] {pull}15795[15795]
 - Add region parameter in googlecloud module. {issue}15780[15780] {pull}16203[16203]
+- Add database_account azure metricset. {issue}15758[15758]
 - Add support for Dropwizard metrics 4.1. {pull}16332[16332]
 - Add support for NATS 2.1. {pull}16317[16317]
 - Improve the `haproxy` module to support metrics exposed via HTTPS. {issue}14579[14579] {pull}16333[16333]

metricbeat/docs/fields.asciidoc

@@ -3103,6 +3103,16 @@ azure module
 
 
 
+*`azure.timegrain`*::
++
+--
+The Azure metric timegrain
+
+
+type: keyword
+
+--
+
 [float]
 === resource
 
@@ -3196,6 +3206,16 @@ type: object
 compute_vm_scaleset
 
 
+type: object
+
+--
+
+*`azure.database_account.*.*`*::
++
+--
+database account
+
+
 type: object
 
 --

metricbeat/docs/modules/azure.asciidoc

@@ -149,6 +149,17 @@ metricbeat.modules:
   client_secret: '${AZURE_CLIENT_SECRET:""}'
   tenant_id: '${AZURE_TENANT_ID:""}'
   subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
+- module: azure
+  metricsets:
+  - database_account
+  enabled: true
+  period: 300s
+  client_id: '${AZURE_CLIENT_ID:""}'
+  client_secret: '${AZURE_CLIENT_SECRET:""}'
+  tenant_id: '${AZURE_TENANT_ID:""}'
+  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
 ----
 
 [float]
@@ -160,6 +171,8 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-azure-compute_vm_scaleset,compute_vm_scaleset>>
 
+* <<metricbeat-metricset-azure-database_account,database_account>>
+
 * <<metricbeat-metricset-azure-monitor,monitor>>
 
 * <<metricbeat-metricset-azure-storage,storage>>
@@ -168,6 +181,8 @@ include::azure/compute_vm.asciidoc[]
 
 include::azure/compute_vm_scaleset.asciidoc[]
 
+include::azure/database_account.asciidoc[]
+
 include::azure/monitor.asciidoc[]
 
 include::azure/storage.asciidoc[]

metricbeat/docs/modules/azure/database_account.asciidoc

@@ -0,0 +1,24 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-azure-database_account]]
+=== azure database_account metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/azure/database_account/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-azure,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/azure/database_account/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -31,8 +31,9 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-aws-usage,usage>> beta[]  
 |<<metricbeat-metricset-aws-vpc,vpc>> beta[]  
 |<<metricbeat-module-azure,azure>>  beta[]   |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.4+| .4+|  |<<metricbeat-metricset-azure-compute_vm,compute_vm>> beta[]  
+.5+| .5+|  |<<metricbeat-metricset-azure-compute_vm,compute_vm>> beta[]  
 |<<metricbeat-metricset-azure-compute_vm_scaleset,compute_vm_scaleset>> beta[]  
+|<<metricbeat-metricset-azure-database_account,database_account>> beta[]  
 |<<metricbeat-metricset-azure-monitor,monitor>> beta[]  
 |<<metricbeat-metricset-azure-storage,storage>> beta[]  
 |<<metricbeat-module-beat,Beat>>     |image:./images/icon-no.png[No prebuilt dashboards]    |  

x-pack/metricbeat/metricbeat.reference.yml

@@ -265,6 +265,17 @@ metricbeat.modules:
   tenant_id: '${AZURE_TENANT_ID:""}'
   subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
 
+- module: azure
+  metricsets:
+  - database_account
+  enabled: true
+  period: 300s
+  client_id: '${AZURE_CLIENT_ID:""}'
+  client_secret: '${AZURE_CLIENT_SECRET:""}'
+  tenant_id: '${AZURE_TENANT_ID:""}'
+  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
+
 #--------------------------------- Beat Module ---------------------------------
 - module: beat
   metricsets:

x-pack/metricbeat/module/azure/_meta/config.reference.yml

@@ -42,3 +42,14 @@
   client_secret: '${AZURE_CLIENT_SECRET:""}'
   tenant_id: '${AZURE_TENANT_ID:""}'
   subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
+- module: azure
+  metricsets:
+  - database_account
+  enabled: true
+  period: 300s
+  client_id: '${AZURE_CLIENT_ID:""}'
+  client_secret: '${AZURE_CLIENT_SECRET:""}'
+  tenant_id: '${AZURE_TENANT_ID:""}'
+  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+

x-pack/metricbeat/module/azure/_meta/config.yml

@@ -46,3 +46,14 @@
 #  tenant_id: '${AZURE_TENANT_ID:""}'
 #  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
 #  refresh_list_interval: 600s
+
+#- module: azure
+#  metricsets:
+#  - database_account
+#  enabled: true
+#  period: 300s
+#  client_id: '${AZURE_CLIENT_ID:""}'
+#  client_secret: '${AZURE_CLIENT_SECRET:""}'
+#  tenant_id: '${AZURE_TENANT_ID:""}'
+#  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+#  refresh_list_interval: 600s

x-pack/metricbeat/module/azure/_meta/fields.yml

@@ -8,6 +8,10 @@
       type: group
       description: >
       fields:
+        - name: timegrain
+          type: keyword
+          description: >
+            The Azure metric timegrain
         - name: resource
           type: group
           description: >

x-pack/metricbeat/module/azure/azure.go

@@ -5,6 +5,7 @@
 package azure
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/elastic/beats/libbeat/common/cfgwarn"
@@ -23,6 +24,7 @@ type Config struct {
 	Period              time.Duration    `config:"period" validate:"nonzero,required"`
 	Resources           []ResourceConfig `config:"resources"`
 	RefreshListInterval time.Duration    `config:"refresh_list_interval"`
+	DefaultResourceType string           `config:"default_resource_type"`
 }
 
 // ResourceConfig contains resource and metric list specific configuration.
@@ -101,7 +103,19 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 				return nil, errors.Errorf("error initializing the monitor client: module azure - %s metricset. No queries allowed, please select one of the allowed options", metricsetName)
 			}
 		}
-
+		// check for lightweight resources if no groups or ids have been entered, if not a new resource is created to check the entire subscription
+		var resources []ResourceConfig
+		for _, resource := range config.Resources {
+			if len(resource.Group) != 0 || len(resource.ID) != 0 {
+				resources = append(resources, resource)
+			}
+		}
+		if len(resources) == 0 && len(config.Resources) != 0 {
+			res := config.Resources[0]
+			res.Query = fmt.Sprintf("resourceType eq '%s'", config.DefaultResourceType)
+			resources = append(resources, res)
+		}
+		config.Resources = resources
 	}
 	// instantiate monitor client
 	monitorClient, err := NewClient(config)

x-pack/metricbeat/module/azure/client.go

@@ -107,13 +107,16 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) []
 			}
 			filter = strings.Join(filterList, " AND ")
 		}
-		resp, err := client.AzureMonitorService.GetMetricValues(metric.Resource.SubID, metric.Namespace, metric.TimeGrain, timespan, metric.Names,
+		resp, timegrain, err := client.AzureMonitorService.GetMetricValues(metric.Resource.SubID, metric.Namespace, metric.TimeGrain, timespan, metric.Names,
 			metric.Aggregations, filter)
 		if err != nil {
 			err = errors.Wrapf(err, "error while listing metric values by resource ID %s and namespace  %s", metric.Resource.SubID, metric.Namespace)
 			client.Log.Error(err)
 			report.Error(err)
 		} else {
+			if metric.TimeGrain == "" {
+				metric.TimeGrain = timegrain
+			}
 			for i, currentMetric := range client.Resources.Metrics {
 				if matchMetrics(currentMetric, metric) {
 					current := mapMetricValues(resp, currentMetric.Values, endTime.Truncate(time.Minute).Add(interval*(-1)), endTime.Truncate(time.Minute))

x-pack/metricbeat/module/azure/client_test.go

@@ -79,7 +79,7 @@ func TestGetMetricValues(t *testing.T) {
 		}
 		m := &MockService{}
 		m.On("GetMetricValues", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Once().
-			Return([]insights.Metric{}, errors.New("invalid parameters or no metrics found"))
+			Return([]insights.Metric{}, "", errors.New("invalid parameters or no metrics found"))
 		client.AzureMonitorService = m
 		mr := MockReporterV2{}
 		mr.On("Error", mock.Anything).Return(true)
@@ -101,7 +101,7 @@ func TestGetMetricValues(t *testing.T) {
 		}
 		m := &MockService{}
 		m.On("GetMetricValues", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
-			Return([]insights.Metric{}, errors.New("invalid parameters or no metrics found"))
+			Return([]insights.Metric{}, "", errors.New("invalid parameters or no metrics found"))
 		client.AzureMonitorService = m
 		mr := MockReporterV2{}
 		mr.On("Error", mock.Anything).Return(true)

x-pack/metricbeat/module/azure/compute_vm/_meta/data.json

@@ -1,6 +1,7 @@
 {
     "@timestamp":"2019-09-20T11:02:00.000Z",
     "azure":{
+        "timegrain" : "PT5M",
         "resource":{
             "name":"obslinux",
             "type":"Microsoft.Compute/virtualMachines",

x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json

@@ -1,6 +1,7 @@
 {
     "@timestamp" : "2019-09-20T11:13:00.000Z",
     "azure" : {
+        "timegrain" : "PT5M",
         "resource" : {
             "name" : "winvmss",
             "type" : "Microsoft.Compute/virtualMachineScaleSets",

x-pack/metricbeat/module/azure/data.go

@@ -122,6 +122,7 @@ func managePropertyName(metric string) string {
 func createEvent(timestamp time.Time, metric Metric, metricValues []MetricValue) (mb.Event, common.MapStr) {
 	event := mb.Event{
 		ModuleFields: common.MapStr{
+			"timegrain": metric.TimeGrain,
 			"resource": common.MapStr{
 				"name":  metric.Resource.Name,
 				"type":  metric.Resource.Type,

x-pack/metricbeat/module/azure/database_account/_meta/data.json

@@ -0,0 +1,40 @@
+{
+    "@timestamp" : "2020-02-25T08:53:00.000Z",
+    "cloud" : {
+        "provider" : "azure",
+        "region" : "westeurope"
+    },
+    "event" : {
+        "module" : "azure",
+        "duration" : 4877063600,
+        "dataset" : "azure.database_account"
+    },
+    "metricset" : {
+        "name" : "database_account",
+        "period" : 300000
+    },
+    "azure" : {
+        "timegrain" : "PT5M",
+        "dimensions" : {
+            "databasename" : "testdb"
+        },
+        "database_account" : {
+            "provisionedthroughput" : {
+                "max" : 400
+            }
+        },
+        "resource" : {
+            "group" : "obs-test",
+            "tags" : {
+                "defaultExperience" : "Core (SQL)"
+            },
+            "name" : "obsaccount",
+            "type" : "Microsoft.DocumentDb/databaseAccounts"
+        },
+        "subscription_id" : "123456-qwer-1234-5678-12345678",
+        "namespace" : "Microsoft.DocumentDb/databaseAccounts"
+    },
+    "service" : {
+        "type" : "azure"
+    }
+}

x-pack/metricbeat/module/azure/database_account/_meta/docs.asciidoc

@@ -0,0 +1,19 @@
+This is the database_account metricset of the module azure.
+
+This metricset allows users to retrieve all metrics from specified database accounts.
+
+include::../../_meta/shared-azure.asciidoc[]
+
+[float]
+==== Config options to identify resources
+
+`resource_id`:: (_[]string_) The fully qualified ID's of the resource, including the resource name and resource type. Has the format /subscriptions/{guid}/resourceGroups/{resource-group-name}/providers/{resource-provider-namespace}/{resource-type}/{resource-name}.
+  Should return a list of resources.
+
+`resource_group`:: (_[]string_) This option should return a list of databases we want to apply our metric configuration options on.
+
+If none of the options are entered then we will select all databases from the entire subscription
+For each metric the primary aggregation assigned will be retrieved.
+A default non configurable timegrain of 5 min is set so users are advised to configure an interval of 300s or  a multiply of it.
+
+

x-pack/metricbeat/module/azure/database_account/_meta/fields.yml

@@ -0,0 +1,7 @@
+- name: database_account.*.*
+  release: beta
+  type: object
+  object_type: float
+  object_type_mapping_type: "*"
+  description: >
+    database account

x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go

@@ -0,0 +1,51 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package database_account
+
+import (
+	"errors"
+	"os"
+	"testing"
+
+	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+)
+
+func TestData(t *testing.T) {
+	config, err := getConfig()
+	if err != nil {
+		t.Skip("Skipping TestData: " + err.Error())
+	}
+
+	metricSet := mbtest.NewFetcher(t, config)
+	metricSet.WriteEvents(t, "/")
+}
+
+func getConfig() (map[string]interface{}, error) {
+	clientId, ok := os.LookupEnv("AZURE_CLIENT_ID")
+	if !ok {
+		return nil, errors.New("missing AZURE_CLIENT_ID key")
+	}
+	clientSecret, ok := os.LookupEnv("AZURE_CLIENT_SECRET")
+	if !ok {
+		return nil, errors.New("missing AZURE_CLIENT_SECRET key")
+	}
+	tenantId, ok := os.LookupEnv("AZURE_TENANT_ID")
+	if !ok {
+		return nil, errors.New("missing AZURE_TENANT_ID key")
+	}
+	subscriptionId, ok := os.LookupEnv("AZURE_SUBSCRIPTION_ID")
+	if !ok {
+		return nil, errors.New("missing AZURE_SUBSCRIPTION_ID key")
+	}
+	config := map[string]interface{}{
+		"module":          "azure",
+		"metricsets":      []string{"database_account"},
+		"client_id":       clientId,
+		"client_secret":   clientSecret,
+		"tenant_id":       tenantId,
+		"subscription_id": subscriptionId,
+	}
+	return config, nil
+}