From 7888276466d1070e63301fe177b9be85409d5088 Mon Sep 17 00:00:00 2001 From: Mariana Dima Date: Thu, 27 Feb 2020 10:52:18 +0100 Subject: [PATCH] Add database_account azure metricset (#16234) * add metricset * work on database_Account * fix * work on tests * mage fmt update * work on manifest * update json * move timegrain * uppercase (cherry picked from commit 6816bb1065913b5ad000c0a9bf9c216d91efb3b6) --- CHANGELOG.next.asciidoc | 4 ++ metricbeat/docs/fields.asciidoc | 20 ++++++++ metricbeat/docs/modules/azure.asciidoc | 15 ++++++ .../modules/azure/database_account.asciidoc | 24 +++++++++ metricbeat/docs/modules_list.asciidoc | 3 +- x-pack/metricbeat/metricbeat.reference.yml | 11 ++++ .../module/azure/_meta/config.reference.yml | 11 ++++ .../metricbeat/module/azure/_meta/config.yml | 11 ++++ .../metricbeat/module/azure/_meta/fields.yml | 4 ++ x-pack/metricbeat/module/azure/azure.go | 16 +++++- x-pack/metricbeat/module/azure/client.go | 5 +- x-pack/metricbeat/module/azure/client_test.go | 4 +- .../module/azure/compute_vm/_meta/data.json | 1 + .../azure/compute_vm_scaleset/_meta/data.json | 1 + x-pack/metricbeat/module/azure/data.go | 1 + .../azure/database_account/_meta/data.json | 40 +++++++++++++++ .../database_account/_meta/docs.asciidoc | 19 +++++++ .../azure/database_account/_meta/fields.yml | 7 +++ .../database_account_integration_test.go | 51 +++++++++++++++++++ .../database_account/database_account_test.go | 17 +++++++ .../azure/database_account/manifest.yml | 47 +++++++++++++++++ x-pack/metricbeat/module/azure/fields.go | 2 +- .../metricbeat/module/azure/mock_service.go | 4 +- x-pack/metricbeat/module/azure/module.yml | 3 ++ .../module/azure/monitor/_meta/data.json | 1 + .../module/azure/monitor_service.go | 8 +-- .../module/azure/service_interface.go | 2 +- .../module/azure/storage/_meta/data.json | 1 + .../metricbeat/modules.d/azure.yml.disabled | 11 ++++ 29 files changed, 332 insertions(+), 12 deletions(-) create mode 100644 metricbeat/docs/modules/azure/database_account.asciidoc create mode 100644 x-pack/metricbeat/module/azure/database_account/_meta/data.json create mode 100644 x-pack/metricbeat/module/azure/database_account/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/azure/database_account/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go create mode 100644 x-pack/metricbeat/module/azure/database_account/database_account_test.go create mode 100644 x-pack/metricbeat/module/azure/database_account/manifest.yml create mode 100644 x-pack/metricbeat/module/azure/module.yml diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index db040cc2d1de..591d2fae4168 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -256,6 +256,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Reuse connections in SQL module. {pull}16001[16001] - Improve the `logstash` module (when `xpack.enabled` is set to `true`) to use the override `cluster_uuid` returned by Logstash APIs. {issue}15772[15772] {pull}15795[15795] - Add region parameter in googlecloud module. {issue}15780[15780] {pull}16203[16203] +- Add kubernetes storage class support via kube-state-metrics. {pull}16145[16145] +- Add database_account azure metricset. {issue}15758[15758] +- Add support for NATS 2.1. {pull}16317[16317] +- Add Load Balancing metricset to GCP {pull}15559[15559] - Add support for Dropwizard metrics 4.1. {pull}16332[16332] - Add support for NATS 2.1. {pull}16317[16317] - Improve the `haproxy` module to support metrics exposed via HTTPS. {issue}14579[14579] {pull}16333[16333] diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 61d7d5683161..33c98e0dfa87 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3097,6 +3097,16 @@ azure module +*`azure.timegrain`*:: ++ +-- +The Azure metric timegrain + + +type: keyword + +-- + [float] === resource @@ -3190,6 +3200,16 @@ type: object compute_vm_scaleset +type: object + +-- + +*`azure.database_account.*.*`*:: ++ +-- +database account + + type: object -- diff --git a/metricbeat/docs/modules/azure.asciidoc b/metricbeat/docs/modules/azure.asciidoc index b442bf294dd0..ed5a266e3d8c 100644 --- a/metricbeat/docs/modules/azure.asciidoc +++ b/metricbeat/docs/modules/azure.asciidoc @@ -149,6 +149,17 @@ metricbeat.modules: client_secret: '${AZURE_CLIENT_SECRET:""}' tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - database_account + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + ---- [float] @@ -160,6 +171,8 @@ The following metricsets are available: * <> +* <> + * <> * <> @@ -168,6 +181,8 @@ include::azure/compute_vm.asciidoc[] include::azure/compute_vm_scaleset.asciidoc[] +include::azure/database_account.asciidoc[] + include::azure/monitor.asciidoc[] include::azure/storage.asciidoc[] diff --git a/metricbeat/docs/modules/azure/database_account.asciidoc b/metricbeat/docs/modules/azure/database_account.asciidoc new file mode 100644 index 000000000000..1d08d27a335d --- /dev/null +++ b/metricbeat/docs/modules/azure/database_account.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-azure-database_account]] +=== azure database_account metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/azure/database_account/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/azure/database_account/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 53abbe84bbee..ff2ca2c93aab 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -30,8 +30,9 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> beta[] |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.4+| .4+| |<> beta[] +.5+| .5+| |<> beta[] |<> beta[] +|<> beta[] |<> beta[] |<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index fc65c6156688..77d2c996c796 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -265,6 +265,17 @@ metricbeat.modules: tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +- module: azure + metricsets: + - database_account + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + + #--------------------------------- Beat Module --------------------------------- - module: beat metricsets: diff --git a/x-pack/metricbeat/module/azure/_meta/config.reference.yml b/x-pack/metricbeat/module/azure/_meta/config.reference.yml index 8b3a6e64ec6e..7dfd3068070f 100644 --- a/x-pack/metricbeat/module/azure/_meta/config.reference.yml +++ b/x-pack/metricbeat/module/azure/_meta/config.reference.yml @@ -42,3 +42,14 @@ client_secret: '${AZURE_CLIENT_SECRET:""}' tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - database_account + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + diff --git a/x-pack/metricbeat/module/azure/_meta/config.yml b/x-pack/metricbeat/module/azure/_meta/config.yml index 009159d21b34..525e421369fd 100644 --- a/x-pack/metricbeat/module/azure/_meta/config.yml +++ b/x-pack/metricbeat/module/azure/_meta/config.yml @@ -46,3 +46,14 @@ # tenant_id: '${AZURE_TENANT_ID:""}' # subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' # refresh_list_interval: 600s + +#- module: azure +# metricsets: +# - database_account +# enabled: true +# period: 300s +# client_id: '${AZURE_CLIENT_ID:""}' +# client_secret: '${AZURE_CLIENT_SECRET:""}' +# tenant_id: '${AZURE_TENANT_ID:""}' +# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +# refresh_list_interval: 600s diff --git a/x-pack/metricbeat/module/azure/_meta/fields.yml b/x-pack/metricbeat/module/azure/_meta/fields.yml index 427f33b9c995..4d51e8240876 100644 --- a/x-pack/metricbeat/module/azure/_meta/fields.yml +++ b/x-pack/metricbeat/module/azure/_meta/fields.yml @@ -8,6 +8,10 @@ type: group description: > fields: + - name: timegrain + type: keyword + description: > + The Azure metric timegrain - name: resource type: group description: > diff --git a/x-pack/metricbeat/module/azure/azure.go b/x-pack/metricbeat/module/azure/azure.go index 2da534d7372d..1d08b60951f1 100644 --- a/x-pack/metricbeat/module/azure/azure.go +++ b/x-pack/metricbeat/module/azure/azure.go @@ -5,6 +5,7 @@ package azure import ( + "fmt" "time" "github.com/elastic/beats/libbeat/common/cfgwarn" @@ -23,6 +24,7 @@ type Config struct { Period time.Duration `config:"period" validate:"nonzero,required"` Resources []ResourceConfig `config:"resources"` RefreshListInterval time.Duration `config:"refresh_list_interval"` + DefaultResourceType string `config:"default_resource_type"` } // ResourceConfig contains resource and metric list specific configuration. @@ -101,7 +103,19 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { return nil, errors.Errorf("error initializing the monitor client: module azure - %s metricset. No queries allowed, please select one of the allowed options", metricsetName) } } - + // check for lightweight resources if no groups or ids have been entered, if not a new resource is created to check the entire subscription + var resources []ResourceConfig + for _, resource := range config.Resources { + if len(resource.Group) != 0 || len(resource.ID) != 0 { + resources = append(resources, resource) + } + } + if len(resources) == 0 && len(config.Resources) != 0 { + res := config.Resources[0] + res.Query = fmt.Sprintf("resourceType eq '%s'", config.DefaultResourceType) + resources = append(resources, res) + } + config.Resources = resources } // instantiate monitor client monitorClient, err := NewClient(config) diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index b200daa3c772..5a3871f1046b 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -107,13 +107,16 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] } filter = strings.Join(filterList, " AND ") } - resp, err := client.AzureMonitorService.GetMetricValues(metric.Resource.SubID, metric.Namespace, metric.TimeGrain, timespan, metric.Names, + resp, timegrain, err := client.AzureMonitorService.GetMetricValues(metric.Resource.SubID, metric.Namespace, metric.TimeGrain, timespan, metric.Names, metric.Aggregations, filter) if err != nil { err = errors.Wrapf(err, "error while listing metric values by resource ID %s and namespace %s", metric.Resource.SubID, metric.Namespace) client.Log.Error(err) report.Error(err) } else { + if metric.TimeGrain == "" { + metric.TimeGrain = timegrain + } for i, currentMetric := range client.Resources.Metrics { if matchMetrics(currentMetric, metric) { current := mapMetricValues(resp, currentMetric.Values, endTime.Truncate(time.Minute).Add(interval*(-1)), endTime.Truncate(time.Minute)) diff --git a/x-pack/metricbeat/module/azure/client_test.go b/x-pack/metricbeat/module/azure/client_test.go index f79f9728df88..559b4efa7eb7 100644 --- a/x-pack/metricbeat/module/azure/client_test.go +++ b/x-pack/metricbeat/module/azure/client_test.go @@ -79,7 +79,7 @@ func TestGetMetricValues(t *testing.T) { } m := &MockService{} m.On("GetMetricValues", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Once(). - Return([]insights.Metric{}, errors.New("invalid parameters or no metrics found")) + Return([]insights.Metric{}, "", errors.New("invalid parameters or no metrics found")) client.AzureMonitorService = m mr := MockReporterV2{} mr.On("Error", mock.Anything).Return(true) @@ -101,7 +101,7 @@ func TestGetMetricValues(t *testing.T) { } m := &MockService{} m.On("GetMetricValues", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return([]insights.Metric{}, errors.New("invalid parameters or no metrics found")) + Return([]insights.Metric{}, "", errors.New("invalid parameters or no metrics found")) client.AzureMonitorService = m mr := MockReporterV2{} mr.On("Error", mock.Anything).Return(true) diff --git a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json index 2751b9327db4..f0692fae3b61 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json @@ -1,6 +1,7 @@ { "@timestamp":"2019-09-20T11:02:00.000Z", "azure":{ + "timegrain" : "PT5M", "resource":{ "name":"obslinux", "type":"Microsoft.Compute/virtualMachines", diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json index f156737e350a..ac7687bd5394 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json @@ -1,6 +1,7 @@ { "@timestamp" : "2019-09-20T11:13:00.000Z", "azure" : { + "timegrain" : "PT5M", "resource" : { "name" : "winvmss", "type" : "Microsoft.Compute/virtualMachineScaleSets", diff --git a/x-pack/metricbeat/module/azure/data.go b/x-pack/metricbeat/module/azure/data.go index 5b5ebc04eed9..2a2e1b41bb8b 100644 --- a/x-pack/metricbeat/module/azure/data.go +++ b/x-pack/metricbeat/module/azure/data.go @@ -122,6 +122,7 @@ func managePropertyName(metric string) string { func createEvent(timestamp time.Time, metric Metric, metricValues []MetricValue) (mb.Event, common.MapStr) { event := mb.Event{ ModuleFields: common.MapStr{ + "timegrain": metric.TimeGrain, "resource": common.MapStr{ "name": metric.Resource.Name, "type": metric.Resource.Type, diff --git a/x-pack/metricbeat/module/azure/database_account/_meta/data.json b/x-pack/metricbeat/module/azure/database_account/_meta/data.json new file mode 100644 index 000000000000..6749feeab713 --- /dev/null +++ b/x-pack/metricbeat/module/azure/database_account/_meta/data.json @@ -0,0 +1,40 @@ +{ + "@timestamp" : "2020-02-25T08:53:00.000Z", + "cloud" : { + "provider" : "azure", + "region" : "westeurope" + }, + "event" : { + "module" : "azure", + "duration" : 4877063600, + "dataset" : "azure.database_account" + }, + "metricset" : { + "name" : "database_account", + "period" : 300000 + }, + "azure" : { + "timegrain" : "PT5M", + "dimensions" : { + "databasename" : "testdb" + }, + "database_account" : { + "provisionedthroughput" : { + "max" : 400 + } + }, + "resource" : { + "group" : "obs-test", + "tags" : { + "defaultExperience" : "Core (SQL)" + }, + "name" : "obsaccount", + "type" : "Microsoft.DocumentDb/databaseAccounts" + }, + "subscription_id" : "123456-qwer-1234-5678-12345678", + "namespace" : "Microsoft.DocumentDb/databaseAccounts" + }, + "service" : { + "type" : "azure" + } +} diff --git a/x-pack/metricbeat/module/azure/database_account/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/database_account/_meta/docs.asciidoc new file mode 100644 index 000000000000..ef085175dac2 --- /dev/null +++ b/x-pack/metricbeat/module/azure/database_account/_meta/docs.asciidoc @@ -0,0 +1,19 @@ +This is the database_account metricset of the module azure. + +This metricset allows users to retrieve all metrics from specified database accounts. + +include::../../_meta/shared-azure.asciidoc[] + +[float] +==== Config options to identify resources + +`resource_id`:: (_[]string_) The fully qualified ID's of the resource, including the resource name and resource type. Has the format /subscriptions/{guid}/resourceGroups/{resource-group-name}/providers/{resource-provider-namespace}/{resource-type}/{resource-name}. + Should return a list of resources. + +`resource_group`:: (_[]string_) This option should return a list of databases we want to apply our metric configuration options on. + +If none of the options are entered then we will select all databases from the entire subscription +For each metric the primary aggregation assigned will be retrieved. +A default non configurable timegrain of 5 min is set so users are advised to configure an interval of 300s or a multiply of it. + + diff --git a/x-pack/metricbeat/module/azure/database_account/_meta/fields.yml b/x-pack/metricbeat/module/azure/database_account/_meta/fields.yml new file mode 100644 index 000000000000..60f53a8fa589 --- /dev/null +++ b/x-pack/metricbeat/module/azure/database_account/_meta/fields.yml @@ -0,0 +1,7 @@ +- name: database_account.*.* + release: beta + type: object + object_type: float + object_type_mapping_type: "*" + description: > + database account diff --git a/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go b/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go new file mode 100644 index 000000000000..ff1600607079 --- /dev/null +++ b/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go @@ -0,0 +1,51 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package database_account + +import ( + "errors" + "os" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + config, err := getConfig() + if err != nil { + t.Skip("Skipping TestData: " + err.Error()) + } + + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} + +func getConfig() (map[string]interface{}, error) { + clientId, ok := os.LookupEnv("AZURE_CLIENT_ID") + if !ok { + return nil, errors.New("missing AZURE_CLIENT_ID key") + } + clientSecret, ok := os.LookupEnv("AZURE_CLIENT_SECRET") + if !ok { + return nil, errors.New("missing AZURE_CLIENT_SECRET key") + } + tenantId, ok := os.LookupEnv("AZURE_TENANT_ID") + if !ok { + return nil, errors.New("missing AZURE_TENANT_ID key") + } + subscriptionId, ok := os.LookupEnv("AZURE_SUBSCRIPTION_ID") + if !ok { + return nil, errors.New("missing AZURE_SUBSCRIPTION_ID key") + } + config := map[string]interface{}{ + "module": "azure", + "metricsets": []string{"database_account"}, + "client_id": clientId, + "client_secret": clientSecret, + "tenant_id": tenantId, + "subscription_id": subscriptionId, + } + return config, nil +} diff --git a/x-pack/metricbeat/module/azure/database_account/database_account_test.go b/x-pack/metricbeat/module/azure/database_account/database_account_test.go new file mode 100644 index 000000000000..8cf7decd64e0 --- /dev/null +++ b/x-pack/metricbeat/module/azure/database_account/database_account_test.go @@ -0,0 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package database_account + +import ( + "os" + + "github.com/elastic/beats/metricbeat/mb" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} diff --git a/x-pack/metricbeat/module/azure/database_account/manifest.yml b/x-pack/metricbeat/module/azure/database_account/manifest.yml new file mode 100644 index 000000000000..d714fccb0896 --- /dev/null +++ b/x-pack/metricbeat/module/azure/database_account/manifest.yml @@ -0,0 +1,47 @@ +default: true +input: + module: azure + metricset: monitor + defaults: + default_resource_type: "Microsoft.DocumentDb/databaseAccounts" + resources: + - resource_group: "" + resource_type: "Microsoft.DocumentDb/databaseAccounts" + metrics: + - name: ["AddRegion", "RemoveRegion", "UpdateAccountReplicationSettings", "UpdateAccountNetworkSettings", "UpdateAccountKeys", "ServiceAvailability", "ReplicationLatency", + "RegionFailover", "DeleteAccount", "CreateAccount", "CassandraConnectionClosures"] + namespace: "Microsoft.DocumentDb/databaseAccounts" + - name: ["AvailableStorage", "DataUsage","DocumentCount", "DocumentQuota", "IndexUsage", "MetadataRequests", + "MongoRequestCharge", "MongoRequests", "MongoRequestsCount", "MongoRequestsDelete","MongoRequestsDelete", "MongoRequestsQuery", "MongoRequestsUpdate", + "ProvisionedThroughput", "TotalRequestUnits", "TotalRequests"] + namespace: "Microsoft.DocumentDb/databaseAccounts" + timegrain: "PT5M" + dimensions: + - name: "DatabaseName" + value: "*" + - name: ["CassandraRequestCharges", "CassandraRequests"] + namespace: "Microsoft.DocumentDb/databaseAccounts" + timegrain: "PT1M" + dimensions: + - name: "DatabaseName" + value: "*" + - resource_id: "" + metrics: + - name: ["AddRegion", "RemoveRegion", "UpdateAccountReplicationSettings", "UpdateAccountNetworkSettings", "UpdateAccountKeys", "ServiceAvailability", "ReplicationLatency", + "RegionFailover", "DeleteAccount", "CreateAccount", "CassandraConnectionClosures"] + namespace: "Microsoft.DocumentDb/databaseAccounts" + - name: ["AvailableStorage", "DataUsage","DocumentCount", "DocumentQuota", "IndexUsage", "MetadataRequests", + "MongoRequestCharge", "MongoRequests", "MongoRequestsCount", "MongoRequestsDelete","MongoRequestsDelete", "MongoRequestsQuery", "MongoRequestsUpdate", + "ProvisionedThroughput", "TotalRequestUnits", "TotalRequests"] + namespace: "Microsoft.DocumentDb/databaseAccounts" + timegrain: "PT5M" + dimensions: + - name: "DatabaseName" + value: "*" + - name: ["CassandraRequestCharges", "CassandraRequests"] + namespace: "Microsoft.DocumentDb/databaseAccounts" + timegrain: "PT1M" + dimensions: + - name: "DatabaseName" + value: "*" + diff --git a/x-pack/metricbeat/module/azure/fields.go b/x-pack/metricbeat/module/azure/fields.go index 8c678616d180..735ee8e10206 100644 --- a/x-pack/metricbeat/module/azure/fields.go +++ b/x-pack/metricbeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzUVDtPwzAQ3vMrThkrtT8gAxISCwMbe+Tal2AaP+QHqPx65KR1nXcLHcCDpfjO3+Pu4i0c8FgA+fIGMwDHXYMF5O13ngEwtNRw7biSBTxkANDlglDMN+GKwQaJxQL26EgGUHFsmC3a1C1IIvACH5Y7aiygNsrr08kERx8mhTJolTcUY2AKcRa1W69vGHHAaqS84siSlCF5KiDsvcBZwAGPn8qwQWxBxllKQARVgUtkTVIHnvtSh+tXUQ+rewfu2IExdjRMarvbTNKq/TtSNwh1h+WSsCSlFERrLutTfr7JbzPx2P4I0UYrdjSvYbeaTAzsWN/KyEYosNggdcnIntms30eIkrPfc6aA8Pw0ImRcoLRcyX6fZnq00p9re7OgueuJQGc4TcWNhFMltHdYfojdpid9+J7dYqlqFJkJ/tTQReeChdJS0qBF91+8RMExPnInlOROmdV3fs7jgo4x9NKD3w2THZR2oYyrpbylnCtWwnrpBIJB541EtpsvqnXKkBr/9picRAKhVHnpsu8AAAD//wCRS6g=" + return "eJzUVb2OnDAQ7vcpRluudPsAFJEipUmRLj0yZiDO4R/Z40Sbp4+8gM9gfpbkij0XSHjM9zPzAS/wircC2B9v8QRAgjos4Hy/P58AanTcCkNCqwI+nQCgPwtS174Lj1jskDksoEJiJ4BGYFe74n70BRST+AYfFt0MFtBa7c2ws8AxhUmhSEhsLRMqVkbIV7z91rZO9heB+/X9B8Ln3giSFXwBd2S06LS3HDPC1MMDdCMOOINcNAJTqXO7qYBwnRTWHe/IGKUERNANUCJrkTrwvC91ePwh6nl334E7TiDHjoZZ666XRVpd/UROs1K/WW4JS46UkhkjVDucP1/Ox0z0iY027mKzvIarM2whsIffkAgFDjvklER2ZHO+ihClqP+fMwWEr18ywlpIVE5oNZ3Tyox25vPobDY0T74iibhMONfSeMLyl7xeJtLnX9AjlppOs5Xivxp607lhoXScdeiQPoqXKDjW82QxYhVzWDLOtVdPbm1UC4PazI7USpC2u7+tNV8b3Dn01v+rfzfcrJ0brdtt35EW7lgJ61svECyStwrr63pGHGnLWnzuaAwiYzL+BgAA///JVKQ8" } diff --git a/x-pack/metricbeat/module/azure/mock_service.go b/x-pack/metricbeat/module/azure/mock_service.go index 0078273097c9..1d45cc6f8477 100644 --- a/x-pack/metricbeat/module/azure/mock_service.go +++ b/x-pack/metricbeat/module/azure/mock_service.go @@ -37,9 +37,9 @@ func (client *MockService) GetMetricNamespaces(resourceID string) (insights.Metr } // GetMetricValues is a mock function for the azure service -func (client *MockService) GetMetricValues(resourceID string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, error) { +func (client *MockService) GetMetricValues(resourceID string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, string, error) { args := client.Called(resourceID, namespace) - return args.Get(0).([]insights.Metric), args.Error(1) + return args.Get(0).([]insights.Metric), args.String(1), args.Error(2) } // MockReporterV2 mock implementation for testing purposes diff --git a/x-pack/metricbeat/module/azure/module.yml b/x-pack/metricbeat/module/azure/module.yml new file mode 100644 index 000000000000..9505846e81c9 --- /dev/null +++ b/x-pack/metricbeat/module/azure/module.yml @@ -0,0 +1,3 @@ +name: azure +metricsets: + - database_account diff --git a/x-pack/metricbeat/module/azure/monitor/_meta/data.json b/x-pack/metricbeat/module/azure/monitor/_meta/data.json index d13ab1986e37..d267245ce1e6 100644 --- a/x-pack/metricbeat/module/azure/monitor/_meta/data.json +++ b/x-pack/metricbeat/module/azure/monitor/_meta/data.json @@ -4,6 +4,7 @@ "type":"azure" }, "azure":{ + "timegrain" : "PT1M", "subscriptionID":"123456-qwer-1234-5678-12345678", "metrics":{ "percentage_cpu":{ diff --git a/x-pack/metricbeat/module/azure/monitor_service.go b/x-pack/metricbeat/module/azure/monitor_service.go index 4800a9c9de54..50649704bd67 100644 --- a/x-pack/metricbeat/module/azure/monitor_service.go +++ b/x-pack/metricbeat/module/azure/monitor_service.go @@ -90,8 +90,9 @@ func (service *MonitorService) GetMetricDefinitions(resourceID string, namespace } // GetMetricValues will return the metric values based on the resource and metric details -func (service *MonitorService) GetMetricValues(resourceID string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, error) { +func (service *MonitorService) GetMetricValues(resourceID string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, string, error) { var tg *string + var interval string if timegrain != "" { tg = &timegrain } @@ -110,10 +111,11 @@ func (service *MonitorService) GetMetricValues(resourceID string, namespace stri service.log.Warnf("Charges amounted to %v are being applied while retrieving the metric values from the resource %s ", *resp.Cost, resourceID) } if err != nil { - return metrics, err + return metrics, "", err } + interval = *resp.Interval metrics = append(metrics, *resp.Value...) } - return metrics, nil + return metrics, interval, nil } diff --git a/x-pack/metricbeat/module/azure/service_interface.go b/x-pack/metricbeat/module/azure/service_interface.go index cca771b4393f..f8310b6ca7b9 100644 --- a/x-pack/metricbeat/module/azure/service_interface.go +++ b/x-pack/metricbeat/module/azure/service_interface.go @@ -14,5 +14,5 @@ type Service interface { GetResourceDefinitions(ID []string, group []string, rType string, query string) (resources.ListResultPage, error) GetMetricDefinitions(resourceID string, namespace string) (insights.MetricDefinitionCollection, error) GetMetricNamespaces(resourceID string) (insights.MetricNamespaceCollection, error) - GetMetricValues(resourceID string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, error) + GetMetricValues(resourceID string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, string, error) } diff --git a/x-pack/metricbeat/module/azure/storage/_meta/data.json b/x-pack/metricbeat/module/azure/storage/_meta/data.json index 908a5da9ae86..6ede71f6edce 100644 --- a/x-pack/metricbeat/module/azure/storage/_meta/data.json +++ b/x-pack/metricbeat/module/azure/storage/_meta/data.json @@ -15,6 +15,7 @@ "period" : 300000 }, "azure" : { + "timegrain" : "PT5M", "namespace" : "Microsoft.Storage/storageAccounts/blobServices", "dimensions" : { "apiname" : "DeleteBlob" diff --git a/x-pack/metricbeat/modules.d/azure.yml.disabled b/x-pack/metricbeat/modules.d/azure.yml.disabled index 35977589fa5c..42b7d6a90f6c 100644 --- a/x-pack/metricbeat/modules.d/azure.yml.disabled +++ b/x-pack/metricbeat/modules.d/azure.yml.disabled @@ -49,3 +49,14 @@ # tenant_id: '${AZURE_TENANT_ID:""}' # subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' # refresh_list_interval: 600s + +#- module: azure +# metricsets: +# - database_account +# enabled: true +# period: 300s +# client_id: '${AZURE_CLIENT_ID:""}' +# client_secret: '${AZURE_CLIENT_SECRET:""}' +# tenant_id: '${AZURE_TENANT_ID:""}' +# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +# refresh_list_interval: 600s