Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cherry-pick #21486 to 7.x: Ignore unsupported metrics in the azure module #21513

Merged
merged 2 commits into from
Oct 5, 2020
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
@@ -817,6 +817,7 @@ field. You can revert this change by configuring tags for the module and omittin
- Add overview and platform health dashboards to Cloud Foundry module. {pull}21124[21124]
- Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255]
- Add dashboard for pubsub metricset in googlecloud module. {pull}21326[21326] {issue}17137[17137]
- Expand unsupported option from namespace to metrics in the azure module. {pull}21486[21486]

*Packetbeat*

Original file line number Diff line number Diff line change
@@ -12,8 +12,10 @@ input:
- name: ["CPU Credits Remaining", "CPU Credits Consumed", "OS Per Disk Read Bytes/sec", "OS Per Disk Write Bytes/sec", "OS Per Disk Read Operations/Sec", "OS Per Disk Write Operations/Sec", "OS Per Disk QD"]
namespace: "Microsoft.Compute/virtualMachineScaleSets"
timegrain: "PT5M"
ignore_unsupported: true
- name: ["Per Disk Read Bytes/sec", "Per Disk Write Bytes/sec", "Per Disk Read Operations/Sec", "Per Disk Write Operations/Sec", "Per Disk QD"]
namespace: "Microsoft.Compute/virtualMachineScaleSets"
ignore_unsupported: true
timegrain: "PT5M"
dimensions:
- name: "SlotId"
@@ -24,6 +26,7 @@ input:
"Premium Data Disk Cache Read Hit", "Outbound Flows Maximum Creation Rate", "Inbound Flows Maximum Creation Rate", "Outbound Flows", "Inbound Flows", "OS Disk IOPS Consumed Percentage", "OS Disk Bandwidth Consumed Percentage",
"OS Disk Queue Depth", "OS Disk Write Operations/Sec", "OS Disk Read Operations/Sec", "OS Disk Write Bytes/sec", "OS Disk Read Bytes/sec", "Data Disk IOPS Consumed Percentage"]
namespace: "Microsoft.Compute/virtualMachineScaleSets"
ignore_unsupported: true
timegrain: "PT5M"
dimensions:
- name: "VMName"
@@ -40,9 +43,11 @@ input:
metrics:
- name: ["CPU Credits Remaining", "CPU Credits Consumed", "OS Per Disk Read Bytes/sec", "OS Per Disk Write Bytes/sec", "OS Per Disk Read Operations/Sec", "OS Per Disk Write Operations/Sec", "OS Per Disk QD"]
namespace: "Microsoft.Compute/virtualMachineScaleSets"
ignore_unsupported: true
timegrain: "PT5M"
- name: ["Per Disk Read Bytes/sec", "Per Disk Write Bytes/sec", "Per Disk Read Operations/Sec", "Per Disk Write Operations/Sec", "Per Disk QD"]
namespace: "Microsoft.Compute/virtualMachineScaleSets"
ignore_unsupported: true
timegrain: "PT5M"
dimensions:
- name: "SlotId"
@@ -53,6 +58,7 @@ input:
"Premium Data Disk Cache Read Hit", "Outbound Flows Maximum Creation Rate", "Inbound Flows Maximum Creation Rate", "Outbound Flows", "Inbound Flows", "OS Disk IOPS Consumed Percentage", "OS Disk Bandwidth Consumed Percentage",
"OS Disk Queue Depth", "OS Disk Write Operations/Sec", "OS Disk Read Operations/Sec", "OS Disk Write Bytes/sec", "OS Disk Read Bytes/sec", "Data Disk IOPS Consumed Percentage"]
namespace: "Microsoft.Compute/virtualMachineScaleSets"
ignore_unsupported: true
timegrain: "PT5M"
dimensions:
- name: "VMName"
2 changes: 1 addition & 1 deletion x-pack/metricbeat/module/azure/config.go
Original file line number Diff line number Diff line change
@@ -41,7 +41,7 @@ type MetricConfig struct {
Dimensions []DimensionConfig `config:"dimensions"`
Timegrain string `config:"timegrain"`
// namespaces can be unsupported by some resources and supported in some, this configuration option makes sure no error messages are returned if namespace is unsupported
// info messages will be logged instead
// info messages will be logged instead. Same situation with metrics, some are being removed from the API, we would like to make sure that does not affect the module
IgnoreUnsupported bool `config:"ignore_unsupported"`
}

Original file line number Diff line number Diff line change
@@ -10,21 +10,25 @@ input:
metrics:
- name: ["CpuUsage", "MemoryUsage"]
namespace: "Microsoft.ContainerInstance/containerGroups"
ignore_unsupported: true
timegrain: "PT5M"
dimensions:
- name: "containerName"
value: "*"
- name: ["NetworkBytesReceivedPerSecond", "NetworkBytesTransmittedPerSecond"]
namespace: "Microsoft.ContainerInstance/containerGroups"
ignore_unsupported: true
timegrain: "PT5M"
- resource_id: ""
metrics:
- name: ["CpuUsage", "MemoryUsage"]
namespace: "Microsoft.ContainerInstance/containerGroups"
ignore_unsupported: true
timegrain: "PT5M"
dimensions:
- name: "containerName"
value: "*"
- name: ["NetworkBytesReceivedPerSecond", "NetworkBytesTransmittedPerSecond"]
namespace: "Microsoft.ContainerInstance/containerGroups"
ignore_unsupported: true
timegrain: "PT5M"
6 changes: 6 additions & 0 deletions x-pack/metricbeat/module/azure/container_service/manifest.yml
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@ input:
metrics:
- name: ["kube_node_status_condition"]
namespace: "Microsoft.ContainerService/managedClusters"
ignore_unsupported: true
timegrain: "PT5M"
dimensions:
- name: "node"
@@ -20,9 +21,11 @@ input:
value: "*"
- name: ["kube_node_status_allocatable_cpu_cores", "kube_node_status_allocatable_memory_bytes"]
namespace: "Microsoft.ContainerService/managedClusters"
ignore_unsupported: true
timegrain: "PT5M"
- name: ["kube_pod_status_ready", "kube_pod_status_phase"]
namespace: "Microsoft.ContainerService/managedClusters"
ignore_unsupported: true
timegrain: "PT5M"
dimensions:
- name: "pod"
@@ -31,6 +34,7 @@ input:
metrics:
- name: ["kube_node_status_condition"]
namespace: "Microsoft.ContainerService/managedClusters"
ignore_unsupported: true
timegrain: "PT5M"
dimensions:
- name: "node"
@@ -41,9 +45,11 @@ input:
value: "*"
- name: ["kube_node_status_allocatable_cpu_cores", "kube_node_status_allocatable_memory_bytes"]
namespace: "Microsoft.ContainerService/managedClusters"
ignore_unsupported: true
timegrain: "PT5M"
- name: ["kube_pod_status_ready", "kube_pod_status_phase"]
namespace: "Microsoft.ContainerService/managedClusters"
ignore_unsupported: true
timegrain: "PT5M"
dimensions:
- name: "pod"
8 changes: 8 additions & 0 deletions x-pack/metricbeat/module/azure/database_account/manifest.yml
Original file line number Diff line number Diff line change
@@ -14,12 +14,14 @@ input:
- name: ["AvailableStorage", "DataUsage","DocumentCount", "DocumentQuota", "IndexUsage", "MetadataRequests", "MongoRequestCharge", "MongoRequests", "MongoRequestsCount",
"MongoRequestsInsert", "MongoRequestsDelete", "MongoRequestsQuery", "MongoRequestsUpdate","ProvisionedThroughput", "NormalizedRUConsumption"]
namespace: "Microsoft.DocumentDb/databaseAccounts"
ignore_unsupported: true
timegrain: "PT5M"
dimensions:
- name: "DatabaseName"
value: "*"
- name: ["TotalRequestUnits", "TotalRequests"]
namespace: "Microsoft.DocumentDb/databaseAccounts"
ignore_unsupported: true
timegrain: "PT5M"
dimensions:
- name: "DatabaseName"
@@ -28,6 +30,7 @@ input:
value: "*"
- name: ["CassandraRequestCharges", "CassandraRequests"]
namespace: "Microsoft.DocumentDb/databaseAccounts"
ignore_unsupported: true
timegrain: "PT1M"
dimensions:
- name: "DatabaseName"
@@ -38,6 +41,7 @@ input:
"SqlContainerDelete", "SqlContainerThroughputUpdate", "SqlContainerUpdate", "SqlDatabaseDelete", "SqlDatabaseThroughputUpdate", "SqlDatabaseUpdate", "TableTableDelete",
"TableTableThroughputUpdate","TableTableUpdate"]
namespace: "Microsoft.DocumentDb/databaseAccounts"
ignore_unsupported: true
dimensions:
- name: "ResourceName"
value: "*"
@@ -49,12 +53,14 @@ input:
- name: ["AvailableStorage", "DataUsage","DocumentCount", "DocumentQuota", "IndexUsage", "MetadataRequests", "MongoRequestCharge", "MongoRequests", "MongoRequestsCount",
"MongoRequestsInsert", "MongoRequestsDelete", "MongoRequestsQuery", "MongoRequestsUpdate","ProvisionedThroughput", "NormalizedRUConsumption"]
namespace: "Microsoft.DocumentDb/databaseAccounts"
ignore_unsupported: true
timegrain: "PT5M"
dimensions:
- name: "DatabaseName"
value: "*"
- name: ["TotalRequestUnits", "TotalRequests"]
namespace: "Microsoft.DocumentDb/databaseAccounts"
ignore_unsupported: true
timegrain: "PT5M"
dimensions:
- name: "DatabaseName"
@@ -63,6 +69,7 @@ input:
value: "*"
- name: ["CassandraRequestCharges", "CassandraRequests"]
namespace: "Microsoft.DocumentDb/databaseAccounts"
ignore_unsupported: true
timegrain: "PT1M"
dimensions:
- name: "DatabaseName"
@@ -73,6 +80,7 @@ input:
"SqlContainerDelete", "SqlContainerThroughputUpdate", "SqlContainerUpdate", "SqlDatabaseDelete", "SqlDatabaseThroughputUpdate", "SqlDatabaseUpdate", "TableTableDelete",
"TableTableThroughputUpdate","TableTableUpdate"]
namespace: "Microsoft.DocumentDb/databaseAccounts"
ignore_unsupported: true
dimensions:
- name: "ResourceName"
value: "*"
3 changes: 3 additions & 0 deletions x-pack/metricbeat/module/azure/monitor/_meta/docs.asciidoc
Original file line number Diff line number Diff line change
@@ -50,6 +50,9 @@ Metrics with dimensions are exported as flattened single dimensional metrics, ag
`name`:: Dimension key
`value`:: Dimension value. (Users can select * to return metric values for each dimension)

`ignore_unsupported`:: (_bool_) Namespaces can be unsupported by some resources and supported in some, this configuration option makes sure no error messages are returned if the namespace is unsupported.
The same will go for the metrics configured, some can be removed from Azure Monitor and it should not affect the state of the module.

Users can select the options to retrieve all metrics from a specific namespace using the following:

["source","yaml"]
10 changes: 5 additions & 5 deletions x-pack/metricbeat/module/azure/monitor/client_helper.go
Original file line number Diff line number Diff line change
@@ -78,20 +78,20 @@ func filterMetricNames(resourceId string, metricConfig azure.MetricConfig, metri
}
} else {
// verify if configured metric names are valid, return log error event for the invalid ones, map only the valid metric names
supportedMetricNames, unsupportedMetricNames = filterSConfiguredMetrics(metricConfig.Name, metricDefinitions)
if len(unsupportedMetricNames) > 0 {
supportedMetricNames, unsupportedMetricNames = filterConfiguredMetrics(metricConfig.Name, metricDefinitions)
if len(unsupportedMetricNames) > 0 && !metricConfig.IgnoreUnsupported {
return nil, errors.Errorf("the metric names configured %s are not supported for the resource %s and namespace %s",
strings.Join(unsupportedMetricNames, ","), resourceId, metricConfig.Namespace)
}
}
if len(supportedMetricNames) == 0 {
if len(supportedMetricNames) == 0 && !metricConfig.IgnoreUnsupported {
return nil, errors.Errorf("the metric names configured : %s are not supported for the resource %s and namespace %s ", strings.Join(metricConfig.Name, ","), resourceId, metricConfig.Namespace)
}
return supportedMetricNames, nil
}

// filterSConfiguredMetrics will filter out any unsupported metrics based on the namespace selected
func filterSConfiguredMetrics(selectedRange []string, allRange []insights.MetricDefinition) ([]string, []string) {
// filterConfiguredMetrics will filter out any unsupported metrics based on the namespace selected
func filterConfiguredMetrics(selectedRange []string, allRange []insights.MetricDefinition) ([]string, []string) {
var inRange []string
var notInRange []string
var allMetrics string
Original file line number Diff line number Diff line change
@@ -108,7 +108,7 @@ func TestMapMetric(t *testing.T) {

func TestFilterSConfiguredMetrics(t *testing.T) {
selectedRange := []string{"TotalRequests", "Capacity", "CPUUsage"}
intersection, difference := filterSConfiguredMetrics(selectedRange, *MockMetricDefinitions())
intersection, difference := filterConfiguredMetrics(selectedRange, *MockMetricDefinitions())
assert.Equal(t, intersection, []string{"TotalRequests", "Capacity"})
assert.Equal(t, difference, []string{"CPUUsage"})
}