diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8588a2f75810..7946e3ca5c23 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -411,6 +411,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add test for documented fields check for metricsets without a http input. {issue}17315[17315] {pull}17334[17334] - Add final tests and move label to GA for the azure module in metricbeat. {pull}17319[17319] - Added documentation for running Metricbeat in Cloud Foundry. {pull}17275[17275] +- Refactor windows/perfmon metricset configuration options and event output. {pull}17596[17596] - Reference kubernetes manifests mount data directory from the host when running metricbeat as daemonset, so data persist between executions in the same node. {pull}17429[17429] - Add more detailed error messages, system tests and small refactoring to the service metricset in windows. {pull}17725[17725] diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index b4975842bd4d..f7fa6360018e 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -18569,20 +18569,13 @@ type: object -- -[float] -=== website - -website - - - -*`iis.website.name`*:: +*`iis.website.*.*`*:: + -- -website name +website -type: keyword +type: object -- @@ -38902,6 +38895,33 @@ Module for Windows +[float] +=== perfmon + +perfmon + + + +*`windows.perfmon.instance`*:: ++ +-- +Instance value. + + +type: keyword + +-- + +*`windows.perfmon.metrics.*.*`*:: ++ +-- +Metric values returned. + + +type: object + +-- + [float] === service diff --git a/metricbeat/docs/modules/windows.asciidoc b/metricbeat/docs/modules/windows.asciidoc index e343a359dccd..60faf2cf6421 100644 --- a/metricbeat/docs/modules/windows.asciidoc +++ b/metricbeat/docs/modules/windows.asciidoc @@ -5,8 +5,13 @@ This file is generated! See scripts/mage/docs_collector.go [[metricbeat-module-windows]] == Windows module -This is the Windows module. It collects metrics from Windows systems, -by default metricset `service` is enabled. +This is the `windows` module which collects metrics from Windows systems. +The module contains the `service` metricset, which is set up by default when the `windows` module is enabled. +The `service` metricset will retrieve status information of the services on the Windows machines. The second `windows` +metricset is `perfmon` which collects Windows performance counter values. + + + [float] @@ -24,11 +29,14 @@ metricbeat.modules: period: 10s perfmon.ignore_non_existent_counters: false perfmon.group_measurements_by_instance: false - perfmon.counters: - # - instance_label: processor.name - # instance_name: total - # measurement_label: processor.time.total.pct - # query: '\Processor Information(_Total)\% Processor Time' + perfmon.queries: +# - object: 'Process' +# instance: ["*"] +# counters: +# - name: 'Disk Writes/sec' +# field: physical_disk.write.per_sec +# format: "float" +# - name: "% Disk Write Time" - module: windows metricsets: ["service"] diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 1a2c1a0d5e26..4bb993c480cc 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -848,11 +848,14 @@ metricbeat.modules: period: 10s perfmon.ignore_non_existent_counters: false perfmon.group_measurements_by_instance: false - perfmon.counters: - # - instance_label: processor.name - # instance_name: total - # measurement_label: processor.time.total.pct - # query: '\Processor Information(_Total)\% Processor Time' + perfmon.queries: +# - object: 'Process' +# instance: ["*"] +# counters: +# - name: 'Disk Writes/sec' +# field: physical_disk.write.per_sec +# format: "float" +# - name: "% Disk Write Time" - module: windows metricsets: ["service"] diff --git a/metricbeat/module/windows/_meta/config.reference.yml b/metricbeat/module/windows/_meta/config.reference.yml index dc9b27a9c3d1..49656feb554a 100644 --- a/metricbeat/module/windows/_meta/config.reference.yml +++ b/metricbeat/module/windows/_meta/config.reference.yml @@ -4,11 +4,14 @@ period: 10s perfmon.ignore_non_existent_counters: false perfmon.group_measurements_by_instance: false - perfmon.counters: - # - instance_label: processor.name - # instance_name: total - # measurement_label: processor.time.total.pct - # query: '\Processor Information(_Total)\% Processor Time' + perfmon.queries: +# - object: 'Process' +# instance: ["*"] +# counters: +# - name: 'Disk Writes/sec' +# field: physical_disk.write.per_sec +# format: "float" +# - name: "% Disk Write Time" - module: windows metricsets: ["service"] diff --git a/metricbeat/module/windows/_meta/config.yml b/metricbeat/module/windows/_meta/config.yml index 3d3269a3b079..57ab272ef237 100644 --- a/metricbeat/module/windows/_meta/config.yml +++ b/metricbeat/module/windows/_meta/config.yml @@ -1,22 +1,17 @@ - module: windows - #metricsets: - # - service + metricsets: + - service period: 1m #- module: windows # metricsets: -# - perfmon +# - perfmon # period: 10s -# perfmon.counters: -# - instance_label: processor.name -# instance_name: total -# measurement_label: processor.time.total.pct -# query: '\Processor Information(_Total)\% Processor Time' -# -# - instance_label: physical_disk.name -# measurement_label: physical_disk.write.per_sec -# query: '\PhysicalDisk(*)\Disk Writes/sec' -# -# - instance_label: physical_disk.name -# measurement_label: physical_disk.write.time.pct -# query: '\PhysicalDisk(*)\% Disk Write Time' +# perfmon.queries: +# - object: 'Process' +# instance: ["*"] +# counters: +# - name: 'Disk Writes/sec' +# field: physical_disk.write.per_sec +# format: "float" +# - name: "% Disk Write Time" diff --git a/metricbeat/module/windows/_meta/docs.asciidoc b/metricbeat/module/windows/_meta/docs.asciidoc index 3a14b2eb03f7..b7ed8584d22c 100644 --- a/metricbeat/module/windows/_meta/docs.asciidoc +++ b/metricbeat/module/windows/_meta/docs.asciidoc @@ -1,2 +1,7 @@ -This is the Windows module. It collects metrics from Windows systems, -by default metricset `service` is enabled. +This is the `windows` module which collects metrics from Windows systems. +The module contains the `service` metricset, which is set up by default when the `windows` module is enabled. +The `service` metricset will retrieve status information of the services on the Windows machines. The second `windows` +metricset is `perfmon` which collects Windows performance counter values. + + + diff --git a/metricbeat/module/windows/fields.go b/metricbeat/module/windows/fields.go index 34f9f8b78612..1087083cc130 100644 --- a/metricbeat/module/windows/fields.go +++ b/metricbeat/module/windows/fields.go @@ -32,5 +32,5 @@ func init() { // AssetWindows returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/windows. func AssetWindows() string { - return "eJysVcFu2zgQvfsrBr30Ehvb9rQ+LOCNN7tebNOiSWEsECBiyJE1CEWqnKEd/31BSXZk2akdIzwY9JB8897MoziER1yPYUXO+BUPAITE4hjezZvIuwGAQdaBKiHvxvDHAADgszfRIuQ+wHx7lAsf5F57l9NiDLmyjAOAgBYV4xgWagCQE1rD4xpkCE6V2E2ehqyrtDn4WLWRA/l3gRqwbaIHFNWJN0kYw5I0buOHEr2YrBlZi5GB9k4UOQYpEFiURO7WYpOMR53zu3XYjL6MLmUyO+EN40dcr3zor+GTKqvUuWL+6frfK/1hUvV2/EJZGhOIjn5EhNm01lJLa3SMYCZADAoKxQX4vF4slS7I4XuGv7/PpqCcSeE93Baj1vRcj4OC0+85kueoealfJ/f2Wd4p1AxxZdX6/myKrTP+WqITuPTWohYfXs+5JVLT2nRi06ZfS2BRQe4T21cIOKWKCTdWNUyfUb2h8sz0YBGWykZkUAEhm0TxpRLS2cUeavan95JdQDYlVg8WTZp/Vi4qm13URstu1ixYZidJPrdn17cw+X77z5dvs9v/7/7zWtmbvY/ICTWaaO2jk6Zj0RkMsCpIF6C2BgzR8REplZLibCWX47v57Hr6ZX5zx3XhPn2846UuPMsInxCGj9DVB8NXfjuuorVr+BGVpZzQ1GRBfG2FnCyCFEqAEpkSnXDXI/vtJ6dtNOQWoMIi1geO91ne2NVKS1S2QT7d1ZfeCblIbnHI1l9V5HqpmTa+/hada4M3ya7bua+qZt4YPv1Hc8zx+ETpCTZvWI0rH57Tb582kII4PQqpMBiCD5DSNp3e+horH4T3IFcFuuZypi6LB27VptISw4qshQessRfoMJDuv617mB0O0VnkHZNBFfySTGrTJjTkCjXlpDsnj93BF15k693ipav34bffP55R740rDtdbMXtNStJVC14nsV9n0yPsYyVU4qjst+NFDbkPpZIxmBhUIttbJldFud9sKslaYtTemX6C01/i99yyhLY5aIDcDvZo8DMAAP//0yHt8w==" + return "eJysVlFv2zYQfvevOOSlQJEYa/s0PxTw4mXzsKRFk8IYEMCiqZN1C0WqvKMdA/vxAyXZkW05dozqoVB51Hffd99HM1fwhKsBLMmmbsk9ACExOICLSb1y0QNIkbWnUsjZAXzuAQDcujQYhMx5mGw+5dx5mWpnM5oPIFOGsQfg0aBiHMBc9QAyQpPyoAK5AqsKbDePj6zKuNm7UDYrHf23gdpgJfqscHaz3gUYnw2tGYpqrXc2q5996F0SLzTIsiircQugZvKEq6Xz6VZlq+t/WyWAcYMFC2UC9ju6FSieNPff9993NHSzf1HLVqFemtb1zDh1sDwtVFmSnTd7L95fHCb+eYf4bUWrps3gUYK3mL4I2DOP0S9oa2jd5r3SNWkwEtDOiiLLIDkCi5LA7cCum3G/KxVz9arNLaPTHdGHTQbAZ1WU8Xjlk093f93oD8NyZ8er8wQYQrD0IyCMR5WWSlqtow9jAWJQkCvOwWVVsVA6J4vvGP74Ph6Bsmlc3sNtMCpNHf60Bcd/z5E8Qc0L/Ta5Dy/yTqGWEpdGraZnU2yS8fsCrcC1Mwa1OP92zg2RitbaibVNr0tgUb4+eG8QcMoUI24oK5hdRtWG0jHTzOD6tCqPkAyDuEIJ6eRyDzX5zTlJLiEZEauZwTS+3yoblEkuq6Al9ysWLJKTJJ/r2d0DDL8//Pnl2/jhn8e/nVbmfu9H5IQZDbV2wUrtWLApeljmpHNQmwD6YPmIlFJJfraS68HjZHw3+jK5f+RqcJ8+PvJC546lj88IV0/Q1gdXb/ztuAnGrOBHUIYywrQiC+KqKGRkECRXAhTJFGiF2xnZt5+sNiElOwfl56H64LjP8pNTrbQEZWrk01N97ayQDWTnXbH+qgJXpfq1zvW3YG2zeB/junl31eW4CXz8P6bHEo/PFP9OSn/iNG6cf2m/udpAcuJ4KcTBoPfOQ2xbO73JNZbOC+9BLnO09eGMLosDbtTG0RLDkoyBGVbYc7QYr/qdu3UPs8UhWIO8FTIovVtQGm1aL11xiZoy0q0vj53BAzeycXZ+6Oh9+OXXj2fMe52K7nkrZqdJSTxq3uko9ut4dIR9KIUK7Be7dhzUkDlfKBlAGryKZHfKZMsg0/WmgowhRu1sutvg9Jv4HTcsoTEHUyC7hd3v/R8AAP//KJpZdg==" } diff --git a/metricbeat/module/windows/perfmon/_meta/data.json b/metricbeat/module/windows/perfmon/_meta/data.json index e6b39dd85880..4e4d87240783 100644 --- a/metricbeat/module/windows/perfmon/_meta/data.json +++ b/metricbeat/module/windows/perfmon/_meta/data.json @@ -1,24 +1,30 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" + "event": { + "dataset": "windows.perfmon", + "duration": 115000, + "module": "windows" }, "metricset": { - "module": "windows", "name": "perfmon", - "rtt": 115 + "period": 10000 + }, + "service": { + "type": "windows" }, "windows": { "perfmon": { - "processor": { - "name": "_Total", - "time": { - "total": { - "pct": 1.4663385364361736 + "instance": "_Total", + "metrics": { + "processor": { + "time": { + "total": { + "pct": 6.310940413107646 + } } } - } + }, + "object": "Processor Information" } } } \ No newline at end of file diff --git a/metricbeat/module/windows/perfmon/_meta/docs.asciidoc b/metricbeat/module/windows/perfmon/_meta/docs.asciidoc index 4c90de92fdd8..f04c9247c049 100644 --- a/metricbeat/module/windows/perfmon/_meta/docs.asciidoc +++ b/metricbeat/module/windows/perfmon/_meta/docs.asciidoc @@ -15,6 +15,28 @@ to collect. The example below collects processor time and disk writes every period: 10s perfmon.ignore_non_existent_counters: true perfmon.group_measurements_by_instance: true + perfmon.queries: + - object: "Process" + instance: ["svchost*", "conhost*"] + counters: + - name: "% Processor Time" + field: time.processor.pct + format: "float" + - name: "Thread Count" + field: thread_count + - name: "IO Read Operations/sec" + - object: "PhysicalDisk" + field : "disk" + instance: "*" + counters: + - name: "Disk Writes/sec" + - name: "% Disk Write Time" + field: "write_time" + format: "float" + + + // deprecated, will be removed in 8.0 + perfmon.counters: - instance_label: processor.name instance_name: total @@ -46,7 +68,33 @@ counter requires three config options - `instance_label`, `measurement_label`, and `query`. [float] -==== Counter Configuration +==== Query Configuration + +Each item in the `query` list specifies multiple perfmon queries to perform. In the +events generated by the metricset these configuration options map to the field +values as shown below. + +*`object`*:: The performance object to query. A performance object can be a physical component, such as processors, disks, and memory, or a system object, such as processes and threads. Required + +*`field`*:: The object field/label. Not required, if not entered, it will be `object`. + +*`instance`*:: Matches the ParentInstance, ObjectInstance, and InstanceIndex are included in the path if multiple instances of the object can exist. Not required for performance counters which do not contain one. + +*`counters`*:: List of the partial counter paths (At least one partial counter path is required). + +*`name`*:: The counter name. Required. This is the counter specified in Performance Data Helper (PDH) syntax. For example in case of the counter path `\Processor Information(_Total)\% Processor Time`, +the value for this configuration option will be `% Processor Time`. + +*`field`*:: The counter path value field/label. Not required, if not entered, it will be generated based on the counter path. + +*`format`*:: Format of the measurement value. The value can be either `float`, `large` or +`long`. The default is `float`. + + + + +[float] +==== Deprecated Counter Configuration Each item in the `counters` list specifies a perfmon query to perform. In the events generated by the metricset these configuration options map to the field @@ -77,6 +125,6 @@ Performance Data Helper (PDH) syntax. This field is required. For example place of an instance name to perform a wildcard query that generates an event for each counter instance (e.g. `\PhysicalDisk(*)\Disk Writes/sec`). -*`format`*:: Format of the measurement value. The value can be either `float` or +*`format`*:: Format of the measurement value. The value can be either `float`, `large` or `long`. The default is `float`. diff --git a/metricbeat/module/windows/perfmon/_meta/fields.yml b/metricbeat/module/windows/perfmon/_meta/fields.yml index 8033a27f5ac5..9e07225e17d8 100644 --- a/metricbeat/module/windows/perfmon/_meta/fields.yml +++ b/metricbeat/module/windows/perfmon/_meta/fields.yml @@ -1 +1,17 @@ -- release: beta +- name: perfmon + type: group + release: beta + description: > + perfmon + fields: + - name: instance + type: keyword + description: | + Instance value. + - name: metrics.*.* + type: object + object_type: float + object_type_mapping_type: "*" + description: > + Metric values returned. + diff --git a/metricbeat/module/windows/perfmon/config.go b/metricbeat/module/windows/perfmon/config.go new file mode 100644 index 000000000000..971d4629b273 --- /dev/null +++ b/metricbeat/module/windows/perfmon/config.go @@ -0,0 +1,112 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package perfmon + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" +) + +var allowedFormats = []string{"float", "large", "long"} + +// Config for the windows perfmon metricset. +type Config struct { + IgnoreNECounters bool `config:"perfmon.ignore_non_existent_counters"` + GroupMeasurements bool `config:"perfmon.group_measurements_by_instance"` + Counters []Counter `config:"perfmon.counters"` + Queries []Query `config:"perfmon.queries"` + GroupAllCountersTo string `config:"perfmon.group_all_counter"` +} + +// Counter for the perfmon counters (old implementation deprecated). +type Counter struct { + InstanceLabel string `config:"instance_label"` + InstanceName string `config:"instance_name"` + MeasurementLabel string `config:"measurement_label" validate:"required"` + Query string `config:"query" validate:"required"` + Format string `config:"format"` +} + +// QueryConfig for perfmon queries. This will be used as the new configuration format +type Query struct { + Name string `config:"object" validate:"required"` + Field string `config:"field"` + Instance []string `config:"instance"` + Counters []QueryCounter `config:"counters" validate:"required,nonzero"` + Namespace string `config:"namespace"` +} + +// QueryConfigCounter for perfmon queries. This will be used as the new configuration format +type QueryCounter struct { + Name string `config:"name" validate:"required"` + Field string `config:"field"` + Format string `config:"format"` +} + +func (query *Query) InitDefaults() { + query.Namespace = "metrics" +} + +func (counter *QueryCounter) InitDefaults() { + counter.Format = "float" +} + +func (counter *Counter) InitDefaults() { + counter.Format = "float" +} + +func (counter *Counter) Validate() error { + if !isValidFormat(counter.Format) { + return errors.Errorf("initialization failed: format '%s' "+ + "for counter '%s' is invalid (must be float, large or long)", + counter.Format, counter.InstanceLabel) + } + return nil +} + +func (counter *QueryCounter) Validate() error { + if !isValidFormat(counter.Format) { + return errors.Errorf("initialization failed: format '%s' "+ + "for counter '%s' is invalid (must be float, large or long)", + counter.Format, counter.Name) + } + return nil +} + +func (conf *Config) Validate() error { + if len(conf.Counters) == 0 && len(conf.Queries) == 0 { + return errors.New("no perfmon counters or queries have been configured") + } + if len(conf.Counters) > 0 { + cfgwarn.Deprecate("8.0", "perfmon.counters configuration option is deprecated and will be removed in the future major version, "+ + "we advise using the perfmon.queries configuration option instead.") + } + return nil +} + +func isValidFormat(format string) bool { + for _, form := range allowedFormats { + if form == format { + return true + } + } + return false +} diff --git a/metricbeat/module/windows/perfmon/config_test.go b/metricbeat/module/windows/perfmon/config_test.go new file mode 100644 index 000000000000..8f96a9ca1ed4 --- /dev/null +++ b/metricbeat/module/windows/perfmon/config_test.go @@ -0,0 +1,73 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package perfmon + +import ( + "testing" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/go-ucfg" + + "github.com/stretchr/testify/assert" +) + +func TestValidate(t *testing.T) { + conf := common.MapStr{ + "module": "windows", + "period": "10s", + "metricsets": []string{"perfmon"}, + "perfmon.group_measurements_by_instance": true, + } + c, err := ucfg.NewFrom(conf) + assert.NoError(t, err) + var config Config + err = c.Unpack(&config) + assert.Error(t, err, "no perfmon counters or queries have been configured") + conf["perfmon.queries"] = []common.MapStr{ + { + "object": "Process", + }, + } + c, err = ucfg.NewFrom(conf) + assert.NoError(t, err) + err = c.Unpack(&config) + assert.Error(t, err, "missing required field accessing 'perfmon.queries.0.counters'") + + conf["perfmon.queries"] = []common.MapStr{ + { + "object": "Process", + "counters": []common.MapStr{ + { + "name": "Thread Count", + }, + }, + }, + } + c, err = ucfg.NewFrom(conf) + assert.NoError(t, err) + err = c.Unpack(&config) + assert.NoError(t, err) + assert.Equal(t, config.Queries[0].Counters[0].Format, "float") + assert.Equal(t, config.Queries[0].Namespace, "metrics") + assert.Equal(t, config.Queries[0].Name, "Process") + assert.Equal(t, config.Queries[0].Counters[0].Name, "Thread Count") + assert.True(t, config.GroupMeasurements) + +} diff --git a/metricbeat/module/windows/perfmon/data.go b/metricbeat/module/windows/perfmon/data.go new file mode 100644 index 000000000000..7db0a338de22 --- /dev/null +++ b/metricbeat/module/windows/perfmon/data.go @@ -0,0 +1,149 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package perfmon + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/metricbeat/helper/windows/pdh" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +var processRegexp = regexp.MustCompile(`(.+?)#[1-9]+`) + +func (re *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.Event { + eventMap := make(map[string]*mb.Event) + for counterPath, values := range counters { + hasCounter, counter := re.getCounter(counterPath) + for ind, val := range values { + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + if val.Err != nil && !re.executed { + re.log.Debugw("Ignoring the first measurement because the data isn't ready", + "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) + continue + } + + var eventKey string + if re.config.GroupMeasurements && val.Err == nil { + // Send measurements with the same instance label as part of the same event + eventKey = val.Instance + } else { + // Send every measurement as an individual event + // If a counter contains an error, it will always be sent as an individual event + eventKey = counterPath + strconv.Itoa(ind) + } + + // Create a new event if the key doesn't exist in the map + if _, ok := eventMap[eventKey]; !ok { + eventMap[eventKey] = &mb.Event{ + MetricSetFields: common.MapStr{}, + Error: errors.Wrapf(val.Err, "failed on query=%v", counterPath), + } + if val.Instance != "" && hasCounter { + //will ignore instance counter + if ok, match := matchesParentProcess(val.Instance); ok { + eventMap[eventKey].MetricSetFields.Put(counter.InstanceField, match) + } else { + eventMap[eventKey].MetricSetFields.Put(counter.InstanceField, val.Instance) + } + } + } + event := eventMap[eventKey] + if val.Measurement != nil { + event.MetricSetFields.Put(counter.QueryField, val.Measurement) + } else { + event.MetricSetFields.Put(counter.QueryField, 0) + } + if counter.ObjectField != "" { + event.MetricSetFields.Put(counter.ObjectField, counter.ObjectName) + } + } + } + // Write the values into the map. + events := make([]mb.Event, 0, len(eventMap)) + for _, val := range eventMap { + events = append(events, *val) + } + return events +} + +func (re *Reader) groupToSingleEvent(counters map[string][]pdh.CounterValue) mb.Event { + event := mb.Event{ + MetricSetFields: common.MapStr{}, + } + measurements := make(map[string]float64, 0) + for counterPath, values := range counters { + _, readerCounter := re.getCounter(counterPath) + for _, val := range values { + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + if val.Err != nil && !re.executed { + re.log.Debugw("Ignoring the first measurement because the data isn't ready", + "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) + continue + } + var counterVal float64 + switch val.Measurement.(type) { + case int64: + counterVal = float64(val.Measurement.(int64)) + case int: + counterVal = float64(val.Measurement.(int)) + default: + counterVal = val.Measurement.(float64) + } + if _, ok := measurements[readerCounter.QueryField]; !ok { + measurements[readerCounter.QueryField] = counterVal + measurements[readerCounter.QueryField+instanceCountLabel] = 1 + } else { + measurements[readerCounter.QueryField+instanceCountLabel] = measurements[readerCounter.QueryField+instanceCountLabel] + 1 + measurements[readerCounter.QueryField] = measurements[readerCounter.QueryField] + counterVal + } + } + } + for key, val := range measurements { + if strings.Contains(key, instanceCountLabel) { + if val == 1 { + continue + } else { + event.MetricSetFields.Put(fmt.Sprintf("%s.%s", strings.Split(key, ".")[0], re.config.GroupAllCountersTo), val) + } + } else { + event.MetricSetFields.Put(key, val) + } + } + return event +} + +// matchParentProcess will try to get the parent process name +func matchesParentProcess(instanceName string) (bool, string) { + matches := processRegexp.FindStringSubmatch(instanceName) + if len(matches) == 2 { + return true, matches[1] + } + return false, instanceName +} diff --git a/metricbeat/module/windows/perfmon/data_test.go b/metricbeat/module/windows/perfmon/data_test.go new file mode 100644 index 000000000000..616d87d686f5 --- /dev/null +++ b/metricbeat/module/windows/perfmon/data_test.go @@ -0,0 +1,167 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package perfmon + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/metricbeat/helper/windows/pdh" +) + +func TestGroupToEvents(t *testing.T) { + reader := Reader{ + query: pdh.Query{}, + executed: true, + log: nil, + counters: []PerfCounter{ + { + QueryField: "datagrams_sent_per_sec", + QueryName: `\UDPv4\Datagrams Sent/sec`, + Format: "float", + ObjectName: "UDPv4", + ObjectField: "object", + ChildQueries: []string{`\UDPv4\Datagrams Sent/sec`}, + }, + }, + } + counters := map[string][]pdh.CounterValue{ + `\UDPv4\Datagrams Sent/sec`: { + { + Instance: "", + Measurement: 23, + Err: nil, + }, + }, + } + events := reader.groupToEvents(counters) + assert.NotNil(t, events) + assert.Equal(t, len(events), 1) + ok, err := events[0].MetricSetFields.HasKey("datagrams_sent_per_sec") + assert.NoError(t, err) + assert.True(t, ok) + ok, err = events[0].MetricSetFields.HasKey("object") + assert.NoError(t, err) + assert.True(t, ok) + val, err := events[0].MetricSetFields.GetValue("datagrams_sent_per_sec") + assert.NoError(t, err) + assert.Equal(t, val, 23) + val, err = events[0].MetricSetFields.GetValue("object") + assert.NoError(t, err) + assert.Equal(t, val, "UDPv4") + +} + +func TestGroupToSingleEvent(t *testing.T) { + reader := Reader{ + query: pdh.Query{}, + executed: true, + log: nil, + config: Config{ + GroupAllCountersTo: "processor_count", + }, + counters: []PerfCounter{ + { + QueryField: "%_processor_time", + QueryName: `\Processor Information(*)\% Processor Time`, + Format: "float", + ObjectName: "Processor Information", + ObjectField: "object", + InstanceName: "*", + InstanceField: "instance", + ChildQueries: []string{`\Processor Information(processor0)\% Processor Time`, `\Processor Information(processor1)\% Processor Time`}, + }, + { + QueryField: "%_user_time", + QueryName: `\Processor Information(*)\% User Time`, + Format: "float", + ObjectName: "Processor Information", + ObjectField: "object", + InstanceName: "*", + InstanceField: "instance", + ChildQueries: []string{`\Processor Information(processor0)\% User Time`, `\Processor Information(processor1)\% User Time`}, + }, + }, + } + + counters := map[string][]pdh.CounterValue{ + `\Processor Information(processor0)\% Processor Time`: { + { + Instance: "processor0", + Measurement: 23, + }, + }, + `\Processor Information(processor1)\% Processor Time`: { + { + Instance: "processor1", + Measurement: 21, + }, + }, + `\Processor Information(processor0)\% User Time`: { + { + Instance: "processor0", + Measurement: 10, + }, + }, + `\Processor Information(processor1)\% User Time`: { + { + Instance: "processor1", + Measurement: 11, + }, + }, + } + event := reader.groupToSingleEvent(counters) + assert.NotNil(t, event) + ok, err := event.MetricSetFields.HasKey("%_processor_time") + assert.NoError(t, err) + assert.True(t, ok) + ok, err = event.MetricSetFields.HasKey("%_processor_time:count") + assert.NoError(t, err) + assert.True(t, ok) + val, err := event.MetricSetFields.GetValue("%_processor_time") + assert.NoError(t, err) + assert.Equal(t, val, float64(44)) + val, err = event.MetricSetFields.GetValue("%_processor_time:count") + assert.NoError(t, err) + assert.Equal(t, val, common.MapStr{"processor_count": float64(2)}) + ok, err = event.MetricSetFields.HasKey("%_user_time") + assert.NoError(t, err) + assert.True(t, ok) + ok, err = event.MetricSetFields.HasKey("%_user_time:count") + assert.NoError(t, err) + assert.True(t, ok) + val, err = event.MetricSetFields.GetValue("%_user_time") + assert.NoError(t, err) + assert.Equal(t, val, float64(21)) + val, err = event.MetricSetFields.GetValue("%_user_time:count") + assert.NoError(t, err) + assert.Equal(t, val, common.MapStr{"processor_count": float64(2)}) +} + +func TestMatchesParentProcess(t *testing.T) { + ok, val := matchesParentProcess("svchost") + assert.False(t, ok) + assert.Equal(t, val, "svchost") + ok, val = matchesParentProcess("svchost#54") + assert.True(t, ok) + assert.Equal(t, val, "svchost") +} diff --git a/metricbeat/module/windows/perfmon/perfmon.go b/metricbeat/module/windows/perfmon/perfmon.go index 76f8833b3f04..c0490a344309 100644 --- a/metricbeat/module/windows/perfmon/perfmon.go +++ b/metricbeat/module/windows/perfmon/perfmon.go @@ -20,8 +20,6 @@ package perfmon import ( - "strings" - "github.com/elastic/beats/v7/metricbeat/mb/parse" "github.com/pkg/errors" @@ -31,23 +29,6 @@ import ( "github.com/elastic/beats/v7/metricbeat/mb" ) -// CounterConfig for perfmon counters. -type CounterConfig struct { - InstanceLabel string `config:"instance_label"` - InstanceName string `config:"instance_name"` - MeasurementLabel string `config:"measurement_label" validate:"required"` - Query string `config:"query" validate:"required"` - Format string `config:"format"` -} - -// Config for the windows perfmon metricset. -type Config struct { - IgnoreNECounters bool `config:"perfmon.ignore_non_existent_counters"` - GroupMeasurements bool `config:"perfmon.group_measurements_by_instance"` - CounterConfig []CounterConfig `config:"perfmon.counters" validate:"required"` - GroupAllCountersTo string `config:"perfmon.group_all_counter"` -} - const metricsetName = "perfmon" func init() { @@ -68,19 +49,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } - for _, value := range config.CounterConfig { - form := strings.ToLower(value.Format) - switch form { - case "", "float": - value.Format = "float" - case "long", "large": - default: - return nil, errors.Errorf("initialization failed: format '%s' "+ - "for counter '%s' is invalid (must be float, large or long)", - value.Format, value.InstanceLabel) - } - - } reader, err := NewReader(config) if err != nil { return nil, errors.Wrap(err, "initialization of reader failed") diff --git a/metricbeat/module/windows/perfmon/perfmon_test.go b/metricbeat/module/windows/perfmon/perfmon_test.go index cd882131b541..069138a4226d 100644 --- a/metricbeat/module/windows/perfmon/perfmon_test.go +++ b/metricbeat/module/windows/perfmon/perfmon_test.go @@ -27,8 +27,6 @@ import ( "github.com/elastic/beats/v7/metricbeat/helper/windows/pdh" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/pkg/errors" "github.com/stretchr/testify/assert" @@ -39,6 +37,37 @@ import ( const processorTimeCounter = `\Processor Information(_Total)\% Processor Time` func TestData(t *testing.T) { + config := map[string]interface{}{ + "module": "windows", + "metricsets": []string{"perfmon"}, + "perfmon.queries": []map[string]interface{}{ + { + "object": "Processor Information", + "instance": []string{"_Total"}, + "counters": []map[string]interface{}{ + { + "name": "% Processor Time", + "field": "processor.time.total.pct", + }, + { + "name": "% User Time", + }, + }, + }, + }, + } + + ms := mbtest.NewReportingMetricSetV2Error(t, config) + mbtest.ReportingFetchV2Error(ms) + time.Sleep(60 * time.Millisecond) + + if err := mbtest.WriteEventsReporterV2Error(ms, t, "/"); err != nil { + t.Fatal("write", err) + } + +} + +func TestDataDeprecated(t *testing.T) { config := map[string]interface{}{ "module": "windows", "metricsets": []string{"perfmon"}, @@ -81,12 +110,14 @@ func TestCounterWithNoInstanceName(t *testing.T) { config := map[string]interface{}{ "module": "windows", "metricsets": []string{"perfmon"}, - "perfmon.counters": []map[string]string{ + "perfmon.queries": []map[string]interface{}{ { - "instance_label": "processor.name", - "measurement_label": "processor.time.total.pct", - "query": `\UDPv4\Datagrams Sent/sec`, - //"query": `\UDPv4\Verzonden datagrammen per seconde`, + "object": "UDPv4", + "counters": []map[string]interface{}{ + { + "name": "Datagrams Sent/sec", + }, + }, }, }, } @@ -102,9 +133,10 @@ func TestCounterWithNoInstanceName(t *testing.T) { if len(events) == 0 { t.Fatal("no events received") } - process := events[0].MetricSetFields["processor"].(common.MapStr) + val, err := events[0].MetricSetFields.GetValue("object") + assert.NoError(t, err) // Check values - assert.EqualValues(t, "UDPv4", process["name"]) + assert.EqualValues(t, "UDPv4", val) } @@ -115,12 +147,11 @@ func TestQuery(t *testing.T) { t.Fatal(err) } defer q.Close() - counter := CounterConfig{Format: "float", InstanceName: "TestInstanceName"} path, err := q.GetCounterPaths(processorTimeCounter) if err != nil { t.Fatal(err) } - err = q.AddCounter(path[0], counter.InstanceName, counter.Format, false) + err = q.AddCounter(path[0], "TestInstanceName", "float", false) if err != nil { t.Fatal(err) } @@ -150,12 +181,36 @@ func TestQuery(t *testing.T) { func TestExistingCounter(t *testing.T) { config := Config{ - CounterConfig: make([]CounterConfig, 1), + Queries: make([]Query, 1), + } + config.Queries[0].Name = "Processor Information" + config.Queries[0].Instance = []string{"_Total"} + config.Queries[0].Counters = []QueryCounter{ + { + Name: "% Processor Time", + }, } - config.CounterConfig[0].InstanceLabel = "processor.name" - config.CounterConfig[0].MeasurementLabel = "processor.time.total.pct" - config.CounterConfig[0].Query = processorTimeCounter - config.CounterConfig[0].Format = "float" + handle, err := NewReader(config) + if err != nil { + t.Fatal(err) + } + defer handle.query.Close() + + values, err := handle.Read() + if err != nil { + t.Fatal(err) + } + t.Log(values) +} + +func TestExistingCounterDeprecated(t *testing.T) { + config := Config{ + Counters: make([]Counter, 1), + } + config.Counters[0].InstanceLabel = "processor.name" + config.Counters[0].MeasurementLabel = "processor.time.total.pct" + config.Counters[0].Query = processorTimeCounter + config.Counters[0].Format = "float" handle, err := NewReader(config) if err != nil { t.Fatal(err) @@ -172,12 +227,34 @@ func TestExistingCounter(t *testing.T) { func TestNonExistingCounter(t *testing.T) { config := Config{ - CounterConfig: make([]CounterConfig, 1), + Queries: make([]Query, 1), + } + config.Queries[0].Name = "Processor Information" + config.Queries[0].Instance = []string{"_Total"} + config.Queries[0].Counters = []QueryCounter{ + { + Name: "% Processor Time time", + }, + } + handle, err := NewReader(config) + if assert.Error(t, err) { + assert.EqualValues(t, pdh.PDH_CSTATUS_NO_COUNTER, errors.Cause(err)) + } + + if handle != nil { + err = handle.query.Close() + assert.NoError(t, err) + } +} + +func TestNonExistingCounterDeprecated(t *testing.T) { + config := Config{ + Counters: make([]Counter, 1), } - config.CounterConfig[0].InstanceLabel = "processor.name" - config.CounterConfig[0].MeasurementLabel = "processor.time.total.pct" - config.CounterConfig[0].Query = "\\Processor Information(_Total)\\not existing counter" - config.CounterConfig[0].Format = "float" + config.Counters[0].InstanceLabel = "processor.name" + config.Counters[0].MeasurementLabel = "processor.time.total.pct" + config.Counters[0].Query = "\\Processor Information(_Total)\\not existing counter" + config.Counters[0].Format = "float" handle, err := NewReader(config) if assert.Error(t, err) { assert.EqualValues(t, pdh.PDH_CSTATUS_NO_COUNTER, errors.Cause(err)) @@ -191,13 +268,41 @@ func TestNonExistingCounter(t *testing.T) { func TestIgnoreNonExistentCounter(t *testing.T) { config := Config{ - CounterConfig: make([]CounterConfig, 1), + Queries: make([]Query, 1), + IgnoreNECounters: true, + } + config.Queries[0].Name = "Processor Information" + config.Queries[0].Instance = []string{"_Total"} + config.Queries[0].Counters = []QueryCounter{ + { + Name: "% Processor Time time", + }, + } + handle, err := NewReader(config) + + values, err := handle.Read() + + if assert.Error(t, err) { + assert.EqualValues(t, pdh.PDH_NO_DATA, errors.Cause(err)) + } + + if handle != nil { + err = handle.query.Close() + assert.NoError(t, err) + } + + t.Log(values) +} + +func TestIgnoreNonExistentCounterDeprecated(t *testing.T) { + config := Config{ + Counters: make([]Counter, 1), IgnoreNECounters: true, } - config.CounterConfig[0].InstanceLabel = "processor.name" - config.CounterConfig[0].MeasurementLabel = "processor.time.total.pct" - config.CounterConfig[0].Query = "\\Processor Information(_Total)\\not existing counter" - config.CounterConfig[0].Format = "float" + config.Counters[0].InstanceLabel = "processor.name" + config.Counters[0].MeasurementLabel = "processor.time.total.pct" + config.Counters[0].Query = "\\Processor Information(_Total)\\not existing counter" + config.Counters[0].Format = "float" handle, err := NewReader(config) values, err := handle.Read() @@ -216,12 +321,34 @@ func TestIgnoreNonExistentCounter(t *testing.T) { func TestNonExistingObject(t *testing.T) { config := Config{ - CounterConfig: make([]CounterConfig, 1), + Queries: make([]Query, 1), + } + config.Queries[0].Name = "Processor MisInformation" + config.Queries[0].Instance = []string{"_Total"} + config.Queries[0].Counters = []QueryCounter{ + { + Name: "% Processor Time", + }, + } + handle, err := NewReader(config) + if assert.Error(t, err) { + assert.EqualValues(t, pdh.PDH_CSTATUS_NO_OBJECT, errors.Cause(err)) + } + + if handle != nil { + err = handle.query.Close() + assert.NoError(t, err) + } +} + +func TestNonExistingObjectDeprecated(t *testing.T) { + config := Config{ + Counters: make([]Counter, 1), } - config.CounterConfig[0].InstanceLabel = "processor.name" - config.CounterConfig[0].MeasurementLabel = "processor.time.total.pct" - config.CounterConfig[0].Query = "\\non existing object\\% Processor Performance" - config.CounterConfig[0].Format = "float" + config.Counters[0].InstanceLabel = "processor.name" + config.Counters[0].MeasurementLabel = "processor.time.total.pct" + config.Counters[0].Query = "\\non existing object\\% Processor Performance" + config.Counters[0].Format = "float" handle, err := NewReader(config) if assert.Error(t, err) { assert.EqualValues(t, pdh.PDH_CSTATUS_NO_OBJECT, errors.Cause(err)) @@ -240,13 +367,12 @@ func TestLongOutputFormat(t *testing.T) { t.Fatal(err) } defer query.Close() - counter := CounterConfig{Format: "long"} path, err := query.GetCounterPaths(processorTimeCounter) if err != nil { t.Fatal(err) } assert.NotZero(t, len(path)) - err = query.AddCounter(path[0], counter.InstanceName, counter.Format, false) + err = query.AddCounter(path[0], "", "long", false) if err != nil && err != pdh.PDH_NO_MORE_DATA { t.Fatal(err) } @@ -280,13 +406,12 @@ func TestFloatOutputFormat(t *testing.T) { t.Fatal(err) } defer query.Close() - counter := CounterConfig{Format: "float"} path, err := query.GetCounterPaths(processorTimeCounter) if err != nil { t.Fatal(err) } assert.NotZero(t, len(path)) - err = query.AddCounter(path[0], counter.InstanceName, counter.Format, false) + err = query.AddCounter(path[0], "", "float", false) if err != nil && err != pdh.PDH_NO_MORE_DATA { t.Fatal(err) } @@ -315,13 +440,15 @@ func TestFloatOutputFormat(t *testing.T) { func TestWildcardQuery(t *testing.T) { config := Config{ - CounterConfig: make([]CounterConfig, 1), + Queries: make([]Query, 1), + } + config.Queries[0].Name = "Processor Information" + config.Queries[0].Instance = []string{"*"} + config.Queries[0].Counters = []QueryCounter{ + { + Name: "% Processor Time", + }, } - config.CounterConfig[0].InstanceLabel = "processor.name" - config.CounterConfig[0].InstanceName = "TestInstanceName" - config.CounterConfig[0].MeasurementLabel = "processor.time.pct" - config.CounterConfig[0].Query = `\Processor Information(*)\% Processor Time` - config.CounterConfig[0].Format = "float" handle, err := NewReader(config) if err != nil { t.Fatal(err) @@ -337,28 +464,26 @@ func TestWildcardQuery(t *testing.T) { t.Fatal(err) } assert.NotZero(t, len(values)) - pctKey, err := values[0].MetricSetFields.HasKey("processor.time.pct") + pctKey, err := values[0].MetricSetFields.HasKey("metrics.%_processor_time") if err != nil { t.Fatal(err) } assert.True(t, pctKey) - - pct, err := values[0].MetricSetFields.GetValue("processor.name") - if err != nil { - t.Fatal(err) - } - assert.NotEqual(t, "TestInstanceName", pct) - t.Log(values) } func TestWildcardQueryNoInstanceName(t *testing.T) { config := Config{ - CounterConfig: make([]CounterConfig, 1), + Queries: make([]Query, 1), + } + config.Queries[0].Name = "Process" + config.Queries[0].Instance = []string{"*"} + config.Queries[0].Counters = []QueryCounter{ + { + Name: "Private Bytes", + }, } - config.CounterConfig[0].InstanceLabel = "process_private_bytes" - config.CounterConfig[0].MeasurementLabel = "process.private.bytes" - config.CounterConfig[0].Query = `\Process(*)\Private Bytes` + handle, err := NewReader(config) if err != nil { t.Fatal(err) @@ -374,18 +499,18 @@ func TestWildcardQueryNoInstanceName(t *testing.T) { t.Fatal(err) } assert.NotZero(t, len(values)) - pctKey, err := values[0].MetricSetFields.HasKey("process.private.bytes") + pctKey, err := values[0].MetricSetFields.HasKey("metrics.private_bytes") if err != nil { t.Fatal(err) } assert.True(t, pctKey) for _, s := range values { - pct, err := s.MetricSetFields.GetValue("process_private_bytes") + instance, err := s.MetricSetFields.GetValue("instance") if err != nil { t.Fatal(err) } - assert.False(t, strings.Contains(pct.(string), "*")) + assert.False(t, strings.Contains(instance.(string), "*")) } t.Log(values) @@ -393,23 +518,80 @@ func TestWildcardQueryNoInstanceName(t *testing.T) { func TestGroupByInstance(t *testing.T) { config := Config{ - CounterConfig: make([]CounterConfig, 3), + Queries: make([]Query, 1), + GroupMeasurements: true, + } + config.Queries[0].Name = "Processor Information" + config.Queries[0].Instance = []string{"_Total"} + config.Queries[0].Counters = []QueryCounter{ + { + Name: "% Processor Time", + }, + { + Name: "% User Time", + }, + { + Name: "% Privileged Time", + }, + } + handle, err := NewReader(config) + if err != nil { + t.Fatal(err) + } + defer handle.query.Close() + + values, _ := handle.Read() + + time.Sleep(time.Millisecond * 1000) + + values, err = handle.Read() + if err != nil { + t.Fatal(err) + } + + assert.EqualValues(t, 1, len(values)) // Assert all metrics have been grouped into a single event + + // Test all keys exist in the event + pctKey, err := values[0].MetricSetFields.HasKey("metrics.%_processor_time") + if err != nil { + t.Fatal(err) + } + assert.True(t, pctKey) + + pctKey, err = values[0].MetricSetFields.HasKey("metrics.%_user_time") + if err != nil { + t.Fatal(err) + } + assert.True(t, pctKey) + + pctKey, err = values[0].MetricSetFields.HasKey("metrics.%_privileged_time") + if err != nil { + t.Fatal(err) + } + assert.True(t, pctKey) + + t.Log(values) +} + +func TestGroupByInstanceDeprecated(t *testing.T) { + config := Config{ + Counters: make([]Counter, 3), GroupMeasurements: true, } - config.CounterConfig[0].InstanceLabel = "processor.name" - config.CounterConfig[0].MeasurementLabel = "processor.time.pct" - config.CounterConfig[0].Query = `\Processor Information(_Total)\% Processor Time` - config.CounterConfig[0].Format = "float" + config.Counters[0].InstanceLabel = "processor.name" + config.Counters[0].MeasurementLabel = "processor.time.pct" + config.Counters[0].Query = `\Processor Information(_Total)\% Processor Time` + config.Counters[0].Format = "float" - config.CounterConfig[1].InstanceLabel = "processor.name" - config.CounterConfig[1].MeasurementLabel = "processor.time.user.pct" - config.CounterConfig[1].Query = `\Processor Information(_Total)\% User Time` - config.CounterConfig[1].Format = "float" + config.Counters[1].InstanceLabel = "processor.name" + config.Counters[1].MeasurementLabel = "processor.time.user.pct" + config.Counters[1].Query = `\Processor Information(_Total)\% User Time` + config.Counters[1].Format = "float" - config.CounterConfig[2].InstanceLabel = "processor.name" - config.CounterConfig[2].MeasurementLabel = "processor.time.privileged.ns" - config.CounterConfig[2].Query = `\Processor Information(_Total)\% Privileged Time` - config.CounterConfig[2].Format = "float" + config.Counters[2].InstanceLabel = "processor.name" + config.Counters[2].MeasurementLabel = "processor.time.privileged.ns" + config.Counters[2].Query = `\Processor Information(_Total)\% Privileged Time` + config.Counters[2].Format = "float" handle, err := NewReader(config) if err != nil { diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index a7ecd6663621..c65c4a8118ae 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -22,30 +22,42 @@ package perfmon import ( "fmt" "regexp" - "strconv" "strings" + "unicode" "github.com/elastic/beats/v7/metricbeat/helper/windows/pdh" "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/metricbeat/mb" ) -var processRegexp = regexp.MustCompile(`(.+?)#[1-9]+`) - -const instanceCountLabel = ":count" +const ( + instanceCountLabel = ":count" + defaultInstanceField = "instance" + defaultObjectField = "object" + replaceUpperCaseRegex = `(?:[^A-Z_\W])([A-Z])[^A-Z]` +) // Reader will contain the config options type Reader struct { - query pdh.Query // PDH Query - instanceLabel map[string]string // Mapping of counter path to key used for the label (e.g. processor.name) - measurement map[string]string // Mapping of counter path to key used for the value (e.g. processor.cpu_time). - executed bool // Indicates if the query has been executed. - log *logp.Logger // - config Config // Metricset configuration + query pdh.Query // PDH Query + executed bool // Indicates if the query has been executed. + log *logp.Logger // + config Config // Metricset configuration + counters []PerfCounter +} + +type PerfCounter struct { + InstanceField string + InstanceName string + QueryField string + QueryName string + Format string + ObjectName string + ObjectField string + ChildQueries []string } // NewReader creates a new instance of Reader. @@ -55,44 +67,43 @@ func NewReader(config Config) (*Reader, error) { return nil, err } r := &Reader{ - query: query, - instanceLabel: map[string]string{}, - measurement: map[string]string{}, - log: logp.NewLogger("perfmon"), - config: config, + query: query, + log: logp.NewLogger("perfmon"), + config: config, } - for _, counter := range config.CounterConfig { - childQueries, err := query.GetCounterPaths(counter.Query) + r.mapCounters(config) + for i, counter := range r.counters { + r.counters[i].ChildQueries = []string{} + childQueries, err := query.GetCounterPaths(counter.QueryName) if err != nil { if config.IgnoreNECounters { switch err { case pdh.PDH_CSTATUS_NO_COUNTER, pdh.PDH_CSTATUS_NO_COUNTERNAME, pdh.PDH_CSTATUS_NO_INSTANCE, pdh.PDH_CSTATUS_NO_OBJECT: r.log.Infow("Ignoring non existent counter", "error", err, - logp.Namespace("perfmon"), "query", counter.Query) + logp.Namespace("perfmon"), "query", counter.QueryName) continue } } else { query.Close() - return nil, errors.Wrapf(err, `failed to expand counter (query="%v")`, counter.Query) + return nil, errors.Wrapf(err, `failed to expand counter (query="%v")`, counter.QueryName) } } // check if the pdhexpandcounterpath/pdhexpandwildcardpath functions have expanded the counter successfully. if len(childQueries) == 0 || (len(childQueries) == 1 && strings.Contains(childQueries[0], "*")) { // covering cases when PdhExpandWildCardPathW returns no counter paths or is unable to expand and the ignore_non_existent_counters flag is set if config.IgnoreNECounters { - r.log.Infow("Ignoring non existent counter", "initial query", counter.Query, + r.log.Infow("Ignoring non existent counter", "initial query", counter.QueryName, logp.Namespace("perfmon"), "expanded query", childQueries) continue } - return nil, errors.Errorf(`failed to expand counter (query="%v")`, counter.Query) + return nil, errors.Errorf(`failed to expand counter (query="%v"), no error returned`, counter.QueryName) } for _, v := range childQueries { if err := query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { - return nil, errors.Wrapf(err, `failed to add counter (query="%v")`, counter.Query) + return nil, errors.Wrapf(err, `failed to add counter (query="%v")`, counter.QueryName) } - r.instanceLabel[v] = counter.InstanceLabel - r.measurement[v] = counter.MeasurementLabel + r.counters[i].ChildQueries = append(r.counters[i].ChildQueries, v) } } return r, nil @@ -101,19 +112,20 @@ func NewReader(config Config) (*Reader, error) { // RefreshCounterPaths will recheck for any new instances and add them to the counter list func (re *Reader) RefreshCounterPaths() error { var newCounters []string - for _, counter := range re.config.CounterConfig { - childQueries, err := re.query.GetCounterPaths(counter.Query) + for i, counter := range re.counters { + re.counters[i].ChildQueries = []string{} + childQueries, err := re.query.GetCounterPaths(counter.QueryName) if err != nil { if re.config.IgnoreNECounters { switch err { case pdh.PDH_CSTATUS_NO_COUNTER, pdh.PDH_CSTATUS_NO_COUNTERNAME, pdh.PDH_CSTATUS_NO_INSTANCE, pdh.PDH_CSTATUS_NO_OBJECT: re.log.Infow("Ignoring non existent counter", "error", err, - logp.Namespace("perfmon"), "query", counter.Query) + logp.Namespace("perfmon"), "query", counter.QueryName) continue } } else { - return errors.Wrapf(err, `failed to expand counter (query="%v")`, counter.Query) + return errors.Wrapf(err, `failed to expand counter (query="%v")`, counter.QueryName) } } newCounters = append(newCounters, childQueries...) @@ -121,10 +133,9 @@ func (re *Reader) RefreshCounterPaths() error { if err == nil && len(childQueries) >= 1 && !strings.Contains(childQueries[0], "*") { for _, v := range childQueries { if err := re.query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { - return errors.Wrapf(err, "failed to add counter (query='%v')", counter.Query) + return errors.Wrapf(err, "failed to add counter (query='%v')", counter.QueryName) } - re.instanceLabel[v] = counter.InstanceLabel - re.measurement[v] = counter.MeasurementLabel + re.counters[i].ChildQueries = append(re.counters[i].ChildQueries, v) } } } @@ -152,7 +163,7 @@ func (re *Reader) Read() ([]mb.Event, error) { var events []mb.Event // GroupAllCountersTo config option where counters for all instances are aggregated and instance count is added in the event under the string value provided by this option. if re.config.GroupAllCountersTo != "" { - event := re.groupToEvent(values) + event := re.groupToSingleEvent(values) events = append(events, event) } else { events = re.groupToEvents(values) @@ -161,114 +172,134 @@ func (re *Reader) Read() ([]mb.Event, error) { return events, nil } -func (re *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.Event { - eventMap := make(map[string]*mb.Event) - - for counterPath, values := range counters { - for ind, val := range values { - // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. - // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if val.Err != nil && !re.executed { - re.log.Debugw("Ignoring the first measurement because the data isn't ready", - "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) - continue - } +// Close will close the PDH query for now. +func (re *Reader) Close() error { + return re.query.Close() +} - var eventKey string - if re.config.GroupMeasurements && val.Err == nil { - // Send measurements with the same instance label as part of the same event - eventKey = val.Instance - } else { - // Send every measurement as an individual event - // If a counter contains an error, it will always be sent as an individual event - eventKey = counterPath + strconv.Itoa(ind) +func (re *Reader) getCounter(query string) (bool, PerfCounter) { + for _, counter := range re.counters { + for _, childQuery := range counter.ChildQueries { + if childQuery == query { + return true, counter } + } + } + return false, PerfCounter{} +} - // Create a new event if the key doesn't exist in the map - if _, ok := eventMap[eventKey]; !ok { - eventMap[eventKey] = &mb.Event{ - MetricSetFields: common.MapStr{}, - Error: errors.Wrapf(val.Err, "failed on query=%v", counterPath), - } - if val.Instance != "" && re.instanceLabel[counterPath] != "" { - //will ignore instance counter - if ok, match := matchesParentProcess(val.Instance); ok { - eventMap[eventKey].MetricSetFields.Put(re.instanceLabel[counterPath], match) - } else { - eventMap[eventKey].MetricSetFields.Put(re.instanceLabel[counterPath], val.Instance) +func (re *Reader) mapCounters(config Config) { + re.counters = []PerfCounter{} + if len(config.Counters) > 0 { + for _, counter := range config.Counters { + re.counters = append(re.counters, PerfCounter{ + InstanceField: counter.InstanceLabel, + InstanceName: counter.InstanceName, + QueryField: counter.MeasurementLabel, + QueryName: counter.Query, + Format: counter.Format, + ChildQueries: nil, + }) + } + } + if len(config.Queries) > 0 { + for _, query := range config.Queries { + for _, counter := range query.Counters { + // counter paths can also not contain any instances + if len(query.Instance) == 0 { + re.counters = append(re.counters, PerfCounter{ + InstanceField: defaultInstanceField, + InstanceName: "", + QueryField: mapCounterPathLabel(query.Namespace, counter.Field, counter.Name), + QueryName: mapQuery(query.Name, "", counter.Name), + Format: counter.Format, + ObjectName: query.Name, + ObjectField: mapObjectName(query.Field), + }) + } else { + for _, instance := range query.Instance { + re.counters = append(re.counters, PerfCounter{ + InstanceField: defaultInstanceField, + InstanceName: instance, + QueryField: mapCounterPathLabel(query.Namespace, counter.Field, counter.Name), + QueryName: mapQuery(query.Name, instance, counter.Name), + Format: counter.Format, + ObjectName: query.Name, + ObjectField: mapObjectName(query.Field), + }) } } } - event := eventMap[eventKey] - if val.Measurement != nil { - event.MetricSetFields.Put(re.measurement[counterPath], val.Measurement) - } else { - event.MetricSetFields.Put(re.measurement[counterPath], 0) - } } } - // Write the values into the map. - events := make([]mb.Event, 0, len(eventMap)) - for _, val := range eventMap { - events = append(events, *val) - } - return events } -func (re *Reader) groupToEvent(counters map[string][]pdh.CounterValue) mb.Event { - event := mb.Event{ - MetricSetFields: common.MapStr{}, +func mapObjectName(objectField string) string { + if objectField != "" { + return objectField } - measurements := make(map[string]float64, 0) - for counterPath, values := range counters { - for _, val := range values { - // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. - // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if val.Err != nil && !re.executed { - re.log.Debugw("Ignoring the first measurement because the data isn't ready", - "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) - continue - } - var counterVal float64 - switch val.Measurement.(type) { - case int64: - counterVal = float64(val.Measurement.(int64)) - default: - counterVal = val.Measurement.(float64) - } - if _, ok := measurements[re.measurement[counterPath]]; !ok { - measurements[re.measurement[counterPath]] = counterVal - measurements[re.measurement[counterPath]+instanceCountLabel] = 1 - } else { - measurements[re.measurement[counterPath]+instanceCountLabel] = measurements[re.measurement[counterPath]+instanceCountLabel] + 1 - measurements[re.measurement[counterPath]] = measurements[re.measurement[counterPath]] + counterVal - } - } + return defaultObjectField +} + +func mapQuery(obj string, instance string, path string) string { + var query string + // trim object + obj = strings.TrimPrefix(obj, "\\") + obj = strings.TrimSuffix(obj, "\\") + query = fmt.Sprintf("\\%s", obj) + + if instance != "" { + // trim instance + instance = strings.TrimPrefix(instance, "(") + instance = strings.TrimSuffix(instance, ")") + query += fmt.Sprintf("(%s)", instance) } - for key, val := range measurements { - if strings.Contains(key, instanceCountLabel) { - if val == 1 { - continue - } else { - event.MetricSetFields.Put(fmt.Sprintf("%s.%s", strings.Split(key, ".")[0], re.config.GroupAllCountersTo), val) - } - } else { - event.MetricSetFields.Put(key, val) - } + + if strings.HasPrefix(path, "\\") { + query += path + } else { + query += fmt.Sprintf("\\%s", path) } - return event + return query } -// Close will close the PDH query for now. -func (re *Reader) Close() error { - return re.query.Close() -} +func mapCounterPathLabel(namespace string, label string, path string) string { + if label == "" { + label = path + } + // replace spaces with underscores + // replace backslashes with "per" + // replace actual percentage symbol with the symbol "pct" + r := strings.NewReplacer(" ", "_", "/sec", "_per_sec", "/_sec", "_per_sec", "\\", "_", "_%_", "_pct_", ":", "_", "_-_", "_") + label = r.Replace(label) + // replace uppercases with underscores + label = replaceUpperCase(label) -// matchParentProcess will try to get the parent process name -func matchesParentProcess(instanceName string) (bool, string) { - matches := processRegexp.FindStringSubmatch(instanceName) - if len(matches) == 2 { - return true, matches[1] + // avoid cases as this "logicaldisk_avg._disk_sec_per_transfer" + obj := strings.Split(label, ".") + for index := range obj { + // in some cases a trailing "_" is found + obj[index] = strings.TrimPrefix(obj[index], "_") + obj[index] = strings.TrimSuffix(obj[index], "_") } - return false, instanceName + label = strings.ToLower(strings.Join(obj, "_")) + label = strings.Replace(label, "__", "_", -1) + return namespace + "." + label +} + +// replaceUpperCase func will replace upper case with '_' +func replaceUpperCase(src string) string { + replaceUpperCaseRegexp := regexp.MustCompile(replaceUpperCaseRegex) + return replaceUpperCaseRegexp.ReplaceAllStringFunc(src, func(str string) string { + var newStr string + for _, r := range str { + // split into fields based on class of unicode character + if unicode.IsUpper(r) { + newStr += "_" + strings.ToLower(string(r)) + } else { + newStr += string(r) + } + } + return newStr + }) } diff --git a/metricbeat/module/windows/perfmon/reader_integration_test.go b/metricbeat/module/windows/perfmon/reader_integration_test.go new file mode 100644 index 000000000000..fd19b1e5c091 --- /dev/null +++ b/metricbeat/module/windows/perfmon/reader_integration_test.go @@ -0,0 +1,86 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration +// +build windows + +package perfmon + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var validQuery = `\Processor Information(_Total)\% Processor Time` + +// TestNewReaderWhenQueryPathNotProvided will check for invalid/no query. +func TestNewReaderWhenQueryPathNotProvided(t *testing.T) { + counter := Counter{Format: "float", InstanceName: "TestInstanceName"} + config := Config{ + IgnoreNECounters: false, + GroupMeasurements: false, + Counters: []Counter{counter}, + } + reader, err := NewReader(config) + assert.NotNil(t, err) + assert.Nil(t, reader) + assert.EqualValues(t, err.Error(), `failed to expand counter (query=""): no query path given`) +} + +// TestNewReaderWithValidQueryPath should successfully instantiate the reader. +func TestNewReaderWithValidQueryPath(t *testing.T) { + counter := Counter{Format: "float", InstanceName: "TestInstanceName", Query: validQuery} + config := Config{ + IgnoreNECounters: false, + GroupMeasurements: false, + Counters: []Counter{counter}, + } + reader, err := NewReader(config) + defer reader.Close() + assert.Nil(t, err) + assert.NotNil(t, reader) + assert.NotNil(t, reader.query) + assert.NotNil(t, reader.query.Handle) + assert.NotNil(t, reader.query.Counters) + assert.NotZero(t, len(reader.query.Counters)) + +} + +// TestReadSuccessfully will test the func read when it first retrieves no events (and ignored) and then starts retrieving events. +func TestReadSuccessfully(t *testing.T) { + counter := Counter{Format: "float", InstanceName: "TestInstanceName", Query: validQuery} + config := Config{ + IgnoreNECounters: false, + GroupMeasurements: false, + Counters: []Counter{counter}, + } + reader, err := NewReader(config) + if err != nil { + t.Fatal(err) + } + //Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we call reader.Read() twice. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + events, err := reader.Read() + assert.Nil(t, err) + assert.NotNil(t, events) + assert.Zero(t, len(events)) + events, err = reader.Read() + assert.Nil(t, err) + assert.NotNil(t, events) + assert.NotZero(t, len(events)) +} diff --git a/metricbeat/module/windows/perfmon/reader_test.go b/metricbeat/module/windows/perfmon/reader_test.go index 7f925d21a623..697281b64e65 100644 --- a/metricbeat/module/windows/perfmon/reader_test.go +++ b/metricbeat/module/windows/perfmon/reader_test.go @@ -23,63 +23,138 @@ import ( "testing" "github.com/stretchr/testify/assert" -) -var validQuery = `\Processor Information(_Total)\% Processor Time` + "github.com/elastic/beats/v7/metricbeat/helper/windows/pdh" +) -// TestNewReaderWhenQueryPathNotProvided will check for invalid/no query. -func TestNewReaderWhenQueryPathNotProvided(t *testing.T) { - counter := CounterConfig{Format: "float", InstanceName: "TestInstanceName"} - config := Config{ - IgnoreNECounters: false, - GroupMeasurements: false, - CounterConfig: []CounterConfig{counter}, +func TestGetCounter(t *testing.T) { + reader := Reader{ + query: pdh.Query{}, + executed: true, + log: nil, + counters: []PerfCounter{ + { + QueryField: "datagrams_sent_per_sec", + QueryName: `\UDPv4\Datagrams Sent/sec`, + Format: "float", + ObjectName: "UDPv4", + ObjectField: "object", + ChildQueries: []string{`\UDPv4\Datagrams Sent/sec`}, + }, + }, } - reader, err := NewReader(config) - assert.NotNil(t, err) - assert.Nil(t, reader) - assert.EqualValues(t, err.Error(), `failed to expand counter (query=""): no query path given`) + ok, val := reader.getCounter(`\UDPv4\Datagrams Sent/sec`) + assert.True(t, ok) + assert.Equal(t, val.QueryField, "datagrams_sent_per_sec") + assert.Equal(t, val.ObjectName, "UDPv4") + } -// TestNewReaderWithValidQueryPath should successfully instantiate the reader. -func TestNewReaderWithValidQueryPath(t *testing.T) { - counter := CounterConfig{Format: "float", InstanceName: "TestInstanceName", Query: validQuery} +func TestMapCounters(t *testing.T) { config := Config{ IgnoreNECounters: false, GroupMeasurements: false, - CounterConfig: []CounterConfig{counter}, + Counters: []Counter{ + { + InstanceLabel: "physical_disk.name", + InstanceName: "total", + MeasurementLabel: "physical_disk.write.time.pct", + Query: `\PhysicalDisk(*)\% Disk Write Time`, + Format: "float", + }, + }, + Queries: []Query{ + { + Name: "Process", + Namespace: "metrics", + Instance: []string{"svchost*"}, + Counters: []QueryCounter{ + { + Name: "% Processor Time", + Format: "float", + }, + }, + }, + { + Name: "Process", + Field: "disk", + Namespace: "metrics", + Instance: []string{"conhost*"}, + Counters: []QueryCounter{ + { + Name: "IO Read Operations/sec", + Field: "read_ops", + Format: "double", + }, + }, + }, + }, + } + reader := Reader{} + reader.mapCounters(config) + assert.Equal(t, len(reader.counters), 3) + for _, readerCounter := range reader.counters { + if readerCounter.InstanceField == "physical_disk.name" { + assert.Equal(t, readerCounter.InstanceName, "total") + assert.Equal(t, readerCounter.ObjectName, "") + assert.Equal(t, readerCounter.ObjectField, "") + assert.Equal(t, readerCounter.QueryField, "physical_disk.write.time.pct") + assert.Equal(t, readerCounter.QueryName, `\PhysicalDisk(*)\% Disk Write Time`) + assert.Equal(t, len(readerCounter.ChildQueries), 0) + assert.Equal(t, readerCounter.Format, "float") + } else if readerCounter.InstanceName == "svchost*" { + assert.Equal(t, readerCounter.ObjectName, "Process") + assert.Equal(t, readerCounter.ObjectField, "object") + assert.Equal(t, readerCounter.QueryField, "metrics.%_processor_time") + assert.Equal(t, readerCounter.QueryName, `\Process(svchost*)\% Processor Time`) + assert.Equal(t, len(readerCounter.ChildQueries), 0) + assert.Equal(t, readerCounter.Format, "float") + } else { + assert.Equal(t, readerCounter.InstanceName, "conhost*") + assert.Equal(t, readerCounter.ObjectName, "Process") + assert.Equal(t, readerCounter.ObjectField, "disk") + assert.Equal(t, readerCounter.QueryField, "metrics.read_ops") + assert.Equal(t, readerCounter.QueryName, `\Process(conhost*)\IO Read Operations/sec`) + assert.Equal(t, len(readerCounter.ChildQueries), 0) + assert.Equal(t, readerCounter.Format, "double") + } } - reader, err := NewReader(config) - defer reader.Close() - assert.Nil(t, err) - assert.NotNil(t, reader) - assert.NotNil(t, reader.query) - assert.NotNil(t, reader.query.Handle) - assert.NotNil(t, reader.query.Counters) - assert.NotZero(t, len(reader.query.Counters)) +} + +func TestMapQuery(t *testing.T) { + //mapQuery(obj string, instance string, path string) string { + obj := "Process" + instance := "*" + path := "% Processor Time" + result := mapQuery(obj, instance, path) + assert.Equal(t, result, `\Process(*)\% Processor Time`) + obj = `\Process\` + instance = "(*" + result = mapQuery(obj, instance, path) + assert.Equal(t, result, `\Process(*)\% Processor Time`) } -// TestReadSuccessfully will test the func read when it first retrieves no events (and ignored) and then starts retrieving events. -func TestReadSuccessfully(t *testing.T) { - counter := CounterConfig{Format: "float", InstanceName: "TestInstanceName", Query: validQuery} - config := Config{ - IgnoreNECounters: false, - GroupMeasurements: false, - CounterConfig: []CounterConfig{counter}, - } - reader, err := NewReader(config) - if err != nil { - t.Fatal(err) - } - //Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we call reader.Read() twice. - // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - events, err := reader.Read() - assert.Nil(t, err) - assert.NotNil(t, events) - assert.Zero(t, len(events)) - events, err = reader.Read() - assert.Nil(t, err) - assert.NotNil(t, events) - assert.NotZero(t, len(events)) +func TestMapCounterPathLabel(t *testing.T) { + result := mapCounterPathLabel("metrics", "", `WININET: Bytes from server`) + assert.Equal(t, result, "metrics.wininet_bytes_from_server") + result = mapCounterPathLabel("metrics", "", `RSC Coalesced Packet Bucket 5 (16To31)`) + assert.Equal(t, result, "metrics.rsc_coalesced_packet_bucket_5_(16_to31)") + result = mapCounterPathLabel("metrics", "", `Total Memory Usage --- Non-Paged Pool`) + assert.Equal(t, result, "metrics.total_memory_usage_---_non-paged_pool") + result = mapCounterPathLabel("metrics", "", `IPv6 NBLs/sec indicated with low-resource flag`) + assert.Equal(t, result, "metrics.ipv6_nbls_per_sec_indicated_with_low-resource_flag") + result = mapCounterPathLabel("metrics", "", `Queued Poison Messages Per Second`) + assert.Equal(t, result, "metrics.queued_poison_messages_per_second") + result = mapCounterPathLabel("metrics", "", `I/O Log Writes Average Latency`) + assert.Equal(t, result, "metrics.i/o_log_writes_average_latency") + result = mapCounterPathLabel("metrics", "io.logwrites.average latency", `I/O Log Writes Average Latency`) + assert.Equal(t, result, "metrics.io_logwrites_average_latency") + + result = mapCounterPathLabel("metrics", "this.is__exceptional-test:case/sec", `RSC Coalesced Packet Bucket 5 (16To31)`) + assert.Equal(t, result, "metrics.this_is_exceptional-test_case_per_sec") + + result = mapCounterPathLabel("metrics", "logicaldisk_avg._disk_sec_per_transfer", `RSC Coalesced Packet Bucket 5 (16To31)`) + assert.Equal(t, result, "metrics.logicaldisk_avg_disk_sec_per_transfer") + } diff --git a/metricbeat/modules.d/windows.yml.disabled b/metricbeat/modules.d/windows.yml.disabled index e699782c74b5..9aec0febd776 100644 --- a/metricbeat/modules.d/windows.yml.disabled +++ b/metricbeat/modules.d/windows.yml.disabled @@ -2,24 +2,19 @@ # Docs: https://www.elastic.co/guide/en/beats/metricbeat/7.x/metricbeat-module-windows.html - module: windows - #metricsets: - # - service + metricsets: + - service period: 1m #- module: windows # metricsets: -# - perfmon +# - perfmon # period: 10s -# perfmon.counters: -# - instance_label: processor.name -# instance_name: total -# measurement_label: processor.time.total.pct -# query: '\Processor Information(_Total)\% Processor Time' -# -# - instance_label: physical_disk.name -# measurement_label: physical_disk.write.per_sec -# query: '\PhysicalDisk(*)\Disk Writes/sec' -# -# - instance_label: physical_disk.name -# measurement_label: physical_disk.write.time.pct -# query: '\PhysicalDisk(*)\% Disk Write Time' +# perfmon.queries: +# - object: 'Process' +# instance: ["*"] +# counters: +# - name: 'Disk Writes/sec' +# field: physical_disk.write.per_sec +# format: "float" +# - name: "% Disk Write Time" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 9383e7dc34b1..20e660fb4696 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1257,11 +1257,14 @@ metricbeat.modules: period: 10s perfmon.ignore_non_existent_counters: false perfmon.group_measurements_by_instance: false - perfmon.counters: - # - instance_label: processor.name - # instance_name: total - # measurement_label: processor.time.total.pct - # query: '\Processor Information(_Total)\% Processor Time' + perfmon.queries: +# - object: 'Process' +# instance: ["*"] +# counters: +# - name: 'Disk Writes/sec' +# field: physical_disk.write.per_sec +# format: "float" +# - name: "% Disk Write Time" - module: windows metricsets: ["service"] diff --git a/x-pack/metricbeat/module/iis/fields.go b/x-pack/metricbeat/module/iis/fields.go index 8a124fdbef0d..18bad877cf93 100644 --- a/x-pack/metricbeat/module/iis/fields.go +++ b/x-pack/metricbeat/module/iis/fields.go @@ -19,5 +19,5 @@ func init() { // AssetIis returns asset data. // This is the base64 encoded gzipped contents of module/iis. func AssetIis() string { - return "eJy0kbGOgzAQRHt/xYgSKfkAF/cryMAm2ovBlu1cxN+ffAYOiEE02YJidzTzzFzwoEGC2QsgcNAkUTD7QgAt+caxDWx6iS8BIOrQmfapSQCONClPEjUFJQRwY9Ktl3/KC3rV0eQcJwyWJO7OPO24yQSsTZZGylrNjYriyhqjZ0HOOc4a73+fTU1zkLHFWqLF7+owIT1oeBnXbm4HABsIRIi1/ZT5otqT+yF3La/liVcnIFN/UxMW67So0vWmjdo5Vp2ylvv7qCzK4twfnTHnbfYlHOgTfb5bv9e4W+J+hYcFjpnJ7zcAAP//navhqA==" + return "eJzMkrGOgzAQRHt/xYgSKfkAF/cryMAm2ovBlr25iL8/+QwcIEAp44JidjTzFu0FDxo0mKMChMWSRsEcCwW0FJvAXtj1Gl8KQPKhc+3TkgICWTKRNGoSoxRwY7Jt1H/OC3rT0ZScngyeNO7BPf2o7BSsQ5ZBxnvLjUnmyjtnZ8NecnprvH99tzW/k44t1hItfVeDCelBw8uFdjM7AdhAIEGs46fOF9WRwg+Fa3kt39g6A7n6mxpZyFmo8vRmnTkYVp3xnvv76CzK4r0/OmPO6u4mLPT5e7DQ6UEcnsPxMZyewtiZ834DAAD///e79wo=" } diff --git a/x-pack/metricbeat/module/iis/webserver/manifest.yml b/x-pack/metricbeat/module/iis/webserver/manifest.yml index c8d148a21c13..32da4b384b60 100644 --- a/x-pack/metricbeat/module/iis/webserver/manifest.yml +++ b/x-pack/metricbeat/module/iis/webserver/manifest.yml @@ -6,158 +6,81 @@ input: perfmon.group_measurements_by_instance: true perfmon.ignore_non_existent_counters: true perfmon.group_all_counter: "worker_process_count" - perfmon.counters: - #network - - instance_label: '' - measurement_label: network.total_bytes_received - query: '\Web Service(_Total)\Total Bytes Received' - - instance_label: '' - measurement_label: network.total_bytes_sent - query: '\Web Service(_Total)\Total Bytes Sent' - - instance_label: '' - measurement_label: network.bytes_sent_per_sec - query: '\Web Service(_Total)\Bytes Sent/sec' - - instance_label: '' - measurement_label: network.bytes_received_per_sec - query: '\Web Service(_Total)\Bytes Received/sec' - - instance_label: '' - measurement_label: network.current_connections - query: '\Web Service(_Total)\Current Connections' - - instance_label: '' - measurement_label: network.maximum_connections - query: '\Web Service(_Total)\Maximum Connections' - - instance_label: '' - measurement_label: network.total_connection_attempts - query: '\Web Service(_Total)\Total Connection Attempts (all instances)' - - instance_label: '' - measurement_label: network.total_get_requests - query: '\Web Service(_Total)\Total Get Requests' - - instance_label: '' - measurement_label: network.get_requests_per_sec - query: '\Web Service(_Total)\Get Requests/sec' - - instance_label: '' - measurement_label: network.total_post_requests - query: '\Web Service(_Total)\Total Post Requests' - - instance_label: '' - measurement_label: network.post_requests_per_sec - query: '\Web Service(_Total)\Post Requests/sec' - - instance_label: '' - measurement_label: network.total_delete_requests - query: '\Web Service(_Total)\Total Delete Requests' - - instance_label: '' - measurement_label: network.delete_requests_per_sec - query: '\Web Service(_Total)\Delete Requests/sec' - - instance_label: '' - measurement_label: network.service_uptime - query: '\Web Service(_Total)\Service Uptime' - - instance_label: '' - measurement_label: network.current_anonymous_users - query: '\Web Service(_Total)\Current Anonymous Users' - - instance_label: '' - measurement_label: network.current_nonanonymous_users - query: '\Web Service(_Total)\Current NonAnonymous Users' - - instance_label: '' - measurement_label: network.total_anonymous_users - query: '\Web Service(_Total)\Total Anonymous Users' - - instance_label: '' - measurement_label: network.anonymous_users_per_sec - query: '\Web Service(_Total)\Anonymous Users/sec' - - instance_label: '' - measurement_label: network.total_nonanonymous_users - query: '\Web Service(_Total)\Total NonAnonymous Users' - + perfmon.queries: + - object: "Web Service" + instance: "_Total" + namespace : "network" + counters: + # network + - name: "Total Bytes Received" + - name: "Total Bytes Sent" + - name: "Bytes Sent/sec" + - name: "Bytes Received/sec" + - name: "Current Connections" + - name: "Maximum Connections" + - name: "Total Connection Attempts (all instances)" + field: "total_connection_attempts" + - name: "Total Get Requests" + - name: "Get Requests/sec" + - name: "Total Post Requests" + - name: "Post Requests/sec" + - name: "Total Delete Requests" + - name: "Delete Requests/sec" + - name: "Service Uptime" + - name: "Current Anonymous Users" + - name: "Current NonAnonymous Users" + - name: "Total Anonymous Users" + - name: "Anonymous Users/sec" + - name: "Total NonAnonymous Users" #asp.net - - instance_label: '' - measurement_label: asp_net_application.errors_per_sec - query: '\ASP.NET Applications(__Total__)\Errors Total/Sec' - - instance_label: '' - measurement_label: asp_net_application.pipeline_instance_count - query: '\ASP.NET Applications(__Total__)\Pipeline Instance Count' - - instance_label: '' - measurement_label: asp_net_application.requests_executing - query: '\ASP.NET Applications(__Total__)\Requests Executing' - - instance_label: '' - measurement_label: asp_net_application.requests_in_application_queue - query: '\ASP.NET Applications(__Total__)\Requests in Application Queue' - format: 'large' - - instance_label: '' - measurement_label: asp_net_application.requests_per_sec - query: '\ASP.NET Applications(__Total__)\Requests/Sec' - - instance_label: '' - measurement_label: asp_net.application_restarts - query: '\ASP.NET\Application Restarts' - - instance_label: '' - measurement_label: asp_net.request_wait_time - query: '\ASP.NET\Request Wait Time' + - object: "ASP.NET Applications" + instance: "__Total__" + namespace: "asp_net" + counters: + - name: "Application Restarts" + - name: "Request Wait Time" + #asp_net_application + - object: "ASP.NET Applications" + instance: "__Total__" + namespace: "asp_net_application" + counters: + - name: "Errors Total/Sec" + - name: "Pipeline Instance Count" + - name: "Requests Executing" + - name: "Requests in Application Queue" + format: 'large' + - name: "Requests/Sec" #cache - - instance_label: '' - measurement_label: cache.current_files_cached - query: '\Web Service Cache\Current Files Cached' - - instance_label: '' - measurement_label: cache.total_files_cached - query: '\Web Service Cache\Total Files Cached' - - instance_label: '' - measurement_label: cache.file_cache_hits - query: '\Web Service Cache\File Cache Hits' - - instance_label: '' - measurement_label: cache.file_cache_misses - query: '\Web Service Cache\File Cache Misses' - - instance_label: '' - measurement_label: cache.current_file_cache_memory_usage - query: '\Web Service Cache\Current File Cache Memory Usage' - - instance_label: '' - measurement_label: cache.maximum_file_cache_memory_usage - query: '\Web Service Cache\Maximum File Cache Memory Usage' - - instance_label: '' - measurement_label: cache.current_uris_cached - query: '\Web Service Cache\Current URIs Cached' - - instance_label: '' - measurement_label: cache.total_uris_cached - query: '\Web Service Cache\Total URIs Cached' - - instance_label: '' - measurement_label: cache.uri_cache_hits - query: '\Web Service Cache\URI Cache Hits' - - instance_label: '' - measurement_label: cache.uri_cache_misses - query: '\Web Service Cache\URI Cache Misses' - - instance_label: '' - measurement_label: cache.output_cache_current_memory_usage - query: '\Web Service Cache\Output Cache Current Memory Usage' - - instance_label: '' - measurement_label: cache.output_cache_current_items - query: '\Web Service Cache\Output Cache Current Items' - - instance_label: '' - measurement_label: cache.output_cache_total_hits - query: '\Web Service Cache\Output Cache Total Hits' - - instance_label: '' - measurement_label: cache.output_cache_total_misses - query: '\Web Service Cache\Output Cache Total Misses' + - object: "Web Service Cache" + namespace: "cache" + counters: + - name: "Current Files Cached" + - name: "Total Files Cached" + - name: "File Cache Hits" + - name: "File Cache Misses" + - name: "Current File Cache Memory Usage" + - name: "Maximum File Cache Memory Usage" + - name: "Current URIs Cached" + - name: "Total URIs Cached" + - name: "URI Cache Hits" + - name: "URI Cache Misses" + - name: "Output Cache Current Memory Usage" + - name: "Output Cache Current Items" + - name: "Output Cache Total Hits" + - name: "Output Cache Total Misses" #process - - instance_label: '' - measurement_label: process.cpu_usage_perc - query: '\Process(w3wp*)\% Processor Time' - - instance_label: '' - measurement_label: process.handle_count - query: '\Process(w3wp*)\Handle Count' - - instance_label: '' - measurement_label: process.thread_count - query: '\Process(w3wp*)\Thread Count' - - instance_label: '' - measurement_label: process.working_set - query: '\Process(w3wp*)\Working Set' - - instance_label: '' - measurement_label: process.private_byte - query: '\Process(w3wp*)\Private Bytes' - - instance_label: '' - measurement_label: process.virtual_bytes - query: '\Process(w3wp*)\Virtual Bytes' - - instance_label: '' - measurement_label: process.page_faults_per_Sec - query: '\Process(w3wp*)\Page Faults/sec' - - instance_label: '' - measurement_label: process.io_read_operations_per_sec - query: '\Process(w3wp*)\IO Read Operations/sec' - - instance_label: '' - measurement_label: process.io_write_operations_per_sec - query: '\Process(w3wp*)\IO Write Operations/sec' + - object: "Process" + field: "process" + namespace: "process" + instance: "w3wp*" + counters: + - name: "% Processor Time" + - name: "Handle Count" + - name: "Thread Count" + - name: "Working Set" + - name: "Private Bytes" + - name: "Virtual Bytes" + - name: "Page Faults/sec" + - name: "IO Read Operations/sec" + - name: "IO Write Operations/sec" diff --git a/x-pack/metricbeat/module/iis/website/_meta/data.json b/x-pack/metricbeat/module/iis/website/_meta/data.json index d9804730f977..e2a6ed9905c2 100644 --- a/x-pack/metricbeat/module/iis/website/_meta/data.json +++ b/x-pack/metricbeat/module/iis/website/_meta/data.json @@ -7,17 +7,24 @@ }, "iis": { "website": { - "current_connections": 0, - "maximum_connections": 1, - "name": "test2.local", - "service_uptime": 346586, - "total_bytes_received": 1666, - "total_bytes_sent": 84224, - "total_connection_attempts": 2, - "total_delete_requests": 0, - "total_get_requests": 4, - "total_post_requests": 0, - "total_put_requests": 0 + "network": { + "bytes_sent_per_sec": 0, + "maximum_connections": 1, + "total_post_requests": 0, + "post_requests_per_sec": 0, + "total_connection_attempts": 1, + "service_uptime": 114161, + "get_requests_per_sec": 0, + "total_put_requests": 0, + "current_connections": 0, + "total_delete_requests": 0, + "bytes_received_per_sec": 0, + "put_requests_per_sec": 0, + "total_bytes_sent": 944, + "total_get_requests": 1, + "delete_requests_per_sec": 0 + }, + "name": "Default Web Site" } }, "metricset": { @@ -27,4 +34,4 @@ "service": { "type": "iis" } -} \ No newline at end of file +} diff --git a/x-pack/metricbeat/module/iis/website/_meta/fields.yml b/x-pack/metricbeat/module/iis/website/_meta/fields.yml index 74be20459303..1bd4ebc3accc 100644 --- a/x-pack/metricbeat/module/iis/website/_meta/fields.yml +++ b/x-pack/metricbeat/module/iis/website/_meta/fields.yml @@ -1,6 +1,8 @@ -- name: website - type: group +- name: website.*.* release: beta + type: object + object_type: float + object_type_mapping_type: "*" description: > website fields: diff --git a/x-pack/metricbeat/module/iis/website/manifest.yml b/x-pack/metricbeat/module/iis/website/manifest.yml index 564ad6db86ce..62ebfd27ed85 100644 --- a/x-pack/metricbeat/module/iis/website/manifest.yml +++ b/x-pack/metricbeat/module/iis/website/manifest.yml @@ -5,57 +5,41 @@ input: defaults: perfmon.group_measurements_by_instance: true perfmon.ignore_non_existent_counters: true - perfmon.counters: - #network - - instance_label: 'name' - measurement_label: total_bytes_received - query: '\Web Service(*)\Total Bytes Received' - - instance_label: 'name' - measurement_label: total_bytes_sent - query: '\Web Service(*)\Total Bytes Sent' - - instance_label: 'name' - measurement_label: bytes_sent_per_sec - query: '\Web Service(*)\Bytes Sent/sec' - - instance_label: 'name' - measurement_label: bytes_received_per_sec - query: '\Web Service(*)\Bytes Received/sec' - - instance_label: 'name' - measurement_label: current_connections - query: '\Web Service(*)\Current Connections' - - instance_label: 'name' - measurement_label: maximum_connections - query: '\Web Service(*)\Maximum Connections' - - instance_label: 'name' - measurement_label: total_connection_attempts - query: '\Web Service(*)\Total Connection Attempts (all instances)' - - instance_label: 'name' - measurement_label: total_get_requests - query: '\Web Service(*)\Total Get Requests' - - instance_label: 'name' - measurement_label: get_requests_per_sec - query: '\Web Service(*)\Get Requests/sec' - - instance_label: 'name' - measurement_label: total_post_requests - query: '\Web Service(*)\Total Post Requests' - - instance_label: 'name' - measurement_label: post_requests_per_sec - query: '\Web Service(*)\Post Requests/sec' - - instance_label: 'name' - measurement_label: total_delete_requests - query: '\Web Service(*)\Total Delete Requests' - - instance_label: 'name' - measurement_label: delete_requests_per_sec - query: '\Web Service(*)\Delete Requests/sec' - - instance_label: 'name' - measurement_label: service_uptime - query: '\Web Service(*)\Service Uptime' - - instance_label: 'name' - measurement_label: total_put_requests - query: '\Web Service(*)\Total PUT Requests' - - instance_label: 'name' - measurement_label: put_requests_per_sec - query: '\Web Service(*)\PUT Requests/sec' + perfmon.queries: + - object: 'Web Service' + instance: "*" + namespace : "network" + counters: + - name: 'Total Bytes Received' + - name: 'Total Bytes Sent' + - name: "Bytes Sent/sec" + - name: "Bytes Received/sec" + - name: "Current Connections" + - name: "Maximum Connections" + - name: "Total Connection Attempts (all instances)" + field: total_connection_attempts + - name: "Total Get Requests" + - name: "Get Requests/sec" + - name: "Total Post Requests" + - name: "Post Requests/sec" + - name: "Total Delete Requests" + - name: "Delete Requests/sec" + - name: "Service Uptime" + - name: "Total PUT Requests" + - name: "PUT Requests/sec" processors: - drop_event.when.equals: iis.website.name: '_Total' +- drop_fields: + fields: "iis.website.object" +- rename: + ignore_missing: true + fields: + - from: "iis.website.instance" + to: "iis.website.name" + + + + +