[8.17](backport #41965) [system/cpu][system/core] - New config for using performance counters

CHANGELOG.next.asciidoc

@@ -280,6 +280,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add support for region/zone for Vertex AI service in GCP module {pull}41551[41551]
 - Add support for location label as an optional configuration parameter in GCP metrics metricset. {issue}41550[41550] {pull}41626[41626]
 - Added `tier_preference`, `creation_date` and `version` fields to the `elasticsearch.index` metricset. {pull}41944[41944]
+- Add `use_performance_counters` to collect CPU metrics using performance counters on Windows for `system/cpu` and `system/core` {pull}41965[41965]
 
 *Metricbeat*
 

NOTICE.txt

@@ -13443,11 +13443,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.17.3
+Version: v0.17.4
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].3/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].4/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -13654,11 +13654,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-l
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-system-metrics
-Version: v0.11.4
+Version: v0.11.5
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].4/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].5/LICENSE.txt:
 
                                  Apache License
                            Version 2.0, January 2004

go.mod

@@ -191,8 +191,8 @@ require (
 	github.com/elastic/bayeux v1.0.5
 	github.com/elastic/ebpfevents v0.6.0
 	github.com/elastic/elastic-agent-autodiscover v0.9.0
-	github.com/elastic/elastic-agent-libs v0.17.3
-	github.com/elastic/elastic-agent-system-metrics v0.11.4
+	github.com/elastic/elastic-agent-libs v0.17.4
+	github.com/elastic/elastic-agent-system-metrics v0.11.5
 	github.com/elastic/go-elasticsearch/v8 v8.14.0
 	github.com/elastic/go-quark v0.2.0
 	github.com/elastic/go-sfdc v0.0.0-20241010131323-8e176480d727

go.sum

@@ -340,10 +340,10 @@ github.com/elastic/elastic-agent-autodiscover v0.9.0 h1:+iWIKh0u3e8I+CJa3FfWe9h0
 github.com/elastic/elastic-agent-autodiscover v0.9.0/go.mod h1:5iUxLHhVdaGSWYTveSwfJEY4RqPXTG13LPiFoxcpFd4=
 github.com/elastic/elastic-agent-client/v7 v7.15.0 h1:nDB7v8TBoNuD6IIzC3z7Q0y+7bMgXoT2DsHfolO2CHE=
 github.com/elastic/elastic-agent-client/v7 v7.15.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI=
-github.com/elastic/elastic-agent-libs v0.17.3 h1:q79P05dhQkc5REzieVkkD9oRKrnptKY4MC6Typ+d8bc=
-github.com/elastic/elastic-agent-libs v0.17.3/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M=
-github.com/elastic/elastic-agent-system-metrics v0.11.4 h1:Z/8CML5RKvGpi6/QUFok1K3EriBAv2kUAXnsk8hCifk=
-github.com/elastic/elastic-agent-system-metrics v0.11.4/go.mod h1:TTW2ysv78uHBQ68hG8TXiaX1m6f29ZHgGWb8XONYsU8=
+github.com/elastic/elastic-agent-libs v0.17.4 h1:kWK5Kn2EQjM97yHqbeXv+cFAIti4IiI9Qj8huM+lZzE=
+github.com/elastic/elastic-agent-libs v0.17.4/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M=
+github.com/elastic/elastic-agent-system-metrics v0.11.5 h1:JSjXFEn8uYZ9hoC/GxZNMgJ622UoP96sjYP/49/Uvuo=
+github.com/elastic/elastic-agent-system-metrics v0.11.5/go.mod h1:nzkrGajQA29YNcfP62gfzhxX9an3/xdQ3RmfQNw9YTI=
 github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA=
 github.com/elastic/elastic-transport-go/v8 v8.6.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk=
 github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4=

metricbeat/docs/modules/system.asciidoc

@@ -265,6 +265,11 @@ metricbeat.modules:
 
   # Filter systemd services based on a name pattern
   #service.pattern_filter: ["ssh*", "nfs*"]
+
+  # This option enables the use of performance counters to collect data for cpu/core metricset.
+  # Only effective for Windows.
+  # You should use this option if running beats on machins with more than 64 cores.
+  #use_performance_counters: true
 ----
 
 [float]

metricbeat/metricbeat.reference.yml

@@ -142,6 +142,11 @@ metricbeat.modules:
   # Filter systemd services based on a name pattern
   #service.pattern_filter: ["ssh*", "nfs*"]
 
+  # This option enables the use of performance counters to collect data for cpu/core metricset.
+  # Only effective for Windows.
+  # You should use this option if running beats on machins with more than 64 cores.
+  #use_performance_counters: true
+
 #------------------------------ Aerospike Module ------------------------------
 - module: aerospike
   metricsets: ["namespace"]

metricbeat/module/system/_meta/config.reference.yml

@@ -81,3 +81,8 @@
 
   # Filter systemd services based on a name pattern
   #service.pattern_filter: ["ssh*", "nfs*"]
+
+  # This option enables the use of performance counters to collect data for cpu/core metricset.
+  # Only effective for Windows.
+  # You should use this option if running beats on machins with more than 64 cores.
+  #use_performance_counters: true

metricbeat/module/system/core/_meta/docs.asciidoc

@@ -14,11 +14,16 @@ This metricset is available on:
 *`core.metrics`*:: This option controls what metrics are reported for each CPU
 core. The value is a list and two metric types are supported - `percentages` and
 `ticks`. The default value is `core.metrics: [percentages]`.
+*`use_performance_counters`*:: This option enables the use of performance counters to 
+collect data for the CPU/core metricset. It is only effective on Windows.
+You should use this option if running beats on machins with more than 64 cores.
+The default value is `use_performance_counters: true`
 +
 [source,yaml]
 ----
 metricbeat.modules:
 - module: system
   metricsets: [core]
   core.metrics: [percentages, ticks]
+  #use_performance_counters: true
 ----

metricbeat/module/system/core/config.go

@@ -34,8 +34,9 @@ const (
 
 // Config for the system core metricset.
 type Config struct {
-	Metrics  []string `config:"core.metrics"`
-	CPUTicks *bool    `config:"cpu_ticks"` // Deprecated.
+	Metrics                 []string `config:"core.metrics"`
+	CPUTicks                *bool    `config:"cpu_ticks"` // Deprecated.
+	UserPerformanceCounters bool     `config:"use_performance_counters"`
 }
 
 // Validate validates the core config.
@@ -65,5 +66,6 @@ func (c Config) Validate() (metrics.MetricOpts, error) {
 }
 
 var defaultConfig = Config{
-	Metrics: []string{percentages},
+	Metrics:                 []string{percentages},
+	UserPerformanceCounters: true,
 }

metricbeat/module/system/core/core.go

@@ -41,6 +41,7 @@ type MetricSet struct {
 	mb.BaseMetricSet
 	opts  metrics.MetricOpts
 	cores *metrics.Monitor
+	sys   resolve.Resolver
 }
 
 // New returns a new core MetricSet.
@@ -58,11 +59,25 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	if config.CPUTicks != nil && *config.CPUTicks {
 		config.Metrics = append(config.Metrics, "ticks")
 	}
-	sys := base.Module().(resolve.Resolver)
+	sys, ok := base.Module().(resolve.Resolver)
+	if !ok {
+		return nil, fmt.Errorf("unexpected module type: %T", base.Module())
+	}
+
+	cpuOpts := make([]metrics.OptionFunc, 0)
+	if config.UserPerformanceCounters {
+		cpuOpts = append(cpuOpts, metrics.WithWindowsPerformanceCounter())
+	}
+	cpu, err := metrics.New(sys, cpuOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error initializing system.cpu metricset: %w", err)
+	}
+
 	return &MetricSet{
 		BaseMetricSet: base,
 		opts:          opts,
-		cores:         metrics.New(sys),
+		cores:         cpu,
+		sys:           sys,
 	}, nil
 }
 
@@ -109,6 +124,5 @@ func (m *MetricSet) Diagnostics() []diagnostics.DiagnosticSetup {
 }
 
 func (m *MetricSet) getDiagData() []byte {
-	sys := m.BaseMetricSet.Module().(resolve.Resolver)
-	return diagnostics.GetRawFileOrErrorString(sys, "/proc/stat")
+	return diagnostics.GetRawFileOrErrorString(m.sys, "/proc/stat")
 }

metricbeat/module/system/cpu/_meta/docs.asciidoc

@@ -15,11 +15,16 @@ This metricset is available on:
 is a list and three metric types are supported - `percentages`,
 `normalized_percentages`, and `ticks`. The default value is
 `cpu.metrics: [percentages]`.
+*`use_performance_counters`*:: This option enables the use of performance counters to 
+collect data for the CPU/core metricset. It is only effective on Windows.
+You should use this option if running beats on machins with more than 64 cores.
+The default value is `use_performance_counters: true`
 +
 [source,yaml]
 ----
 metricbeat.modules:
 - module: system
   metricsets: [cpu]
   cpu.metrics: [percentages, normalized_percentages, ticks]
+  #use_performance_counters: true
 ----

metricbeat/module/system/cpu/config.go

@@ -35,8 +35,9 @@ const (
 
 // Config for the system cpu metricset.
 type Config struct {
-	Metrics  []string `config:"cpu.metrics"`
-	CPUTicks *bool    `config:"cpu_ticks"` // Deprecated.
+	Metrics                 []string `config:"cpu.metrics"`
+	CPUTicks                *bool    `config:"cpu_ticks"` // Deprecated.
+	UserPerformanceCounters bool     `config:"use_performance_counters"`
 }
 
 // Validate validates the cpu config.
@@ -69,5 +70,6 @@ func (c Config) Validate() (metrics.MetricOpts, error) {
 }
 
 var defaultConfig = Config{
-	Metrics: []string{percentages, normalizedPercentages},
+	Metrics:                 []string{percentages, normalizedPercentages},
+	UserPerformanceCounters: true,
 }

metricbeat/module/system/cpu/cpu.go

@@ -44,6 +44,7 @@ type MetricSet struct {
 	mb.BaseMetricSet
 	opts metrics.MetricOpts
 	cpu  *metrics.Monitor
+	sys  resolve.Resolver
 }
 
 // New is a mb.MetricSetFactory that returns a cpu.MetricSet.
@@ -61,11 +62,25 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	if config.CPUTicks != nil && *config.CPUTicks {
 		config.Metrics = append(config.Metrics, "ticks")
 	}
-	sys := base.Module().(resolve.Resolver)
+	sys, ok := base.Module().(resolve.Resolver)
+	if !ok {
+		return nil, fmt.Errorf("unexpected module type: %T", base.Module())
+	}
+
+	cpuOpts := make([]metrics.OptionFunc, 0)
+	if config.UserPerformanceCounters {
+		cpuOpts = append(cpuOpts, metrics.WithWindowsPerformanceCounter())
+	}
+	cpu, err := metrics.New(sys, cpuOpts...)
+
+	if err != nil {
+		return nil, fmt.Errorf("error initializing system.cpu metricset: %w", err)
+	}
 	return &MetricSet{
 		BaseMetricSet: base,
 		opts:          opts,
-		cpu:           metrics.New(sys),
+		cpu:           cpu,
+		sys:           sys,
 	}, nil
 }
 
@@ -125,13 +140,11 @@ func (m *MetricSet) Diagnostics() []diagnostics.DiagnosticSetup {
 }
 
 func (m *MetricSet) fetchRawCPU() []byte {
-	sys := m.BaseMetricSet.Module().(resolve.Resolver)
-	return diagnostics.GetRawFileOrErrorString(sys, "/proc/stat")
+	return diagnostics.GetRawFileOrErrorString(m.sys, "/proc/stat")
 }
 
 func (m *MetricSet) fetchCPUInfo() []byte {
-	sys := m.BaseMetricSet.Module().(resolve.Resolver)
-	return diagnostics.GetRawFileOrErrorString(sys, "/proc/cpuinfo")
+	return diagnostics.GetRawFileOrErrorString(m.sys, "/proc/cpuinfo")
 }
 
 // copyFieldsOrDefault copies the field specified by key to the given map. It will

x-pack/metricbeat/metricbeat.reference.yml

@@ -142,6 +142,11 @@ metricbeat.modules:
   # Filter systemd services based on a name pattern
   #service.pattern_filter: ["ssh*", "nfs*"]
 
+  # This option enables the use of performance counters to collect data for cpu/core metricset.
+  # Only effective for Windows.
+  # You should use this option if running beats on machins with more than 64 cores.
+  #use_performance_counters: true
+
 #------------------------------- ActiveMQ Module -------------------------------
 - module: activemq
   metricsets: ['broker', 'queue', 'topic']