From 4928231bed654a6d14c3d27668e1e50e29280a38 Mon Sep 17 00:00:00 2001 From: Zelin Hao Date: Tue, 23 Apr 2024 13:45:11 -0700 Subject: [PATCH 01/21] Add release notes for 1.3.16 (#648) Signed-off-by: Zelin Hao --- ...pensearch-performance-analyzer.release-notes-1.3.16.0.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 release-notes/opensearch-performance-analyzer.release-notes-1.3.16.0.md diff --git a/release-notes/opensearch-performance-analyzer.release-notes-1.3.16.0.md b/release-notes/opensearch-performance-analyzer.release-notes-1.3.16.0.md new file mode 100644 index 00000000..ec9351eb --- /dev/null +++ b/release-notes/opensearch-performance-analyzer.release-notes-1.3.16.0.md @@ -0,0 +1,6 @@ +## Version 1.3.16.0 Release Notes + +Compatible with OpenSearch 1.3.16 + +### Maintenance +* Bump netty to match version from core's buildSrc/version.properties and bump grpc to 1.56.1 ([#546](https://github.com/opensearch-project/performance-analyzer-rca/pull/546)) From 71f8ee93d8020c58aa779a8d66fa7cb866f6f868 Mon Sep 17 00:00:00 2001 From: Varun Lodaya Date: Thu, 30 May 2024 01:23:34 +0530 Subject: [PATCH 02/21] Updating security reachout email (#655) Signed-off-by: varun-lodaya --- README.md | 2 +- SECURITY.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f377ea2a..430a0a3d 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ This project has adopted the [Amazon Open Source Code of Conduct](CODE_OF_CONDUC ## Security -If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public GitHub issue. +If you discover a potential security issue in this project we ask that you notify OpenSearch Security directly via email to security@opensearch.org. Please do **not** create a public GitHub issue. ## License diff --git a/SECURITY.md b/SECURITY.md index 0b85ca04..be4ac746 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,3 +1,3 @@ ## Reporting a Vulnerability -If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to aws-security@amazon.com. Please do **not** create a public GitHub issue. \ No newline at end of file +If you discover a potential security issue in this project we ask that you notify OpenSearch Security directly via email to security@opensearch.org. Please do **not** create a public GitHub issue. From 489f318585ab4c572d2e1c4bc143a4a99807858c Mon Sep 17 00:00:00 2001 From: Atharva Sharma <60044988+atharvasharma61@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:28:52 +0530 Subject: [PATCH 03/21] Fixed the bug in CacheConfigMetricsCollector (#657) * Fixed the bug in CacheConfigMetricsCollector Signed-off-by: Atharva Sharma * Fixes error logging Signed-off-by: Gagan Juneja --------- Signed-off-by: Atharva Sharma Signed-off-by: Gagan Juneja Co-authored-by: Gagan Juneja --- .../CacheConfigMetricsCollector.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/CacheConfigMetricsCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/CacheConfigMetricsCollector.java index e2828483..419a251e 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/collectors/CacheConfigMetricsCollector.java +++ b/src/main/java/org/opensearch/performanceanalyzer/collectors/CacheConfigMetricsCollector.java @@ -17,6 +17,8 @@ import java.security.AccessController; import java.security.PrivilegedAction; import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.common.cache.Cache; import org.opensearch.indices.IndicesService; import org.opensearch.performanceanalyzer.OpenSearchResources; @@ -43,6 +45,7 @@ public class CacheConfigMetricsCollector extends PerformanceAnalyzerMetricsColle implements MetricsProcessor { public static final int SAMPLING_TIME_INTERVAL = MetricsConfiguration.CONFIG_MAP.get(CacheConfigMetricsCollector.class).samplingInterval; + private static final Logger LOG = LogManager.getLogger(CacheConfigMetricsCollector.class); private static final int KEYS_PATH_LENGTH = 0; private StringBuilder value; @@ -104,10 +107,14 @@ public void collectMetrics(long startTime) { indicesService, "indicesRequestCache", true); + Object openSearchOnHeapCache = + FieldUtils.readField(reqCache, "cache", true); Cache requestCache = (Cache) FieldUtils.readField( - reqCache, "cache", true); + openSearchOnHeapCache, + "cache", + true); Long requestCacheMaxSize = (Long) FieldUtils.readField( @@ -118,10 +125,15 @@ public void collectMetrics(long startTime) { SHARD_REQUEST_CACHE.toString(), requestCacheMaxSize); } catch (Exception e) { - return new CacheMaxSizeStatus( - SHARD_REQUEST_CACHE.toString(), null); + LOG.error("Error while evaluating requestCacheMaxSize", e); + return null; } }); + + if (shardRequestCacheMaxSizeStatus == null) { + return; + } + value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor) .append(shardRequestCacheMaxSizeStatus.serialize()); saveMetricValues(value.toString(), startTime); From 17611035b0519b96de809da2ccb07ea9b4107ede Mon Sep 17 00:00:00 2001 From: Atharva Sharma <60044988+atharvasharma61@users.noreply.github.com> Date: Tue, 11 Jun 2024 18:18:02 +0530 Subject: [PATCH 04/21] changes to merge PA-RTF (#651) * PA RTF merging init Signed-off-by: Atharva Sharma * working model Signed-off-by: Atharva Sharma * working model tip Signed-off-by: Atharva Sharma * functional model init Signed-off-by: Atharva Sharma * Migrated HeapMetricsCollector Signed-off-by: Atharva Sharma * Added RTFThreadPoolMetricsCollector Signed-off-by: Atharva Sharma * migrated NodeStats and DiskMetricsCollector Signed-off-by: Atharva Sharma * Added gauge data model for Heap_Max metric Signed-off-by: Atharva Sharma * implemented TelemetryAwarePlugin Signed-off-by: Atharva Sharma * Framework changes for PA RTF merging Signed-off-by: Atharva Sharma * refactored Signed-off-by: Atharva Sharma * spotless applied Signed-off-by: Atharva Sharma * Addressed small comments Signed-off-by: Atharva Sharma * Added different flag for RCA collectors Signed-off-by: Atharva Sharma * Addressed more comments Signed-off-by: Atharva Sharma * Added RTF collectors in config map Signed-off-by: Atharva Sharma * Added UTs Signed-off-by: Atharva Sharma * Added further UTs Signed-off-by: Atharva Sharma * Added dynamic control support to all collectors Signed-off-by: Atharva Sharma * fixed UT Signed-off-by: Atharva Sharma * refactoring Signed-off-by: Atharva Sharma * Revert "refactoring" This reverts commit 25d66e8bb3554d9d7674a87201beee0b08a2a33d. Signed-off-by: Atharva Sharma * Revert "fixed UT" This reverts commit 369bd957f473c52ae39284f1d7257e254be8d036. Signed-off-by: Atharva Sharma * Revert "Added dynamic control support to all collectors" This reverts commit 447e15f15084ce911b8944446e54d8d36e0e2a6f. Signed-off-by: Atharva Sharma * Adding two new collector interfaces Signed-off-by: Atharva Sharma * simplified interfaces Signed-off-by: Atharva Sharma * Added units and javadocs Signed-off-by: Atharva Sharma * Changes metrics semantic conventions Signed-off-by: Atharva Sharma * refactored Signed-off-by: Atharva Sharma * fixed UT Signed-off-by: Atharva Sharma * Added stats metrics for rtf collectors Signed-off-by: Atharva Sharma * reverted test delete Signed-off-by: Atharva Sharma * Fixes javadoc compilation issue Signed-off-by: Gagan Juneja --------- Signed-off-by: Atharva Sharma Signed-off-by: Gagan Juneja Co-authored-by: Gagan Juneja --- .../OpenSearchResources.java | 11 + .../PerformanceAnalyzerPlugin.java | 86 ++-- .../collectors/ValueCalculator.java | 2 +- .../telemetry/RTFDisksCollector.java | 113 +++++ .../telemetry/RTFHeapMetricsCollector.java | 130 ++++++ ...RTFNodeStatsAllShardsMetricsCollector.java | 405 ++++++++++++++++++ .../RTFThreadPoolMetricsCollector.java | 228 ++++++++++ .../config/PerformanceAnalyzerController.java | 15 + .../PerformanceAnalyzerClusterSettings.java | 11 + ...manceAnalyzerCollectorsSettingHandler.java | 65 +++ ...erformanceAnalyzerClusterConfigAction.java | 21 +- .../performanceanalyzer/util/Utils.java | 8 + .../PerformanceAnalyzerPluginTests.java | 27 +- .../collectors/CollectorTestBase.java | 30 ++ .../telemetry/RTFDisksCollectorTests.java | 42 ++ .../RTFHeapMetricsCollectorTests.java | 68 +++ ...deStatsAllShardsMetricsCollectorTests.java | 138 ++++++ .../RTFThreadPoolMetricsCollectorTests.java | 91 ++++ ...AnalyzerCollectorsSettingHandlerTests.java | 41 ++ ...manceAnalyzerClusterConfigActionTests.java | 10 +- 20 files changed, 1512 insertions(+), 30 deletions(-) create mode 100644 src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFDisksCollector.java create mode 100644 src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFHeapMetricsCollector.java create mode 100644 src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFNodeStatsAllShardsMetricsCollector.java create mode 100644 src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFThreadPoolMetricsCollector.java create mode 100644 src/main/java/org/opensearch/performanceanalyzer/config/setting/handler/PerformanceAnalyzerCollectorsSettingHandler.java create mode 100644 src/test/java/org/opensearch/performanceanalyzer/collectors/CollectorTestBase.java create mode 100644 src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFDisksCollectorTests.java create mode 100644 src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFHeapMetricsCollectorTests.java create mode 100644 src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFNodeStatsAllShardsMetricsCollectorTests.java create mode 100644 src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFThreadPoolMetricsCollectorTests.java create mode 100644 src/test/java/org/opensearch/performanceanalyzer/config/setting/handler/PerformanceAnalyzerCollectorsSettingHandlerTests.java diff --git a/src/main/java/org/opensearch/performanceanalyzer/OpenSearchResources.java b/src/main/java/org/opensearch/performanceanalyzer/OpenSearchResources.java index c0540642..0ddab1a6 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/OpenSearchResources.java +++ b/src/main/java/org/opensearch/performanceanalyzer/OpenSearchResources.java @@ -11,6 +11,7 @@ import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.env.Environment; import org.opensearch.indices.IndicesService; +import org.opensearch.telemetry.metrics.MetricsRegistry; import org.opensearch.threadpool.ThreadPool; public final class OpenSearchResources { @@ -20,6 +21,7 @@ public final class OpenSearchResources { private CircuitBreakerService circuitBreakerService; private ClusterService clusterService; private IndicesService indicesService; + private MetricsRegistry metricsRegistry; private Settings settings; private Environment environment; private java.nio.file.Path configPath; @@ -32,6 +34,7 @@ private OpenSearchResources() { clusterService = null; settings = null; indicesService = null; + metricsRegistry = null; environment = null; configPath = null; pluginFileLocation = null; @@ -101,6 +104,14 @@ public void setIndicesService(IndicesService indicesService) { this.indicesService = indicesService; } + public MetricsRegistry getMetricsRegistry() { + return metricsRegistry; + } + + public void setMetricsRegistry(MetricsRegistry metricsRegistry) { + this.metricsRegistry = metricsRegistry; + } + public void setClient(final Client client) { this.client = client; } diff --git a/src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java b/src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java index bfbcbcc8..143d1f1a 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java +++ b/src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java @@ -55,6 +55,10 @@ import org.opensearch.performanceanalyzer.collectors.ShardIndexingPressureMetricsCollector; import org.opensearch.performanceanalyzer.collectors.ShardStateCollector; import org.opensearch.performanceanalyzer.collectors.ThreadPoolMetricsCollector; +import org.opensearch.performanceanalyzer.collectors.telemetry.RTFDisksCollector; +import org.opensearch.performanceanalyzer.collectors.telemetry.RTFHeapMetricsCollector; +import org.opensearch.performanceanalyzer.collectors.telemetry.RTFNodeStatsAllShardsMetricsCollector; +import org.opensearch.performanceanalyzer.collectors.telemetry.RTFThreadPoolMetricsCollector; import org.opensearch.performanceanalyzer.commons.OSMetricsGeneratorFactory; import org.opensearch.performanceanalyzer.commons.collectors.DisksCollector; import org.opensearch.performanceanalyzer.commons.collectors.GCInfoCollector; @@ -74,6 +78,7 @@ import org.opensearch.performanceanalyzer.config.setting.handler.ConfigOverridesClusterSettingHandler; import org.opensearch.performanceanalyzer.config.setting.handler.NodeStatsSettingHandler; import org.opensearch.performanceanalyzer.config.setting.handler.PerformanceAnalyzerClusterSettingHandler; +import org.opensearch.performanceanalyzer.config.setting.handler.PerformanceAnalyzerCollectorsSettingHandler; import org.opensearch.performanceanalyzer.http_action.config.PerformanceAnalyzerClusterConfigAction; import org.opensearch.performanceanalyzer.http_action.config.PerformanceAnalyzerConfigAction; import org.opensearch.performanceanalyzer.http_action.config.PerformanceAnalyzerOverridesClusterConfigAction; @@ -88,9 +93,11 @@ import org.opensearch.plugins.NetworkPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.SearchPlugin; +import org.opensearch.plugins.TelemetryAwarePlugin; import org.opensearch.repositories.RepositoriesService; import org.opensearch.rest.RestController; import org.opensearch.script.ScriptService; +import org.opensearch.telemetry.metrics.MetricsRegistry; import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transport; @@ -98,7 +105,7 @@ import org.opensearch.watcher.ResourceWatcherService; public final class PerformanceAnalyzerPlugin extends Plugin - implements ActionPlugin, NetworkPlugin, SearchPlugin { + implements ActionPlugin, NetworkPlugin, SearchPlugin, TelemetryAwarePlugin { private static final Logger LOG = LogManager.getLogger(PerformanceAnalyzerPlugin.class); public static final String PLUGIN_NAME = "opensearch-performance-analyzer"; private static final String ADD_FAULT_DETECTION_METHOD = "addFaultDetectionListener"; @@ -108,6 +115,8 @@ public final class PerformanceAnalyzerPlugin extends Plugin private static SecurityManager sm = null; private final PerformanceAnalyzerClusterSettingHandler perfAnalyzerClusterSettingHandler; private final NodeStatsSettingHandler nodeStatsSettingHandler; + private final PerformanceAnalyzerCollectorsSettingHandler + performanceAnalyzerCollectorsSettingHandler; private final ConfigOverridesClusterSettingHandler configOverridesClusterSettingHandler; private final ConfigOverridesWrapper configOverridesWrapper; private final PerformanceAnalyzerController performanceAnalyzerController; @@ -166,7 +175,8 @@ public PerformanceAnalyzerPlugin(final Settings settings, final java.nio.file.Pa new ClusterSettingsManager( Arrays.asList( PerformanceAnalyzerClusterSettings.COMPOSITE_PA_SETTING, - PerformanceAnalyzerClusterSettings.PA_NODE_STATS_SETTING), + PerformanceAnalyzerClusterSettings.PA_NODE_STATS_SETTING, + PerformanceAnalyzerClusterSettings.PA_COLLECTORS_SETTING), Collections.singletonList( PerformanceAnalyzerClusterSettings.CONFIG_OVERRIDES_SETTING)); configOverridesClusterSettingHandler = @@ -189,27 +199,62 @@ public PerformanceAnalyzerPlugin(final Settings settings, final java.nio.file.Pa clusterSettingsManager.addSubscriberForIntSetting( PerformanceAnalyzerClusterSettings.PA_NODE_STATS_SETTING, nodeStatsSettingHandler); + performanceAnalyzerCollectorsSettingHandler = + new PerformanceAnalyzerCollectorsSettingHandler( + performanceAnalyzerController, clusterSettingsManager); + clusterSettingsManager.addSubscriberForIntSetting( + PerformanceAnalyzerClusterSettings.PA_COLLECTORS_SETTING, + performanceAnalyzerCollectorsSettingHandler); + + scheduleTelemetryCollectors(); + scheduleRcaCollectors(); + + scheduledMetricCollectorsExecutor.start(); + + EventLog eventLog = new EventLog(); + EventLogFileHandler eventLogFileHandler = + new EventLogFileHandler(eventLog, PluginSettings.instance().getMetricsLocation()); + new EventLogQueueProcessor( + eventLogFileHandler, + MetricsConfiguration.SAMPLING_INTERVAL, + QUEUE_PURGE_INTERVAL_MS, + performanceAnalyzerController) + .scheduleExecutor(); + } + + private void scheduleTelemetryCollectors() { + scheduledMetricCollectorsExecutor.addScheduledMetricCollector( + new RTFDisksCollector(performanceAnalyzerController, configOverridesWrapper)); + scheduledMetricCollectorsExecutor.addScheduledMetricCollector( + new RTFHeapMetricsCollector(performanceAnalyzerController, configOverridesWrapper)); + scheduledMetricCollectorsExecutor.addScheduledMetricCollector( + new RTFThreadPoolMetricsCollector( + performanceAnalyzerController, configOverridesWrapper)); + scheduledMetricCollectorsExecutor.addScheduledMetricCollector( + new RTFNodeStatsAllShardsMetricsCollector( + performanceAnalyzerController, configOverridesWrapper)); + } + + private void scheduleRcaCollectors() { scheduledMetricCollectorsExecutor.addScheduledMetricCollector( new ThreadPoolMetricsCollector()); + scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new HeapMetricsCollector()); + scheduledMetricCollectorsExecutor.addScheduledMetricCollector( + new NodeStatsAllShardsMetricsCollector(performanceAnalyzerController)); + scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new DisksCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector( new CacheConfigMetricsCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector( new CircuitBreakerCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new OSMetricsCollector()); - scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new HeapMetricsCollector()); - scheduledMetricCollectorsExecutor.addScheduledMetricCollector( new NodeDetailsCollector(configOverridesWrapper)); - scheduledMetricCollectorsExecutor.addScheduledMetricCollector( - new NodeStatsAllShardsMetricsCollector(performanceAnalyzerController)); scheduledMetricCollectorsExecutor.addScheduledMetricCollector( new ClusterManagerServiceMetrics()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector( new ClusterManagerServiceEventMetrics()); - scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new DisksCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector( new NetworkInterfaceCollector()); - scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new GCInfoCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector(StatsCollector.instance()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector( new FaultDetectionMetricsCollector( @@ -222,11 +267,11 @@ public PerformanceAnalyzerPlugin(final Settings settings, final java.nio.file.Pa scheduledMetricCollectorsExecutor.addScheduledMetricCollector( new SearchBackPressureStatsCollector( performanceAnalyzerController, configOverridesWrapper)); - scheduledMetricCollectorsExecutor.addScheduledMetricCollector( new AdmissionControlMetricsCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector( new ElectionTermCollector(performanceAnalyzerController, configOverridesWrapper)); + scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new GCInfoCollector()); try { Class.forName(ShardIndexingPressureMetricsCollector.SHARD_INDEXING_PRESSURE_CLASS_NAME); scheduledMetricCollectorsExecutor.addScheduledMetricCollector( @@ -236,17 +281,6 @@ public PerformanceAnalyzerPlugin(final Settings settings, final java.nio.file.Pa LOG.info( "Shard IndexingPressure not present in this OpenSearch version. Skipping ShardIndexingPressureMetricsCollector"); } - scheduledMetricCollectorsExecutor.start(); - - EventLog eventLog = new EventLog(); - EventLogFileHandler eventLogFileHandler = - new EventLogFileHandler(eventLog, PluginSettings.instance().getMetricsLocation()); - new EventLogQueueProcessor( - eventLogFileHandler, - MetricsConfiguration.SAMPLING_INTERVAL, - QUEUE_PURGE_INTERVAL_MS, - performanceAnalyzerController) - .scheduleExecutor(); } // - http level: bulk, search @@ -319,7 +353,8 @@ public List getRestHandlers( settings, restController, perfAnalyzerClusterSettingHandler, - nodeStatsSettingHandler); + nodeStatsSettingHandler, + performanceAnalyzerCollectorsSettingHandler); PerformanceAnalyzerOverridesClusterConfigAction paOverridesConfigClusterAction = new PerformanceAnalyzerOverridesClusterConfigAction( settings, @@ -346,12 +381,14 @@ public Collection createComponents( NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier) { + Supplier repositoriesServiceSupplier, + Tracer tracer, + MetricsRegistry metricsRegistry) { OpenSearchResources.INSTANCE.setClusterService(clusterService); OpenSearchResources.INSTANCE.setThreadPool(threadPool); OpenSearchResources.INSTANCE.setEnvironment(environment); OpenSearchResources.INSTANCE.setClient(client); - + OpenSearchResources.INSTANCE.setMetricsRegistry(metricsRegistry); // ClusterSettingsManager needs ClusterService to have been created before we can // initialize it. This is the earliest point at which we know ClusterService is created. // So, call the initialize method here. @@ -379,6 +416,7 @@ public List> getSettings() { return Arrays.asList( PerformanceAnalyzerClusterSettings.COMPOSITE_PA_SETTING, PerformanceAnalyzerClusterSettings.PA_NODE_STATS_SETTING, - PerformanceAnalyzerClusterSettings.CONFIG_OVERRIDES_SETTING); + PerformanceAnalyzerClusterSettings.CONFIG_OVERRIDES_SETTING, + PerformanceAnalyzerClusterSettings.PA_COLLECTORS_SETTING); } } diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/ValueCalculator.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/ValueCalculator.java index 83c0e59d..2242607d 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/collectors/ValueCalculator.java +++ b/src/main/java/org/opensearch/performanceanalyzer/collectors/ValueCalculator.java @@ -8,6 +8,6 @@ import org.opensearch.action.admin.indices.stats.ShardStats; @FunctionalInterface -interface ValueCalculator { +public interface ValueCalculator { long calculateValue(ShardStats shardStats); } diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFDisksCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFDisksCollector.java new file mode 100644 index 00000000..14fb730d --- /dev/null +++ b/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFDisksCollector.java @@ -0,0 +1,113 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.collectors.telemetry; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.performanceanalyzer.OpenSearchResources; +import org.opensearch.performanceanalyzer.commons.OSMetricsGeneratorFactory; +import org.opensearch.performanceanalyzer.commons.collectors.PerformanceAnalyzerMetricsCollector; +import org.opensearch.performanceanalyzer.commons.collectors.TelemetryCollector; +import org.opensearch.performanceanalyzer.commons.config.overrides.ConfigOverridesWrapper; +import org.opensearch.performanceanalyzer.commons.metrics.MetricsConfiguration; +import org.opensearch.performanceanalyzer.commons.metrics.RTFMetrics; +import org.opensearch.performanceanalyzer.commons.metrics_generator.DiskMetricsGenerator; +import org.opensearch.performanceanalyzer.commons.metrics_generator.OSMetricsGenerator; +import org.opensearch.performanceanalyzer.commons.stats.metrics.StatExceptionCode; +import org.opensearch.performanceanalyzer.commons.stats.metrics.StatMetrics; +import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; +import org.opensearch.telemetry.metrics.Histogram; +import org.opensearch.telemetry.metrics.MetricsRegistry; +import org.opensearch.telemetry.metrics.tags.Tags; + +public class RTFDisksCollector extends PerformanceAnalyzerMetricsCollector + implements TelemetryCollector { + + private Histogram diskWaitTimeMetrics; + private Histogram diskServiceRateMetrics; + private Histogram diskUtilizationMetrics; + private MetricsRegistry metricsRegistry; + private boolean metricsInitialised; + private static final Logger LOG = LogManager.getLogger(RTFDisksCollector.class); + private PerformanceAnalyzerController performanceAnalyzerController; + private ConfigOverridesWrapper configOverridesWrapper; + + public RTFDisksCollector( + PerformanceAnalyzerController performanceAnalyzerController, + ConfigOverridesWrapper configOverridesWrapper) { + super( + MetricsConfiguration.CONFIG_MAP.get(RTFDisksCollector.class).samplingInterval, + "RTFDisksCollector", + StatMetrics.RTF_DISKS_COLLECTOR_EXECUTION_TIME, + StatExceptionCode.RTF_DISK_METRICS_COLLECTOR_ERROR); + this.metricsInitialised = false; + this.performanceAnalyzerController = performanceAnalyzerController; + this.configOverridesWrapper = configOverridesWrapper; + } + + @Override + public void collectMetrics(long startTime) { + if (performanceAnalyzerController.isCollectorDisabled( + configOverridesWrapper, getCollectorName())) { + LOG.info("RTFDisksCollector is disabled. Skipping collection."); + return; + } + + OSMetricsGenerator generator = OSMetricsGeneratorFactory.getInstance(); + if (generator == null) { + LOG.error("could not get the instance of OSMetricsGeneratorFactory class"); + return; + } + + metricsRegistry = OpenSearchResources.INSTANCE.getMetricsRegistry(); + if (metricsRegistry == null) { + LOG.error("could not get the instance of MetricsRegistry class"); + return; + } + + LOG.debug("Executing collect metrics for RTFDisksCollector"); + + initialiseMetricsIfNeeded(); + DiskMetricsGenerator diskMetricsGenerator = generator.getDiskMetricsGenerator(); + diskMetricsGenerator.addSample(); + + recordMetrics(diskMetricsGenerator); + } + + public void recordMetrics(DiskMetricsGenerator diskMetricsGenerator) { + for (String disk : diskMetricsGenerator.getAllDisks()) { + Tags diskNameTag = + Tags.create().addTag(RTFMetrics.DiskDimension.DISK_NAME.toString(), disk); + double diskWaitTime = diskMetricsGenerator.getAwait(disk); + double diskServiceRate = diskMetricsGenerator.getServiceRate(disk); + double diskUtilization = diskMetricsGenerator.getDiskUtilization(disk); + diskWaitTimeMetrics.record(diskWaitTime, diskNameTag); + diskUtilizationMetrics.record(diskUtilization, diskNameTag); + diskServiceRateMetrics.record(diskServiceRate, diskNameTag); + } + } + + private void initialiseMetricsIfNeeded() { + if (metricsInitialised == false) { + diskWaitTimeMetrics = + metricsRegistry.createHistogram( + RTFMetrics.DiskValue.Constants.WAIT_VALUE, + "DiskWaitTimeMetrics", + RTFMetrics.MetricUnits.MILLISECOND.toString()); + diskServiceRateMetrics = + metricsRegistry.createHistogram( + RTFMetrics.DiskValue.Constants.SRATE_VALUE, + "DiskServiceRateMetrics", + RTFMetrics.MetricUnits.MEGABYTE_PER_SEC.toString()); + diskUtilizationMetrics = + metricsRegistry.createHistogram( + RTFMetrics.DiskValue.Constants.UTIL_VALUE, + "DiskUtilizationMetrics", + RTFMetrics.MetricUnits.PERCENT.toString()); + metricsInitialised = true; + } + } +} diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFHeapMetricsCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFHeapMetricsCollector.java new file mode 100644 index 00000000..6872089d --- /dev/null +++ b/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFHeapMetricsCollector.java @@ -0,0 +1,130 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.collectors.telemetry; + +import java.lang.management.MemoryUsage; +import java.util.Map; +import java.util.function.Supplier; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.performanceanalyzer.OpenSearchResources; +import org.opensearch.performanceanalyzer.commons.collectors.PerformanceAnalyzerMetricsCollector; +import org.opensearch.performanceanalyzer.commons.collectors.TelemetryCollector; +import org.opensearch.performanceanalyzer.commons.config.overrides.ConfigOverridesWrapper; +import org.opensearch.performanceanalyzer.commons.jvm.GCMetrics; +import org.opensearch.performanceanalyzer.commons.jvm.HeapMetrics; +import org.opensearch.performanceanalyzer.commons.metrics.MetricsConfiguration; +import org.opensearch.performanceanalyzer.commons.metrics.RTFMetrics; +import org.opensearch.performanceanalyzer.commons.stats.metrics.StatExceptionCode; +import org.opensearch.performanceanalyzer.commons.stats.metrics.StatMetrics; +import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; +import org.opensearch.telemetry.metrics.Histogram; +import org.opensearch.telemetry.metrics.MetricsRegistry; +import org.opensearch.telemetry.metrics.tags.Tags; + +public class RTFHeapMetricsCollector extends PerformanceAnalyzerMetricsCollector + implements TelemetryCollector { + private static final Logger LOG = LogManager.getLogger(RTFHeapMetricsCollector.class); + public static final int SAMPLING_TIME_INTERVAL = + MetricsConfiguration.CONFIG_MAP.get(RTFHeapMetricsCollector.class).samplingInterval; + private Histogram gcCollectionEventMetrics; + private Histogram gcCollectionTimeMetrics; + private Histogram heapUsedMetrics; + private MetricsRegistry metricsRegistry; + private final String memTypeAttributeKey = "mem_type"; + private boolean metricsInitialised; + private PerformanceAnalyzerController performanceAnalyzerController; + private ConfigOverridesWrapper configOverridesWrapper; + + public RTFHeapMetricsCollector( + PerformanceAnalyzerController performanceAnalyzerController, + ConfigOverridesWrapper configOverridesWrapper) { + super( + SAMPLING_TIME_INTERVAL, + "RTFHeapMetricsCollector", + StatMetrics.RTF_HEAP_METRICS_COLLECTOR_EXECUTION_TIME, + StatExceptionCode.RTF_HEAP_METRICS_COLLECTOR_ERROR); + this.metricsInitialised = false; + this.performanceAnalyzerController = performanceAnalyzerController; + this.configOverridesWrapper = configOverridesWrapper; + } + + @Override + public void collectMetrics(long startTime) { + if (performanceAnalyzerController.isCollectorDisabled( + configOverridesWrapper, getCollectorName())) { + LOG.info("RTFDisksCollector is disabled. Skipping collection."); + return; + } + + metricsRegistry = OpenSearchResources.INSTANCE.getMetricsRegistry(); + if (metricsRegistry == null) { + LOG.error("could not get the instance of MetricsRegistry class"); + return; + } + + initialiseMetricsIfNeeded(); + GCMetrics.runGCMetrics(); + LOG.debug("Executing collect metrics for RTFHeapMetricsCollector"); + recordMetrics(); + } + + private void initialiseMetricsIfNeeded() { + if (metricsInitialised == false) { + gcCollectionEventMetrics = + metricsRegistry.createHistogram( + RTFMetrics.HeapValue.Constants.COLLECTION_COUNT_VALUE, + "GC Collection Event PA Metrics", + RTFMetrics.MetricUnits.COUNT.toString()); + + gcCollectionTimeMetrics = + metricsRegistry.createHistogram( + RTFMetrics.HeapValue.Constants.COLLECTION_TIME_VALUE, + "GC Collection Time PA Metrics", + RTFMetrics.MetricUnits.MILLISECOND.toString()); + + heapUsedMetrics = + metricsRegistry.createHistogram( + RTFMetrics.HeapValue.Constants.USED_VALUE, + "GC Heap Used PA Metrics", + RTFMetrics.MetricUnits.BYTE.toString()); + metricsInitialised = true; + } + } + + private void recordMetrics() { + Tags totYoungGCTag = + Tags.create() + .addTag( + RTFMetrics.HeapDimension.MEM_TYPE.getName(), + RTFMetrics.GCType.TOT_YOUNG_GC.toString()); + + Tags totFullGCTag = + Tags.create().addTag(memTypeAttributeKey, RTFMetrics.GCType.TOT_FULL_GC.toString()); + + gcCollectionEventMetrics.record(GCMetrics.getTotYoungGCCollectionCount(), totYoungGCTag); + + gcCollectionEventMetrics.record(GCMetrics.getTotFullGCCollectionCount(), totFullGCTag); + + gcCollectionTimeMetrics.record(GCMetrics.getTotYoungGCCollectionTime(), totYoungGCTag); + + gcCollectionTimeMetrics.record(GCMetrics.getTotFullGCCollectionTime(), totFullGCTag); + + for (Map.Entry> entry : + HeapMetrics.getMemoryUsageSuppliers().entrySet()) { + MemoryUsage memoryUsage = entry.getValue().get(); + heapUsedMetrics.record( + memoryUsage.getUsed(), + Tags.create().addTag(memTypeAttributeKey, entry.getKey())); + metricsRegistry.createGauge( + RTFMetrics.HeapValue.Constants.MAX_VALUE, + "Heap Max PA metrics", + "", + () -> (double) memoryUsage.getMax(), + Tags.create().addTag(memTypeAttributeKey, entry.getKey())); + } + } +} diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFNodeStatsAllShardsMetricsCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFNodeStatsAllShardsMetricsCollector.java new file mode 100644 index 00000000..63033dba --- /dev/null +++ b/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFNodeStatsAllShardsMetricsCollector.java @@ -0,0 +1,405 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.collectors.telemetry; + +import static org.opensearch.performanceanalyzer.commons.stats.metrics.StatExceptionCode.RTF_NODESTATS_COLLECTION_ERROR; +import static org.opensearch.performanceanalyzer.commons.stats.metrics.StatMetrics.RTF_NODE_STATS_ALL_SHARDS_METRICS_COLLECTOR_EXECUTION_TIME; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableMap; +import java.util.HashMap; +import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.admin.indices.stats.CommonStatsFlags; +import org.opensearch.action.admin.indices.stats.IndexShardStats; +import org.opensearch.action.admin.indices.stats.ShardStats; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.indices.IndicesService; +import org.opensearch.performanceanalyzer.OpenSearchResources; +import org.opensearch.performanceanalyzer.collectors.ValueCalculator; +import org.opensearch.performanceanalyzer.commons.collectors.MetricStatus; +import org.opensearch.performanceanalyzer.commons.collectors.PerformanceAnalyzerMetricsCollector; +import org.opensearch.performanceanalyzer.commons.collectors.TelemetryCollector; +import org.opensearch.performanceanalyzer.commons.config.overrides.ConfigOverridesWrapper; +import org.opensearch.performanceanalyzer.commons.metrics.MetricsConfiguration; +import org.opensearch.performanceanalyzer.commons.metrics.RTFMetrics; +import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; +import org.opensearch.performanceanalyzer.util.Utils; +import org.opensearch.telemetry.metrics.Counter; +import org.opensearch.telemetry.metrics.MetricsRegistry; +import org.opensearch.telemetry.metrics.tags.Tags; + +public class RTFNodeStatsAllShardsMetricsCollector extends PerformanceAnalyzerMetricsCollector + implements TelemetryCollector { + public static final int SAMPLING_TIME_INTERVAL = + MetricsConfiguration.CONFIG_MAP.get(RTFNodeStatsAllShardsMetricsCollector.class) + .samplingInterval; + private static final Logger LOG = + LogManager.getLogger(RTFNodeStatsAllShardsMetricsCollector.class); + private Map currentShards; + private Map currentPerShardStats; + private Map prevPerShardStats; + private MetricsRegistry metricsRegistry; + private Counter cacheQueryHitMetrics; + private Counter cacheQueryMissMetrics; + private Counter cacheQuerySizeMetrics; + private Counter cacheFieldDataEvictionMetrics; + private Counter cacheFieldDataSizeMetrics; + private Counter cacheRequestHitMetrics; + private Counter cacheRequestMissMetrics; + private Counter cacheRequestEvictionMetrics; + private Counter cacheRequestSizeMetrics; + private boolean metricsInitialised; + private PerformanceAnalyzerController performanceAnalyzerController; + private ConfigOverridesWrapper configOverridesWrapper; + + public RTFNodeStatsAllShardsMetricsCollector( + PerformanceAnalyzerController performanceAnalyzerController, + ConfigOverridesWrapper configOverridesWrapper) { + super( + SAMPLING_TIME_INTERVAL, + "RTFNodeStatsMetricsCollector", + RTF_NODE_STATS_ALL_SHARDS_METRICS_COLLECTOR_EXECUTION_TIME, + RTF_NODESTATS_COLLECTION_ERROR); + currentShards = new HashMap<>(); + prevPerShardStats = new HashMap<>(); + currentPerShardStats = new HashMap<>(); + this.metricsInitialised = false; + this.performanceAnalyzerController = performanceAnalyzerController; + this.configOverridesWrapper = configOverridesWrapper; + } + + private void populateCurrentShards() { + if (!currentShards.isEmpty()) { + prevPerShardStats.putAll(currentPerShardStats); + currentPerShardStats.clear(); + } + currentShards.clear(); + currentShards = Utils.getShards(); + } + + private static final ImmutableMap valueCalculators = + ImmutableMap.of( + RTFMetrics.ShardStatsValue.INDEXING_THROTTLE_TIME.toString(), + (shardStats) -> + shardStats + .getStats() + .getIndexing() + .getTotal() + .getThrottleTime() + .millis(), + RTFMetrics.ShardStatsValue.CACHE_QUERY_HIT.toString(), + (shardStats) -> shardStats.getStats().getQueryCache().getHitCount(), + RTFMetrics.ShardStatsValue.CACHE_QUERY_MISS.toString(), + (shardStats) -> shardStats.getStats().getQueryCache().getMissCount(), + RTFMetrics.ShardStatsValue.CACHE_QUERY_SIZE.toString(), + (shardStats) -> shardStats.getStats().getQueryCache().getMemorySizeInBytes(), + RTFMetrics.ShardStatsValue.CACHE_FIELDDATA_EVICTION.toString(), + (shardStats) -> shardStats.getStats().getFieldData().getEvictions(), + RTFMetrics.ShardStatsValue.CACHE_FIELDDATA_SIZE.toString(), + (shardStats) -> shardStats.getStats().getFieldData().getMemorySizeInBytes(), + RTFMetrics.ShardStatsValue.CACHE_REQUEST_HIT.toString(), + (shardStats) -> shardStats.getStats().getRequestCache().getHitCount(), + RTFMetrics.ShardStatsValue.CACHE_REQUEST_MISS.toString(), + (shardStats) -> shardStats.getStats().getRequestCache().getMissCount(), + RTFMetrics.ShardStatsValue.CACHE_REQUEST_EVICTION.toString(), + (shardStats) -> shardStats.getStats().getRequestCache().getEvictions(), + RTFMetrics.ShardStatsValue.CACHE_REQUEST_SIZE.toString(), + (shardStats) -> shardStats.getStats().getRequestCache().getMemorySizeInBytes()); + + @Override + public void collectMetrics(long startTime) { + if (performanceAnalyzerController.isCollectorDisabled( + configOverridesWrapper, getCollectorName())) { + LOG.info("RTFDisksCollector is disabled. Skipping collection."); + return; + } + IndicesService indicesService = OpenSearchResources.INSTANCE.getIndicesService(); + if (indicesService == null) { + return; + } + + metricsRegistry = OpenSearchResources.INSTANCE.getMetricsRegistry(); + if (metricsRegistry == null) { + LOG.error("could not get the instance of MetricsRegistry class"); + return; + } + + LOG.debug("Executing collect metrics for RTFNodeStatsAllShardsMetricsCollector"); + initialiseMetricsIfNeeded(); + populateCurrentShards(); + populatePerShardStats(indicesService); + + for (Map.Entry currentShard : currentPerShardStats.entrySet()) { + ShardId shardId = (ShardId) currentShard.getKey(); + ShardStats currentShardStats = (ShardStats) currentShard.getValue(); + if (prevPerShardStats.size() == 0) { + // Populating value for the first run. + recordMetrics( + new NodeStatsMetricsAllShardsPerCollectionStatus(currentShardStats), + shardId.getIndexName(), + String.valueOf(shardId.id())); + continue; + } + ShardStats prevShardStats = prevPerShardStats.get(shardId); + if (prevShardStats == null) { + // Populate value for shards which are new and were not present in the previous + // run. + recordMetrics( + new NodeStatsMetricsAllShardsPerCollectionStatus(currentShardStats), + shardId.getIndexName(), + String.valueOf(shardId.id())); + continue; + } + NodeStatsMetricsAllShardsPerCollectionStatus prevValue = + new NodeStatsMetricsAllShardsPerCollectionStatus(prevShardStats); + NodeStatsMetricsAllShardsPerCollectionStatus currValue = + new NodeStatsMetricsAllShardsPerCollectionStatus(currentShardStats); + populateDiffMetricValue(prevValue, currValue, shardId.getIndexName(), shardId.id()); + } + } + + private void initialiseMetricsIfNeeded() { + if (metricsInitialised == false) { + cacheQueryHitMetrics = + metricsRegistry.createCounter( + RTFMetrics.ShardStatsValue.Constants.QUEY_CACHE_HIT_COUNT_VALUE, + "CacheQueryHit Metrics", + RTFMetrics.MetricUnits.COUNT.toString()); + + cacheQueryMissMetrics = + metricsRegistry.createCounter( + RTFMetrics.ShardStatsValue.Constants.QUERY_CACHE_MISS_COUNT_VALUE, + "CacheQueryMiss Metrics", + RTFMetrics.MetricUnits.COUNT.toString()); + + cacheQuerySizeMetrics = + metricsRegistry.createCounter( + RTFMetrics.ShardStatsValue.Constants.QUERY_CACHE_IN_BYTES_VALUE, + "CacheQuerySize Metrics", + RTFMetrics.MetricUnits.BYTE.toString()); + + cacheFieldDataEvictionMetrics = + metricsRegistry.createCounter( + RTFMetrics.ShardStatsValue.Constants.FIELDDATA_EVICTION_VALUE, + "CacheFieldDataEviction Metrics", + RTFMetrics.MetricUnits.COUNT.toString()); + + cacheFieldDataSizeMetrics = + metricsRegistry.createCounter( + RTFMetrics.ShardStatsValue.Constants.FIELD_DATA_IN_BYTES_VALUE, + "CacheFieldDataSize Metrics", + RTFMetrics.MetricUnits.BYTE.toString()); + + cacheRequestHitMetrics = + metricsRegistry.createCounter( + RTFMetrics.ShardStatsValue.Constants.REQUEST_CACHE_HIT_COUNT_VALUE, + "CacheRequestHit Metrics", + RTFMetrics.MetricUnits.COUNT.toString()); + + cacheRequestMissMetrics = + metricsRegistry.createCounter( + RTFMetrics.ShardStatsValue.Constants.REQUEST_CACHE_MISS_COUNT_VALUE, + "CacheRequestMiss Metrics", + RTFMetrics.MetricUnits.COUNT.toString()); + + cacheRequestEvictionMetrics = + metricsRegistry.createCounter( + RTFMetrics.ShardStatsValue.Constants.REQUEST_CACHE_EVICTION_VALUE, + "CacheRequestEviction Metrics", + RTFMetrics.MetricUnits.COUNT.toString()); + + cacheRequestSizeMetrics = + metricsRegistry.createCounter( + RTFMetrics.ShardStatsValue.Constants.REQUEST_CACHE_IN_BYTES_VALUE, + "CacheRequestSize Metrics", + RTFMetrics.MetricUnits.BYTE.toString()); + metricsInitialised = true; + } + } + + public void populatePerShardStats(IndicesService indicesService) { + // Populate the shard stats per shard. + for (Map.Entry currentShard : currentShards.entrySet()) { + IndexShard currentIndexShard = (IndexShard) currentShard.getValue(); + IndexShardStats currentIndexShardStats = + Utils.indexShardStats( + indicesService, + currentIndexShard, + new CommonStatsFlags( + CommonStatsFlags.Flag.QueryCache, + CommonStatsFlags.Flag.FieldData, + CommonStatsFlags.Flag.RequestCache)); + for (ShardStats shardStats : currentIndexShardStats.getShards()) { + currentPerShardStats.put(currentIndexShardStats.getShardId(), shardStats); + } + } + } + + private void recordMetrics( + NodeStatsMetricsAllShardsPerCollectionStatus metrics, + String indexName, + String shardId) { + Tags nodeStatsMetricsTag = + Tags.create() + .addTag(RTFMetrics.CommonDimension.INDEX_NAME.toString(), indexName) + .addTag(RTFMetrics.CommonDimension.SHARD_ID.toString(), shardId); + + cacheQueryMissMetrics.add(metrics.getQueryCacheMissCount(), nodeStatsMetricsTag); + cacheQuerySizeMetrics.add(metrics.getQueryCacheInBytes(), nodeStatsMetricsTag); + cacheQueryHitMetrics.add(metrics.getQueryCacheHitCount(), nodeStatsMetricsTag); + + cacheFieldDataEvictionMetrics.add(metrics.getFieldDataEvictions(), nodeStatsMetricsTag); + cacheFieldDataSizeMetrics.add(metrics.getFieldDataInBytes(), nodeStatsMetricsTag); + + cacheRequestEvictionMetrics.add(metrics.getRequestCacheEvictions(), nodeStatsMetricsTag); + cacheRequestHitMetrics.add(metrics.getRequestCacheHitCount(), nodeStatsMetricsTag); + cacheRequestMissMetrics.add(metrics.getRequestCacheMissCount(), nodeStatsMetricsTag); + cacheRequestSizeMetrics.add(metrics.getRequestCacheInBytes(), nodeStatsMetricsTag); + } + + public void populateDiffMetricValue( + NodeStatsMetricsAllShardsPerCollectionStatus prevValue, + NodeStatsMetricsAllShardsPerCollectionStatus currValue, + String indexName, + int shardId) { + + NodeStatsMetricsAllShardsPerCollectionStatus metrics = + new NodeStatsMetricsAllShardsPerCollectionStatus( + Math.max((currValue.queryCacheHitCount - prevValue.queryCacheHitCount), 0), + Math.max( + (currValue.queryCacheMissCount - prevValue.queryCacheMissCount), 0), + currValue.queryCacheInBytes, + Math.max((currValue.fieldDataEvictions - prevValue.fieldDataEvictions), 0), + currValue.fieldDataInBytes, + Math.max( + (currValue.requestCacheHitCount - prevValue.requestCacheHitCount), + 0), + Math.max( + (currValue.requestCacheMissCount - prevValue.requestCacheMissCount), + 0), + Math.max( + (currValue.requestCacheEvictions - prevValue.requestCacheEvictions), + 0), + currValue.requestCacheInBytes); + + recordMetrics(metrics, indexName, String.valueOf(shardId)); + } + + public static class NodeStatsMetricsAllShardsPerCollectionStatus extends MetricStatus { + + @JsonIgnore private ShardStats shardStats; + + private final long queryCacheHitCount; + private final long queryCacheMissCount; + private final long queryCacheInBytes; + private final long fieldDataEvictions; + private final long fieldDataInBytes; + private final long requestCacheHitCount; + private final long requestCacheMissCount; + private final long requestCacheEvictions; + private final long requestCacheInBytes; + + public NodeStatsMetricsAllShardsPerCollectionStatus(ShardStats shardStats) { + super(); + this.shardStats = shardStats; + + this.queryCacheHitCount = calculate(RTFMetrics.ShardStatsValue.CACHE_QUERY_HIT); + this.queryCacheMissCount = calculate(RTFMetrics.ShardStatsValue.CACHE_QUERY_MISS); + this.queryCacheInBytes = calculate(RTFMetrics.ShardStatsValue.CACHE_QUERY_SIZE); + this.fieldDataEvictions = + calculate(RTFMetrics.ShardStatsValue.CACHE_FIELDDATA_EVICTION); + this.fieldDataInBytes = calculate(RTFMetrics.ShardStatsValue.CACHE_FIELDDATA_SIZE); + this.requestCacheHitCount = calculate(RTFMetrics.ShardStatsValue.CACHE_REQUEST_HIT); + this.requestCacheMissCount = calculate(RTFMetrics.ShardStatsValue.CACHE_REQUEST_MISS); + this.requestCacheEvictions = + calculate(RTFMetrics.ShardStatsValue.CACHE_REQUEST_EVICTION); + this.requestCacheInBytes = calculate(RTFMetrics.ShardStatsValue.CACHE_REQUEST_SIZE); + } + + @SuppressWarnings("checkstyle:parameternumber") + public NodeStatsMetricsAllShardsPerCollectionStatus( + long queryCacheHitCount, + long queryCacheMissCount, + long queryCacheInBytes, + long fieldDataEvictions, + long fieldDataInBytes, + long requestCacheHitCount, + long requestCacheMissCount, + long requestCacheEvictions, + long requestCacheInBytes) { + super(); + this.shardStats = null; + + this.queryCacheHitCount = queryCacheHitCount; + this.queryCacheMissCount = queryCacheMissCount; + this.queryCacheInBytes = queryCacheInBytes; + this.fieldDataEvictions = fieldDataEvictions; + this.fieldDataInBytes = fieldDataInBytes; + this.requestCacheHitCount = requestCacheHitCount; + this.requestCacheMissCount = requestCacheMissCount; + this.requestCacheEvictions = requestCacheEvictions; + this.requestCacheInBytes = requestCacheInBytes; + } + + private long calculate(RTFMetrics.ShardStatsValue nodeMetric) { + return valueCalculators.get(nodeMetric.toString()).calculateValue(shardStats); + } + + @JsonIgnore + public ShardStats getShardStats() { + return shardStats; + } + + @JsonProperty(RTFMetrics.ShardStatsValue.Constants.QUEY_CACHE_HIT_COUNT_VALUE) + public long getQueryCacheHitCount() { + return queryCacheHitCount; + } + + @JsonProperty(RTFMetrics.ShardStatsValue.Constants.QUERY_CACHE_MISS_COUNT_VALUE) + public long getQueryCacheMissCount() { + return queryCacheMissCount; + } + + @JsonProperty(RTFMetrics.ShardStatsValue.Constants.QUERY_CACHE_IN_BYTES_VALUE) + public long getQueryCacheInBytes() { + return queryCacheInBytes; + } + + @JsonProperty(RTFMetrics.ShardStatsValue.Constants.FIELDDATA_EVICTION_VALUE) + public long getFieldDataEvictions() { + return fieldDataEvictions; + } + + @JsonProperty(RTFMetrics.ShardStatsValue.Constants.FIELD_DATA_IN_BYTES_VALUE) + public long getFieldDataInBytes() { + return fieldDataInBytes; + } + + @JsonProperty(RTFMetrics.ShardStatsValue.Constants.REQUEST_CACHE_HIT_COUNT_VALUE) + public long getRequestCacheHitCount() { + return requestCacheHitCount; + } + + @JsonProperty(RTFMetrics.ShardStatsValue.Constants.REQUEST_CACHE_MISS_COUNT_VALUE) + public long getRequestCacheMissCount() { + return requestCacheMissCount; + } + + @JsonProperty(RTFMetrics.ShardStatsValue.Constants.REQUEST_CACHE_EVICTION_VALUE) + public long getRequestCacheEvictions() { + return requestCacheEvictions; + } + + @JsonProperty(RTFMetrics.ShardStatsValue.Constants.REQUEST_CACHE_IN_BYTES_VALUE) + public long getRequestCacheInBytes() { + return requestCacheInBytes; + } + } +} diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFThreadPoolMetricsCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFThreadPoolMetricsCollector.java new file mode 100644 index 00000000..46ad8ae9 --- /dev/null +++ b/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFThreadPoolMetricsCollector.java @@ -0,0 +1,228 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.collectors.telemetry; + +import static org.opensearch.performanceanalyzer.commons.stats.metrics.StatExceptionCode.RTF_THREADPOOL_METRICS_COLLECTOR_ERROR; +import static org.opensearch.performanceanalyzer.commons.stats.metrics.StatExceptionCode.THREADPOOL_METRICS_COLLECTOR_ERROR; +import static org.opensearch.performanceanalyzer.commons.stats.metrics.StatMetrics.RTF_THREADPOOL_METRICS_COLLECTOR_EXECUTION_TIME; + +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ThreadPoolExecutor; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.util.concurrent.SizeBlockingQueue; +import org.opensearch.performanceanalyzer.OpenSearchResources; +import org.opensearch.performanceanalyzer.commons.collectors.PerformanceAnalyzerMetricsCollector; +import org.opensearch.performanceanalyzer.commons.collectors.StatsCollector; +import org.opensearch.performanceanalyzer.commons.collectors.TelemetryCollector; +import org.opensearch.performanceanalyzer.commons.config.overrides.ConfigOverridesWrapper; +import org.opensearch.performanceanalyzer.commons.metrics.MetricsConfiguration; +import org.opensearch.performanceanalyzer.commons.metrics.RTFMetrics; +import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; +import org.opensearch.telemetry.metrics.Histogram; +import org.opensearch.telemetry.metrics.MetricsRegistry; +import org.opensearch.telemetry.metrics.tags.Tags; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.threadpool.ThreadPoolStats; + +public class RTFThreadPoolMetricsCollector extends PerformanceAnalyzerMetricsCollector + implements TelemetryCollector { + + private static final Logger LOG = LogManager.getLogger(RTFThreadPoolMetricsCollector.class); + public static final int SAMPLING_TIME_INTERVAL = + MetricsConfiguration.CONFIG_MAP.get(RTFThreadPoolMetricsCollector.class) + .samplingInterval; + private final Map statsRecordMap; + private Histogram threadPoolQueueSizeMetrics; + private Histogram threadPoolRejectedReqsMetrics; + private Histogram threadPoolTotalThreadsMetrics; + private Histogram threadPoolActiveThreadsMetrics; + + // Skipping ThreadPoolQueueLatencyMetrics since they are always emitting -1 in the original + // collector + // private Histogram ThreadPoolQueueLatencyMetrics; + private Histogram ThreadPoolQueueCapacityMetrics; + private MetricsRegistry metricsRegistry; + private boolean metricsInitialised; + private PerformanceAnalyzerController performanceAnalyzerController; + private ConfigOverridesWrapper configOverridesWrapper; + + public RTFThreadPoolMetricsCollector( + PerformanceAnalyzerController performanceAnalyzerController, + ConfigOverridesWrapper configOverridesWrapper) { + super( + SAMPLING_TIME_INTERVAL, + "RTFThreadPoolMetricsCollector", + RTF_THREADPOOL_METRICS_COLLECTOR_EXECUTION_TIME, + RTF_THREADPOOL_METRICS_COLLECTOR_ERROR); + statsRecordMap = new HashMap<>(); + this.metricsInitialised = false; + this.performanceAnalyzerController = performanceAnalyzerController; + this.configOverridesWrapper = configOverridesWrapper; + } + + @Override + public void collectMetrics(long startTime) { + if (performanceAnalyzerController.isCollectorDisabled( + configOverridesWrapper, getCollectorName())) { + LOG.info("RTFDisksCollector is disabled. Skipping collection."); + return; + } + + if (OpenSearchResources.INSTANCE.getThreadPool() == null) { + return; + } + + metricsRegistry = OpenSearchResources.INSTANCE.getMetricsRegistry(); + if (metricsRegistry == null) { + LOG.error("could not get the instance of MetricsRegistry class"); + return; + } + + LOG.debug("Executing collect metrics for RTFThreadPoolMetricsCollector"); + + initialiseMetricsIfNeeded(); + + Iterator statsIterator = + OpenSearchResources.INSTANCE.getThreadPool().stats().iterator(); + + while (statsIterator.hasNext()) { + ThreadPoolStats.Stats stats = statsIterator.next(); + long rejectionDelta = 0; + String threadPoolName = stats.getName(); + if (statsRecordMap.containsKey(threadPoolName)) { + ThreadPoolStatsRecord lastRecord = statsRecordMap.get(threadPoolName); + // if the timestamp in previous record is greater than 15s (3 * intervals), + // then the scheduler might hang or freeze due to long GC etc. We simply drop + // previous record here and set rejectionDelta to 0. + if (startTime - lastRecord.getTimestamp() <= SAMPLING_TIME_INTERVAL * 3L) { + rejectionDelta = stats.getRejected() - lastRecord.getRejected(); + // we might not run into this as rejection is a LongAdder which never decrement + // its count. + // regardless, let's set it to 0 to be safe. + if (rejectionDelta < 0) { + rejectionDelta = 0; + } + } + } + statsRecordMap.put( + threadPoolName, new ThreadPoolStatsRecord(startTime, stats.getRejected())); + final long finalRejectionDelta = rejectionDelta; + final int capacity = + AccessController.doPrivileged( + (PrivilegedAction) + () -> { + try { + ThreadPool threadPool = + (ThreadPool) + FieldUtils.readField( + OpenSearchResources.INSTANCE + .getIndicesService(), + "threadPool", + true); + ThreadPoolExecutor threadPoolExecutor = + (ThreadPoolExecutor) + threadPool.executor(threadPoolName); + Object queue = threadPoolExecutor.getQueue(); + // TODO: we might want to read the capacity of + // SifiResizableBlockingQueue in the future. + // In order to do that we can create a new + // PerformanceAnalyzerLibrary package and push + // all the code which depends on core OpenSearch + // specific + // changes into that library. + if (queue instanceof SizeBlockingQueue) { + return ((SizeBlockingQueue) queue).capacity(); + } + } catch (Exception e) { + LOG.warn("Fail to read queue capacity via reflection"); + StatsCollector.instance() + .logException( + THREADPOOL_METRICS_COLLECTOR_ERROR); + } + return -1; + }); + + recordMetrics(stats, finalRejectionDelta, capacity); + } + } + + private void recordMetrics( + ThreadPoolStats.Stats stats, long finalRejectionDelta, int capacity) { + Tags threadPoolTypeTag = + Tags.create() + .addTag( + RTFMetrics.ThreadPoolDimension.THREAD_POOL_TYPE.toString(), + stats.getName()); + + threadPoolQueueSizeMetrics.record(stats.getQueue(), threadPoolTypeTag); + threadPoolRejectedReqsMetrics.record(finalRejectionDelta, threadPoolTypeTag); + threadPoolActiveThreadsMetrics.record(stats.getActive(), threadPoolTypeTag); + threadPoolTotalThreadsMetrics.record(stats.getThreads(), threadPoolTypeTag); + + if (capacity >= 0) { + ThreadPoolQueueCapacityMetrics.record(capacity, threadPoolTypeTag); + } + } + + private void initialiseMetricsIfNeeded() { + if (metricsInitialised == false) { + threadPoolQueueSizeMetrics = + metricsRegistry.createHistogram( + RTFMetrics.ThreadPoolValue.Constants.QUEUE_SIZE_VALUE, + "ThreadPool Queue Size Metrics", + RTFMetrics.MetricUnits.COUNT.toString()); + + threadPoolRejectedReqsMetrics = + metricsRegistry.createHistogram( + RTFMetrics.ThreadPoolValue.Constants.REJECTED_VALUE, + "ThreadPool Rejected Reqs Metrics", + RTFMetrics.MetricUnits.COUNT.toString()); + + threadPoolTotalThreadsMetrics = + metricsRegistry.createHistogram( + RTFMetrics.ThreadPoolValue.Constants.THREADS_COUNT_VALUE, + "ThreadPool Total Threads Metrics", + RTFMetrics.MetricUnits.COUNT.toString()); + + threadPoolActiveThreadsMetrics = + metricsRegistry.createHistogram( + RTFMetrics.ThreadPoolValue.Constants.THREADS_ACTIVE_VALUE, + "ThreadPool Active Threads Metrics", + RTFMetrics.MetricUnits.COUNT.toString()); + + ThreadPoolQueueCapacityMetrics = + metricsRegistry.createHistogram( + RTFMetrics.ThreadPoolValue.Constants.QUEUE_CAPACITY_VALUE, + "ThreadPool Queue Capacity Metrics", + RTFMetrics.MetricUnits.COUNT.toString()); + metricsInitialised = true; + } + } + + private static class ThreadPoolStatsRecord { + private final long timestamp; + private final long rejected; + + ThreadPoolStatsRecord(long timestamp, long rejected) { + this.timestamp = timestamp; + this.rejected = rejected; + } + + public long getTimestamp() { + return timestamp; + } + + public long getRejected() { + return rejected; + } + } +} diff --git a/src/main/java/org/opensearch/performanceanalyzer/config/PerformanceAnalyzerController.java b/src/main/java/org/opensearch/performanceanalyzer/config/PerformanceAnalyzerController.java index 1fb6994d..faaebc7c 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/config/PerformanceAnalyzerController.java +++ b/src/main/java/org/opensearch/performanceanalyzer/config/PerformanceAnalyzerController.java @@ -35,6 +35,7 @@ public class PerformanceAnalyzerController { "thread_contention_monitoring_enabled.conf"; private static final Logger LOG = LogManager.getLogger(PerformanceAnalyzerController.class); public static final int DEFAULT_NUM_OF_SHARDS_PER_COLLECTION = 0; + public static final int DEFAULT_COLLECTORS_SETTING_VALUE = 0; private boolean paEnabled; private boolean rcaEnabled; @@ -42,6 +43,7 @@ public class PerformanceAnalyzerController { private boolean batchMetricsEnabled; private boolean threadContentionMonitoringEnabled; private volatile int shardsPerCollection; + private volatile int collectorsSettingValue; private static final boolean paEnabledDefaultValue = false; private static final boolean rcaEnabledDefaultValue = true; private static final boolean loggingEnabledDefaultValue = false; @@ -58,6 +60,7 @@ public PerformanceAnalyzerController( initBatchMetricsStateFromConf(); initThreadContentionMonitoringStateFromConf(); shardsPerCollection = DEFAULT_NUM_OF_SHARDS_PER_COLLECTION; + collectorsSettingValue = DEFAULT_COLLECTORS_SETTING_VALUE; } /** @@ -115,6 +118,18 @@ public void updateNodeStatsShardsPerCollection(int value) { shardsPerCollection = value; } + /** + * Updates the collector mode setting + * + * @param value the desired collector mode value + */ + public void updateCollectorsSetting(int value) { + collectorsSettingValue = value; + if (scheduledMetricCollectorsExecutor != null) { + scheduledMetricCollectorsExecutor.setCollectorsSetting(collectorsSettingValue); + } + } + /** * Updates the state of performance analyzer(writer and engine). * diff --git a/src/main/java/org/opensearch/performanceanalyzer/config/setting/PerformanceAnalyzerClusterSettings.java b/src/main/java/org/opensearch/performanceanalyzer/config/setting/PerformanceAnalyzerClusterSettings.java index 035817c7..e1775128 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/config/setting/PerformanceAnalyzerClusterSettings.java +++ b/src/main/java/org/opensearch/performanceanalyzer/config/setting/PerformanceAnalyzerClusterSettings.java @@ -39,6 +39,17 @@ public enum PerformanceAnalyzerFeatureBits { Setting.Property.NodeScope, Setting.Property.Dynamic); + /** + * Cluster setting controlling the collector mode : 0 - only RCA Collectors enabled (Default) 1 + * - only Telemetry Collectors enabled 2 - both RCA and Telemetry Collectors enabled + */ + public static final Setting PA_COLLECTORS_SETTING = + Setting.intSetting( + "cluster.metadata.perf_analyzer.collectors.mode", + 0, + Setting.Property.NodeScope, + Setting.Property.Dynamic); + /** * Cluster setting controlling the config overrides to be applied on performance analyzer * components. diff --git a/src/main/java/org/opensearch/performanceanalyzer/config/setting/handler/PerformanceAnalyzerCollectorsSettingHandler.java b/src/main/java/org/opensearch/performanceanalyzer/config/setting/handler/PerformanceAnalyzerCollectorsSettingHandler.java new file mode 100644 index 00000000..a5034262 --- /dev/null +++ b/src/main/java/org/opensearch/performanceanalyzer/config/setting/handler/PerformanceAnalyzerCollectorsSettingHandler.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.config.setting.handler; + +import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; +import org.opensearch.performanceanalyzer.config.setting.ClusterSettingListener; +import org.opensearch.performanceanalyzer.config.setting.ClusterSettingsManager; +import org.opensearch.performanceanalyzer.config.setting.PerformanceAnalyzerClusterSettings; + +/* + * This class is responsible for handling the collector setting value updates through API calls. + * This also acts as a listener to the collector mode setting update + */ +public class PerformanceAnalyzerCollectorsSettingHandler + implements ClusterSettingListener { + private final PerformanceAnalyzerController controller; + private final ClusterSettingsManager clusterSettingsManager; + + private Integer currentClusterSetting = + PerformanceAnalyzerController.DEFAULT_COLLECTORS_SETTING_VALUE; + + public PerformanceAnalyzerCollectorsSettingHandler( + PerformanceAnalyzerController controller, + ClusterSettingsManager clusterSettingsManager) { + this.controller = controller; + this.clusterSettingsManager = clusterSettingsManager; + } + + /** + * Updates the Collectors mode setting across the cluster. + * + * @param value The desired collector mode amongst: 0 - only RCA Collectors enabled (Default) * + * 1 - only Telemetry Collectors enabled * 2 - both RCA and Telemetry Collectors enabled + */ + public void updateCollectorsSetting(final int value) { + clusterSettingsManager.updateSetting( + PerformanceAnalyzerClusterSettings.PA_COLLECTORS_SETTING, value); + } + + /** + * Handler that gets called when there is a new value for the setting that this listener is + * listening to. + * + * @param newSettingValue The value of the new setting. + */ + @Override + public void onSettingUpdate(final Integer newSettingValue) { + if (newSettingValue != null) { + currentClusterSetting = newSettingValue; + controller.updateCollectorsSetting(newSettingValue); + } + } + + /** + * Gets the current(last seen) cluster setting value. + * + * @return integer value for setting. + */ + public int getCollectorsEnabledSetting() { + return currentClusterSetting; + } +} diff --git a/src/main/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerClusterConfigAction.java b/src/main/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerClusterConfigAction.java index df7b503c..b1495134 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerClusterConfigAction.java +++ b/src/main/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerClusterConfigAction.java @@ -22,6 +22,7 @@ import org.opensearch.performanceanalyzer.commons.config.PluginSettings; import org.opensearch.performanceanalyzer.config.setting.handler.NodeStatsSettingHandler; import org.opensearch.performanceanalyzer.config.setting.handler.PerformanceAnalyzerClusterSettingHandler; +import org.opensearch.performanceanalyzer.config.setting.handler.PerformanceAnalyzerCollectorsSettingHandler; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestController; @@ -40,6 +41,7 @@ public class PerformanceAnalyzerClusterConfigAction extends BaseRestHandler { "batchMetricsRetentionPeriodMinutes"; public static final String ENABLED = "enabled"; public static final String SHARDS_PER_COLLECTION = "shardsPerCollection"; + public static final String COLLECTORS_SETTING = "collectorsSetting"; public static final String PA_CLUSTER_CONFIG_PATH = RestConfig.PA_BASE_URI + "/cluster/config"; public static final String RCA_CLUSTER_CONFIG_PATH = @@ -115,14 +117,20 @@ public class PerformanceAnalyzerClusterConfigAction extends BaseRestHandler { private final PerformanceAnalyzerClusterSettingHandler clusterSettingHandler; private final NodeStatsSettingHandler nodeStatsSettingHandler; + private final PerformanceAnalyzerCollectorsSettingHandler + performanceAnalyzerCollectorsSettingHandler; public PerformanceAnalyzerClusterConfigAction( final Settings settings, final RestController restController, final PerformanceAnalyzerClusterSettingHandler clusterSettingHandler, - final NodeStatsSettingHandler nodeStatsSettingHandler) { + final NodeStatsSettingHandler nodeStatsSettingHandler, + final PerformanceAnalyzerCollectorsSettingHandler + performanceAnalyzerCollectorsSettingHandler) { this.clusterSettingHandler = clusterSettingHandler; this.nodeStatsSettingHandler = nodeStatsSettingHandler; + this.performanceAnalyzerCollectorsSettingHandler = + performanceAnalyzerCollectorsSettingHandler; } /** @@ -195,6 +203,14 @@ protected RestChannelConsumer prepareRequest(final RestRequest request, final No (Integer) shardPerCollectionValue); } } + + if (map.containsKey(COLLECTORS_SETTING)) { + Object collectorsSettingValue = map.get(COLLECTORS_SETTING); + if (collectorsSettingValue instanceof Integer) { + performanceAnalyzerCollectorsSettingHandler.updateCollectorsSetting( + (Integer) collectorsSettingValue); + } + } } return channel -> { @@ -207,6 +223,9 @@ protected RestChannelConsumer prepareRequest(final RestRequest request, final No ? clusterSettingHandler.getCurrentClusterSettingValueVerbose() : clusterSettingHandler.getCurrentClusterSettingValue()); builder.field(SHARDS_PER_COLLECTION, nodeStatsSettingHandler.getNodeStatsSetting()); + builder.field( + COLLECTORS_SETTING, + performanceAnalyzerCollectorsSettingHandler.getCollectorsEnabledSetting()); builder.field( BATCH_METRICS_RETENTION_PERIOD_MINUTES, PluginSettings.instance().getBatchMetricsRetentionPeriodMinutes()); diff --git a/src/main/java/org/opensearch/performanceanalyzer/util/Utils.java b/src/main/java/org/opensearch/performanceanalyzer/util/Utils.java index 08f62cc0..e4963ebc 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/util/Utils.java +++ b/src/main/java/org/opensearch/performanceanalyzer/util/Utils.java @@ -19,6 +19,10 @@ import org.opensearch.indices.IndicesService; import org.opensearch.performanceanalyzer.OpenSearchResources; import org.opensearch.performanceanalyzer.collectors.*; +import org.opensearch.performanceanalyzer.collectors.telemetry.RTFDisksCollector; +import org.opensearch.performanceanalyzer.collectors.telemetry.RTFHeapMetricsCollector; +import org.opensearch.performanceanalyzer.collectors.telemetry.RTFNodeStatsAllShardsMetricsCollector; +import org.opensearch.performanceanalyzer.collectors.telemetry.RTFThreadPoolMetricsCollector; import org.opensearch.performanceanalyzer.commons.metrics.MetricsConfiguration; import org.opensearch.performanceanalyzer.commons.stats.ServiceMetrics; @@ -43,6 +47,10 @@ public static void configureMetrics() { MetricsConfiguration.CONFIG_MAP.put(SearchBackPressureStatsCollector.class, cdefault); MetricsConfiguration.CONFIG_MAP.put(ElectionTermCollector.class, cdefault); MetricsConfiguration.CONFIG_MAP.put(ShardIndexingPressureMetricsCollector.class, cdefault); + MetricsConfiguration.CONFIG_MAP.put(RTFDisksCollector.class, cdefault); + MetricsConfiguration.CONFIG_MAP.put(RTFHeapMetricsCollector.class, cdefault); + MetricsConfiguration.CONFIG_MAP.put(RTFNodeStatsAllShardsMetricsCollector.class, cdefault); + MetricsConfiguration.CONFIG_MAP.put(RTFThreadPoolMetricsCollector.class, cdefault); } // These methods are utility functions for the Node Stat Metrics Collectors. These methods are diff --git a/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPluginTests.java b/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPluginTests.java index e3045e1a..fc5f95fa 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPluginTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPluginTests.java @@ -22,6 +22,7 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.support.ActionFilter; import org.opensearch.client.node.NodeClient; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; @@ -43,6 +44,8 @@ import org.opensearch.plugins.ActionPlugin.ActionHandler; import org.opensearch.rest.RestController; import org.opensearch.rest.RestHandler; +import org.opensearch.telemetry.metrics.MetricsRegistry; +import org.opensearch.telemetry.metrics.NoopMetricsRegistryFactory; import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; @@ -64,6 +67,8 @@ public class PerformanceAnalyzerPluginTests extends OpenSearchTestCase { private ClusterSettings clusterSettings; private IdentityService identityService; + private MetricsRegistry metricsRegistry; + @Before public void setup() { initMocks(this); @@ -78,7 +83,19 @@ public void setup() { threadPool = new TestThreadPool("test"); nodeClient = new NodeClient(settings, threadPool); environment = TestEnvironment.newEnvironment(settings); - clusterService = new ClusterService(settings, clusterSettings, threadPool); + NoopMetricsRegistryFactory metricsRegistryFactory = new NoopMetricsRegistryFactory(); + metricsRegistry = metricsRegistryFactory.getMetricsRegistry(); + try { + metricsRegistryFactory.close(); + } catch (Exception e) { + e.printStackTrace(); + } + clusterService = + new ClusterService( + settings, + clusterSettings, + threadPool, + new ClusterManagerMetrics(metricsRegistry)); identityService = new IdentityService(Settings.EMPTY, List.of()); restController = new RestController( @@ -144,12 +161,15 @@ public void testCreateComponents() { null, null, null, - null); + null, + null, + metricsRegistry); assertEquals(1, components.size()); assertEquals(settings, OpenSearchResources.INSTANCE.getSettings()); assertEquals(threadPool, OpenSearchResources.INSTANCE.getThreadPool()); assertEquals(environment, OpenSearchResources.INSTANCE.getEnvironment()); assertEquals(nodeClient, OpenSearchResources.INSTANCE.getClient()); + assertEquals(metricsRegistry, OpenSearchResources.INSTANCE.getMetricsRegistry()); } @Test @@ -172,9 +192,10 @@ public void testGetTransports() { @Test public void testGetSettings() { List> list = plugin.getSettings(); - assertEquals(3, list.size()); + assertEquals(4, list.size()); assertEquals(PerformanceAnalyzerClusterSettings.COMPOSITE_PA_SETTING, list.get(0)); assertEquals(PerformanceAnalyzerClusterSettings.PA_NODE_STATS_SETTING, list.get(1)); assertEquals(PerformanceAnalyzerClusterSettings.CONFIG_OVERRIDES_SETTING, list.get(2)); + assertEquals(PerformanceAnalyzerClusterSettings.PA_COLLECTORS_SETTING, list.get(3)); } } diff --git a/src/test/java/org/opensearch/performanceanalyzer/collectors/CollectorTestBase.java b/src/test/java/org/opensearch/performanceanalyzer/collectors/CollectorTestBase.java new file mode 100644 index 00000000..26f6baa1 --- /dev/null +++ b/src/test/java/org/opensearch/performanceanalyzer/collectors/CollectorTestBase.java @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.collectors; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.MockitoAnnotations.initMocks; + +import org.junit.Before; +import org.mockito.Mockito; +import org.opensearch.performanceanalyzer.commons.config.overrides.ConfigOverridesWrapper; +import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; + +public class CollectorTestBase { + protected PerformanceAnalyzerController mockController; + protected ConfigOverridesWrapper mockWrapper; + + @Before + public void setUp() throws Exception { + initMocks(this); + System.setProperty("performanceanalyzer.metrics.log.enabled", "false"); + mockController = mock(PerformanceAnalyzerController.class); + mockWrapper = mock(ConfigOverridesWrapper.class); + Mockito.when(mockController.isCollectorDisabled(any(), anyString())).thenReturn(false); + } +} diff --git a/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFDisksCollectorTests.java b/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFDisksCollectorTests.java new file mode 100644 index 00000000..4dc9bab8 --- /dev/null +++ b/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFDisksCollectorTests.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.collectors.telemetry; + +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import java.io.IOException; +import org.junit.Before; +import org.junit.Test; +import org.opensearch.performanceanalyzer.OpenSearchResources; +import org.opensearch.performanceanalyzer.collectors.CollectorTestBase; +import org.opensearch.performanceanalyzer.commons.metrics.MetricsConfiguration; +import org.opensearch.telemetry.metrics.Histogram; +import org.opensearch.telemetry.metrics.MetricsRegistry; + +public class RTFDisksCollectorTests extends CollectorTestBase { + private RTFDisksCollector rtfDisksCollector; + private static MetricsRegistry metricsRegistry; + private static Histogram testHistogram; + + @Before + public void init() { + MetricsConfiguration.CONFIG_MAP.put(RTFDisksCollector.class, MetricsConfiguration.cdefault); + System.setProperty("os.name", "Linux"); + metricsRegistry = mock(MetricsRegistry.class); + testHistogram = mock(Histogram.class); + OpenSearchResources.INSTANCE.setMetricsRegistry(metricsRegistry); + when(metricsRegistry.createHistogram(anyString(), anyString(), anyString())) + .thenReturn(testHistogram); + rtfDisksCollector = spy(new RTFDisksCollector(mockController, mockWrapper)); + } + + @Test + public void testCollectMetrics() throws IOException { + rtfDisksCollector.collectMetrics(System.currentTimeMillis()); + verify(rtfDisksCollector, atLeastOnce()).recordMetrics(any()); + } +} diff --git a/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFHeapMetricsCollectorTests.java b/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFHeapMetricsCollectorTests.java new file mode 100644 index 00000000..928a5a29 --- /dev/null +++ b/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFHeapMetricsCollectorTests.java @@ -0,0 +1,68 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.collectors.telemetry; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyDouble; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; +import static org.mockito.Mockito.atLeastOnce; + +import java.io.IOException; +import org.junit.Before; +import org.junit.Test; +import org.opensearch.performanceanalyzer.OpenSearchResources; +import org.opensearch.performanceanalyzer.collectors.CollectorTestBase; +import org.opensearch.performanceanalyzer.commons.metrics.MetricsConfiguration; +import org.opensearch.performanceanalyzer.commons.metrics.RTFMetrics; +import org.opensearch.telemetry.metrics.Histogram; +import org.opensearch.telemetry.metrics.MetricsRegistry; + +public class RTFHeapMetricsCollectorTests extends CollectorTestBase { + private RTFHeapMetricsCollector rtfHeapMetricsCollector; + + private static MetricsRegistry metricsRegistry; + private static Histogram gcCollectionEventHistogram; + private static Histogram gcCollectionTimeHistogram; + private static Histogram heapUsedHistogram; + + @Before + public void init() { + MetricsConfiguration.CONFIG_MAP.put( + RTFHeapMetricsCollector.class, MetricsConfiguration.cdefault); + + metricsRegistry = mock(MetricsRegistry.class); + gcCollectionEventHistogram = mock(Histogram.class); + gcCollectionTimeHistogram = mock(Histogram.class); + heapUsedHistogram = mock(Histogram.class); + OpenSearchResources.INSTANCE.setMetricsRegistry(metricsRegistry); + + when(metricsRegistry.createHistogram(anyString(), anyString(), anyString())) + .thenAnswer( + invocationOnMock -> { + String histogramName = (String) invocationOnMock.getArguments()[0]; + if (histogramName.contains( + RTFMetrics.HeapValue.Constants.COLLECTION_COUNT_VALUE)) { + return gcCollectionEventHistogram; + } else if (histogramName.contains( + RTFMetrics.HeapValue.Constants.COLLECTION_TIME_VALUE)) { + return gcCollectionTimeHistogram; + } + return heapUsedHistogram; + }); + rtfHeapMetricsCollector = new RTFHeapMetricsCollector(mockController, mockWrapper); + } + + @Test + public void testCollectMetrics() throws IOException { + rtfHeapMetricsCollector.collectMetrics(System.currentTimeMillis()); + verify(heapUsedHistogram, atLeastOnce()).record(anyDouble(), any()); + verify(gcCollectionTimeHistogram, atLeastOnce()).record(anyDouble(), any()); + verify(gcCollectionEventHistogram, atLeastOnce()).record(anyDouble(), any()); + verify(metricsRegistry, atLeastOnce()) + .createGauge(anyString(), anyString(), anyString(), any(), any()); + } +} diff --git a/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFNodeStatsAllShardsMetricsCollectorTests.java b/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFNodeStatsAllShardsMetricsCollectorTests.java new file mode 100644 index 00000000..089a685b --- /dev/null +++ b/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFNodeStatsAllShardsMetricsCollectorTests.java @@ -0,0 +1,138 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.collectors.telemetry; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyDouble; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; +import static org.mockito.Mockito.atLeastOnce; + +import java.io.IOException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.opensearch.indices.IndicesService; +import org.opensearch.performanceanalyzer.OpenSearchResources; +import org.opensearch.performanceanalyzer.commons.config.overrides.ConfigOverridesWrapper; +import org.opensearch.performanceanalyzer.commons.metrics.MetricsConfiguration; +import org.opensearch.performanceanalyzer.commons.metrics.RTFMetrics; +import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; +import org.opensearch.telemetry.metrics.Counter; +import org.opensearch.telemetry.metrics.MetricsRegistry; +import org.opensearch.test.OpenSearchSingleNodeTestCase; + +public class RTFNodeStatsAllShardsMetricsCollectorTests extends OpenSearchSingleNodeTestCase { + + private long startTimeInMills = 1153721339; + private static final String TEST_INDEX = "test"; + private RTFNodeStatsAllShardsMetricsCollector rtfNodeStatsAllShardsMetricsCollector; + private static MetricsRegistry metricsRegistry; + private static Counter cacheQueryHitCounter; + private static Counter cacheQueryMissCounter; + private static Counter cacheQuerySizeCounter; + private static Counter cacheFieldDataEvictionCounter; + private static Counter cacheFieldDataSizeCounter; + private static Counter cacheRequestHitCounter; + private static Counter cacheRequestMissCounter; + private static Counter cacheRequestEvictionCounter; + private static Counter cacheRequestSizeCounter; + + @Before + public void init() { + MetricsConfiguration.CONFIG_MAP.put( + RTFNodeStatsAllShardsMetricsCollector.class, MetricsConfiguration.cdefault); + + IndicesService indicesService = getInstanceFromNode(IndicesService.class); + OpenSearchResources.INSTANCE.setIndicesService(indicesService); + metricsRegistry = mock(MetricsRegistry.class); + cacheFieldDataEvictionCounter = mock(Counter.class); + cacheFieldDataSizeCounter = mock(Counter.class); + cacheQueryHitCounter = mock(Counter.class); + cacheQueryMissCounter = mock(Counter.class); + cacheRequestEvictionCounter = mock(Counter.class); + cacheRequestMissCounter = mock(Counter.class); + cacheRequestHitCounter = mock(Counter.class); + cacheRequestSizeCounter = mock(Counter.class); + cacheQuerySizeCounter = mock(Counter.class); + + OpenSearchResources.INSTANCE.setMetricsRegistry(metricsRegistry); + + when(metricsRegistry.createCounter(anyString(), anyString(), anyString())) + .thenAnswer( + invocationOnMock -> { + String counterName = (String) invocationOnMock.getArguments()[0]; + if (counterName.contains( + RTFMetrics.ShardStatsValue.Constants + .REQUEST_CACHE_IN_BYTES_VALUE)) { + return cacheRequestSizeCounter; + } else if (counterName.contains( + RTFMetrics.ShardStatsValue.Constants + .REQUEST_CACHE_HIT_COUNT_VALUE)) { + return cacheRequestHitCounter; + } else if (counterName.contains( + RTFMetrics.ShardStatsValue.Constants + .REQUEST_CACHE_MISS_COUNT_VALUE)) { + return cacheRequestMissCounter; + } else if (counterName.contains( + RTFMetrics.ShardStatsValue.Constants + .REQUEST_CACHE_EVICTION_VALUE)) { + return cacheRequestEvictionCounter; + } else if (counterName.contains( + RTFMetrics.ShardStatsValue.Constants + .FIELDDATA_EVICTION_VALUE)) { + return cacheFieldDataEvictionCounter; + } else if (counterName.contains( + RTFMetrics.ShardStatsValue.Constants + .FIELD_DATA_IN_BYTES_VALUE)) { + return cacheFieldDataSizeCounter; + } else if (counterName.contains( + RTFMetrics.ShardStatsValue.Constants + .QUERY_CACHE_IN_BYTES_VALUE)) { + return cacheQuerySizeCounter; + } else if (counterName.contains( + RTFMetrics.ShardStatsValue.Constants + .QUEY_CACHE_HIT_COUNT_VALUE)) { + return cacheQueryHitCounter; + } + return cacheQueryMissCounter; + }); + + ConfigOverridesWrapper mockWrapper = mock(ConfigOverridesWrapper.class); + PerformanceAnalyzerController mockController = mock(PerformanceAnalyzerController.class); + Mockito.when(mockController.isCollectorDisabled(any(), anyString())).thenReturn(false); + + rtfNodeStatsAllShardsMetricsCollector = + spy(new RTFNodeStatsAllShardsMetricsCollector(mockController, mockWrapper)); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + } + + @Test + public void testCollectMetrics() throws IOException { + createIndex(TEST_INDEX); + rtfNodeStatsAllShardsMetricsCollector.collectMetrics(startTimeInMills); + verify(rtfNodeStatsAllShardsMetricsCollector, never()) + .populateDiffMetricValue(any(), any(), anyString(), anyInt()); + startTimeInMills += 500; + rtfNodeStatsAllShardsMetricsCollector.collectMetrics(startTimeInMills); + verify(rtfNodeStatsAllShardsMetricsCollector, times(1)) + .populateDiffMetricValue(any(), any(), anyString(), anyInt()); + verify(cacheFieldDataEvictionCounter, atLeastOnce()).add(anyDouble(), any()); + verify(cacheFieldDataSizeCounter, atLeastOnce()).add(anyDouble(), any()); + verify(cacheQueryMissCounter, atLeastOnce()).add(anyDouble(), any()); + verify(cacheQueryHitCounter, atLeastOnce()).add(anyDouble(), any()); + verify(cacheQuerySizeCounter, atLeastOnce()).add(anyDouble(), any()); + verify(cacheRequestEvictionCounter, atLeastOnce()).add(anyDouble(), any()); + verify(cacheRequestHitCounter, atLeastOnce()).add(anyDouble(), any()); + verify(cacheRequestMissCounter, atLeastOnce()).add(anyDouble(), any()); + verify(cacheRequestSizeCounter, atLeastOnce()).add(anyDouble(), any()); + } +} diff --git a/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFThreadPoolMetricsCollectorTests.java b/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFThreadPoolMetricsCollectorTests.java new file mode 100644 index 00000000..895e8be7 --- /dev/null +++ b/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFThreadPoolMetricsCollectorTests.java @@ -0,0 +1,91 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.collectors.telemetry; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; +import static org.mockito.Mockito.atLeastOnce; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.opensearch.performanceanalyzer.OpenSearchResources; +import org.opensearch.performanceanalyzer.collectors.CollectorTestBase; +import org.opensearch.performanceanalyzer.commons.metrics.MetricsConfiguration; +import org.opensearch.performanceanalyzer.commons.metrics.RTFMetrics; +import org.opensearch.telemetry.metrics.Histogram; +import org.opensearch.telemetry.metrics.MetricsRegistry; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.threadpool.ThreadPoolStats; + +public class RTFThreadPoolMetricsCollectorTests extends CollectorTestBase { + private RTFThreadPoolMetricsCollector rtfThreadPoolMetricsCollector; + private static MetricsRegistry metricsRegistry; + private static Histogram threadPoolQueueSizeHistogram; + private static Histogram threadPoolRejectedReqsHistogram; + private static Histogram threadPoolTotalThreadsHistogram; + private static Histogram threadPoolActiveThreadsHistogram; + private static Histogram threadPoolQueueCapacityHistogram; + @Mock private ThreadPool mockThreadPool; + + @Before + public void init() { + MetricsConfiguration.CONFIG_MAP.put( + RTFThreadPoolMetricsCollector.class, MetricsConfiguration.cdefault); + metricsRegistry = mock(MetricsRegistry.class); + threadPoolQueueSizeHistogram = mock(Histogram.class); + threadPoolRejectedReqsHistogram = mock(Histogram.class); + threadPoolActiveThreadsHistogram = mock(Histogram.class); + threadPoolTotalThreadsHistogram = mock(Histogram.class); + threadPoolQueueCapacityHistogram = mock(Histogram.class); + + OpenSearchResources.INSTANCE.setMetricsRegistry(metricsRegistry); + OpenSearchResources.INSTANCE.setThreadPool(mockThreadPool); + + when(metricsRegistry.createHistogram(anyString(), anyString(), anyString())) + .thenAnswer( + invocationOnMock -> { + String histogramName = (String) invocationOnMock.getArguments()[0]; + if (histogramName.contains( + RTFMetrics.ThreadPoolValue.Constants.QUEUE_SIZE_VALUE)) { + return threadPoolQueueSizeHistogram; + } else if (histogramName.contains( + RTFMetrics.ThreadPoolValue.Constants.REJECTED_VALUE)) { + return threadPoolRejectedReqsHistogram; + } else if (histogramName.contains( + RTFMetrics.ThreadPoolValue.Constants.THREADS_ACTIVE_VALUE)) { + return threadPoolActiveThreadsHistogram; + } else if (histogramName.contains( + RTFMetrics.ThreadPoolValue.Constants.QUEUE_CAPACITY_VALUE)) { + return threadPoolQueueCapacityHistogram; + } + return threadPoolTotalThreadsHistogram; + }); + + rtfThreadPoolMetricsCollector = + new RTFThreadPoolMetricsCollector(mockController, mockWrapper); + } + + @Test + public void testCollectMetrics() throws IOException { + when(mockThreadPool.stats()).thenReturn(generateThreadPoolStat()); + rtfThreadPoolMetricsCollector.collectMetrics(System.currentTimeMillis()); + verify(mockThreadPool, atLeastOnce()).stats(); + verify(threadPoolQueueSizeHistogram, atLeastOnce()).record(anyDouble(), any()); + verify(threadPoolRejectedReqsHistogram, atLeastOnce()).record(anyDouble(), any()); + verify(threadPoolActiveThreadsHistogram, atLeastOnce()).record(anyDouble(), any()); + verify(threadPoolTotalThreadsHistogram, atLeastOnce()).record(anyDouble(), any()); + } + + private ThreadPoolStats generateThreadPoolStat() { + List stats = new ArrayList<>(); + stats.add(new ThreadPoolStats.Stats("write", 0, 0, 0, 2, 0, 0L, 20L)); + return new ThreadPoolStats(stats); + } +} diff --git a/src/test/java/org/opensearch/performanceanalyzer/config/setting/handler/PerformanceAnalyzerCollectorsSettingHandlerTests.java b/src/test/java/org/opensearch/performanceanalyzer/config/setting/handler/PerformanceAnalyzerCollectorsSettingHandlerTests.java new file mode 100644 index 00000000..8a86a235 --- /dev/null +++ b/src/test/java/org/opensearch/performanceanalyzer/config/setting/handler/PerformanceAnalyzerCollectorsSettingHandlerTests.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.config.setting.handler; + +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.MockitoAnnotations.initMocks; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; +import org.opensearch.performanceanalyzer.config.setting.ClusterSettingsManager; + +public class PerformanceAnalyzerCollectorsSettingHandlerTests { + private PerformanceAnalyzerCollectorsSettingHandler handler; + @Mock private PerformanceAnalyzerController controller; + @Mock private ClusterSettingsManager clusterSettingsManager; + + @Before + public void init() { + initMocks(this); + handler = + new PerformanceAnalyzerCollectorsSettingHandler(controller, clusterSettingsManager); + } + + @Test + public void testOnSettingUpdate() { + Integer newSettingValue = null; + handler.onSettingUpdate(newSettingValue); + verify(controller, never()).updateCollectorsSetting(anyInt()); + + newSettingValue = 1; + handler.onSettingUpdate(newSettingValue); + verify(controller).updateCollectorsSetting(newSettingValue); + } +} diff --git a/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerClusterConfigActionTests.java b/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerClusterConfigActionTests.java index 7c23332d..f0e082f5 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerClusterConfigActionTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerClusterConfigActionTests.java @@ -35,6 +35,7 @@ import org.opensearch.performanceanalyzer.config.setting.ClusterSettingsManager; import org.opensearch.performanceanalyzer.config.setting.handler.NodeStatsSettingHandler; import org.opensearch.performanceanalyzer.config.setting.handler.PerformanceAnalyzerClusterSettingHandler; +import org.opensearch.performanceanalyzer.config.setting.handler.PerformanceAnalyzerCollectorsSettingHandler; import org.opensearch.rest.RestController; import org.opensearch.rest.RestRequest; import org.opensearch.test.rest.FakeRestChannel; @@ -52,6 +53,7 @@ public class PerformanceAnalyzerClusterConfigActionTests { private ClusterSettings clusterSettings; private PerformanceAnalyzerClusterSettingHandler clusterSettingHandler; private NodeStatsSettingHandler nodeStatsSettingHandler; + private PerformanceAnalyzerCollectorsSettingHandler performanceAnalyzerCollectorsSettingHandler; private IdentityService identityService; @Mock private PerformanceAnalyzerController controller; @@ -81,12 +83,16 @@ public void init() { clusterSettingHandler = new PerformanceAnalyzerClusterSettingHandler(controller, clusterSettingsManager); nodeStatsSettingHandler = new NodeStatsSettingHandler(controller, clusterSettingsManager); + performanceAnalyzerCollectorsSettingHandler = + new PerformanceAnalyzerCollectorsSettingHandler(controller, clusterSettingsManager); + configAction = new PerformanceAnalyzerClusterConfigAction( Settings.EMPTY, restController, clusterSettingHandler, - nodeStatsSettingHandler); + nodeStatsSettingHandler, + performanceAnalyzerCollectorsSettingHandler); restController.registerHandler(configAction); } @@ -167,6 +173,7 @@ private void testWithRequestPath(String requestPath) throws IOException { assertTrue(responseStr.contains(PerformanceAnalyzerClusterConfigAction.CURRENT)); assertTrue( responseStr.contains(PerformanceAnalyzerClusterConfigAction.SHARDS_PER_COLLECTION)); + assertTrue(responseStr.contains(PerformanceAnalyzerClusterConfigAction.COLLECTORS_SETTING)); assertTrue( responseStr.contains( PerformanceAnalyzerClusterConfigAction @@ -179,6 +186,7 @@ private FakeRestRequest buildRequest(String requestPath) throws IOException { .startObject() .field(PerformanceAnalyzerClusterConfigAction.ENABLED, true) .field(PerformanceAnalyzerClusterConfigAction.SHARDS_PER_COLLECTION, 1) + .field(PerformanceAnalyzerClusterConfigAction.COLLECTORS_SETTING, 2) .endObject(); return new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY) From a661055aadb15bf590789ff7bacde8d85b7526f6 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 11 Jun 2024 18:58:30 -0400 Subject: [PATCH 05/21] Bump PA to use 1.4.0 PA commons lib (#664) * Bump PA to use 1.4.0 PA commons lib Signed-off-by: Peter Zhu * new commit Signed-off-by: Peter Zhu --------- Signed-off-by: Peter Zhu --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 67f5d1fc..0b8aac4e 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,7 @@ buildscript { // The PA Commons (https://github.com/opensearch-project/performance-analyzer-commons) // is a library dependency with hardcoded versioning in PA and RCA repos. - paCommonsVersion = "1.3.0" + paCommonsVersion = "1.4.0" // 3.0.0-SNAPSHOT -> 3.0.0.0-SNAPSHOT version_tokens = opensearch_version.tokenize('-') From 9cb092fdea3e9c16873d3f6230ed3d4c4c779f28 Mon Sep 17 00:00:00 2001 From: DevJhaAbhishek <137988638+DevJhaAbhishek@users.noreply.github.com> Date: Mon, 17 Jun 2024 20:02:14 +0530 Subject: [PATCH 06/21] Create opensearch-performance-analyzer.release-notes-2.15.0.0.md (#1) (#671) --- ...ch-performance-analyzer.release-notes-2.15.0.0.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 release-notes/release-notes/opensearch-performance-analyzer.release-notes-2.15.0.0.md diff --git a/release-notes/release-notes/opensearch-performance-analyzer.release-notes-2.15.0.0.md b/release-notes/release-notes/opensearch-performance-analyzer.release-notes-2.15.0.0.md new file mode 100644 index 00000000..f72091b7 --- /dev/null +++ b/release-notes/release-notes/opensearch-performance-analyzer.release-notes-2.15.0.0.md @@ -0,0 +1,12 @@ +## Version 2.15.0 Release Notes + +Compatible with OpenSearch 2.15.0 + +### Infrastructure +* Bump PA to use 1.4.0 PA commons lib [#664](https://github.com/opensearch-project/performance-analyzer/pull/664) + +### Feature +* Framework changes to merge PA with RTF [#662](https://github.com/opensearch-project/performance-analyzer/pull/662) + +### Bug Fix +* Fixed the bug in CacheConfigMetricsCollector [#657](https://github.com/opensearch-project/performance-analyzer/pull/657) From bf2b5ea8e00012f6c327f23c08b6836b0437b3d3 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 17 Jun 2024 15:08:12 -0400 Subject: [PATCH 07/21] Move 2.15.0 release notes to correct location (#674) Signed-off-by: Peter Zhu --- .../opensearch-performance-analyzer.release-notes-2.15.0.0.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename release-notes/{release-notes => }/opensearch-performance-analyzer.release-notes-2.15.0.0.md (100%) diff --git a/release-notes/release-notes/opensearch-performance-analyzer.release-notes-2.15.0.0.md b/release-notes/opensearch-performance-analyzer.release-notes-2.15.0.0.md similarity index 100% rename from release-notes/release-notes/opensearch-performance-analyzer.release-notes-2.15.0.0.md rename to release-notes/opensearch-performance-analyzer.release-notes-2.15.0.0.md From c95feb6377205e7f4ec64c3402f7c07a3f3edd02 Mon Sep 17 00:00:00 2001 From: Gagan Juneja Date: Tue, 23 Jul 2024 21:09:43 +0530 Subject: [PATCH 08/21] Adds the listener for resource utilization metrics (#687) * Adds the listener for resource utilization metrics Signed-off-by: Gagan Juneja Co-authored-by: Gagan Juneja --- .../PerformanceAnalyzerPlugin.java | 10 +- .../config/PerformanceAnalyzerController.java | 9 + .../PerformanceAnalyzerSearchListener.java | 14 +- .../RTFPerformanceAnalyzerSearchListener.java | 304 ++++++++++++++++++ ...rmanceAnalyzerTransportRequestHandler.java | 10 +- ...TFPerformanceAnalyzerTransportChannel.java | 128 ++++++++ ...rformanceAnalyzerTransportInterceptor.java | 34 ++ ...rmanceAnalyzerTransportRequestHandler.java | 124 +++++++ .../performanceanalyzer/util/Utils.java | 37 +++ .../PerformanceAnalyzerPluginTests.java | 10 +- ...erformanceAnalyzerSearchListenerTests.java | 16 + ...erformanceAnalyzerSearchListenerTests.java | 175 ++++++++++ ...eAnalyzerTransportRequestHandlerTests.java | 28 ++ ...formanceAnalyzerTransportChannelTests.java | 82 +++++ ...eAnalyzerTransportRequestHandlerTests.java | 120 +++++++ .../performanceanalyzer/util/UtilsTests.java | 32 ++ 16 files changed, 1120 insertions(+), 13 deletions(-) create mode 100644 src/main/java/org/opensearch/performanceanalyzer/listener/RTFPerformanceAnalyzerSearchListener.java create mode 100644 src/main/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportChannel.java create mode 100644 src/main/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportInterceptor.java create mode 100644 src/main/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportRequestHandler.java create mode 100644 src/test/java/org/opensearch/performanceanalyzer/listener/RTFPerformanceAnalyzerSearchListenerTests.java create mode 100644 src/test/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportChannelTests.java create mode 100644 src/test/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportRequestHandlerTests.java create mode 100644 src/test/java/org/opensearch/performanceanalyzer/util/UtilsTests.java diff --git a/src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java b/src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java index 143d1f1a..06f19fa2 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java +++ b/src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java @@ -86,7 +86,9 @@ import org.opensearch.performanceanalyzer.http_action.whoami.TransportWhoAmIAction; import org.opensearch.performanceanalyzer.http_action.whoami.WhoAmIAction; import org.opensearch.performanceanalyzer.listener.PerformanceAnalyzerSearchListener; +import org.opensearch.performanceanalyzer.listener.RTFPerformanceAnalyzerSearchListener; import org.opensearch.performanceanalyzer.transport.PerformanceAnalyzerTransportInterceptor; +import org.opensearch.performanceanalyzer.transport.RTFPerformanceAnalyzerTransportInterceptor; import org.opensearch.performanceanalyzer.util.Utils; import org.opensearch.performanceanalyzer.writer.EventLogQueueProcessor; import org.opensearch.plugins.ActionPlugin; @@ -302,7 +304,10 @@ public List getActionFilters() { public void onIndexModule(IndexModule indexModule) { PerformanceAnalyzerSearchListener performanceanalyzerSearchListener = new PerformanceAnalyzerSearchListener(performanceAnalyzerController); + RTFPerformanceAnalyzerSearchListener rtfPerformanceAnalyzerSearchListener = + new RTFPerformanceAnalyzerSearchListener(performanceAnalyzerController); indexModule.addSearchOperationListener(performanceanalyzerSearchListener); + indexModule.addSearchOperationListener(rtfPerformanceAnalyzerSearchListener); } // follower check, leader check @@ -330,8 +335,9 @@ public void onDiscovery(Discovery discovery) { @Override public List getTransportInterceptors( NamedWriteableRegistry namedWriteableRegistry, ThreadContext threadContext) { - return singletonList( - new PerformanceAnalyzerTransportInterceptor(performanceAnalyzerController)); + return Arrays.asList( + new PerformanceAnalyzerTransportInterceptor(performanceAnalyzerController), + new RTFPerformanceAnalyzerTransportInterceptor(performanceAnalyzerController)); } @Override diff --git a/src/main/java/org/opensearch/performanceanalyzer/config/PerformanceAnalyzerController.java b/src/main/java/org/opensearch/performanceanalyzer/config/PerformanceAnalyzerController.java index faaebc7c..6915e0fe 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/config/PerformanceAnalyzerController.java +++ b/src/main/java/org/opensearch/performanceanalyzer/config/PerformanceAnalyzerController.java @@ -383,4 +383,13 @@ public boolean isCollectorDisabled( return disabledCollectorsList.contains(collectorName); } + + /** + * Collectors Setting value. + * + * @return collectorsSettingValue + */ + public int getCollectorsRunModeValue() { + return collectorsSettingValue; + } } diff --git a/src/main/java/org/opensearch/performanceanalyzer/listener/PerformanceAnalyzerSearchListener.java b/src/main/java/org/opensearch/performanceanalyzer/listener/PerformanceAnalyzerSearchListener.java index 7719cdd6..99d46e3d 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/listener/PerformanceAnalyzerSearchListener.java +++ b/src/main/java/org/opensearch/performanceanalyzer/listener/PerformanceAnalyzerSearchListener.java @@ -7,6 +7,7 @@ import static org.opensearch.performanceanalyzer.commons.stats.metrics.StatExceptionCode.OPENSEARCH_REQUEST_INTERCEPTOR_ERROR; +import com.google.common.annotations.VisibleForTesting; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.index.shard.SearchOperationListener; @@ -16,6 +17,7 @@ import org.opensearch.performanceanalyzer.commons.metrics.MetricsProcessor; import org.opensearch.performanceanalyzer.commons.metrics.PerformanceAnalyzerMetrics; import org.opensearch.performanceanalyzer.commons.util.ThreadIDUtil; +import org.opensearch.performanceanalyzer.commons.util.Util; import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; import org.opensearch.search.internal.SearchContext; @@ -36,8 +38,16 @@ public String toString() { return PerformanceAnalyzerSearchListener.class.getSimpleName(); } - private SearchListener getSearchListener() { - return controller.isPerformanceAnalyzerEnabled() ? this : NO_OP_SEARCH_LISTENER; + @VisibleForTesting + SearchListener getSearchListener() { + return isSearchListenerEnabled() ? this : NO_OP_SEARCH_LISTENER; + } + + private boolean isSearchListenerEnabled() { + return controller.isPerformanceAnalyzerEnabled() + && (controller.getCollectorsRunModeValue() == Util.CollectorMode.DUAL.getValue() + || controller.getCollectorsRunModeValue() + == Util.CollectorMode.RCA.getValue()); } @Override diff --git a/src/main/java/org/opensearch/performanceanalyzer/listener/RTFPerformanceAnalyzerSearchListener.java b/src/main/java/org/opensearch/performanceanalyzer/listener/RTFPerformanceAnalyzerSearchListener.java new file mode 100644 index 00000000..6b7921cc --- /dev/null +++ b/src/main/java/org/opensearch/performanceanalyzer/listener/RTFPerformanceAnalyzerSearchListener.java @@ -0,0 +1,304 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.listener; + +import static org.opensearch.performanceanalyzer.commons.stats.metrics.StatExceptionCode.OPENSEARCH_REQUEST_INTERCEPTOR_ERROR; + +import com.google.common.annotations.VisibleForTesting; +import java.util.HashMap; +import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.core.action.NotifyOnceListener; +import org.opensearch.index.shard.SearchOperationListener; +import org.opensearch.performanceanalyzer.OpenSearchResources; +import org.opensearch.performanceanalyzer.commons.collectors.StatsCollector; +import org.opensearch.performanceanalyzer.commons.metrics.RTFMetrics; +import org.opensearch.performanceanalyzer.commons.util.Util; +import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; +import org.opensearch.performanceanalyzer.util.Utils; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.tasks.Task; +import org.opensearch.telemetry.metrics.Histogram; +import org.opensearch.telemetry.metrics.MetricsRegistry; +import org.opensearch.telemetry.metrics.tags.Tags; + +/** + * {@link SearchOperationListener} to capture the resource utilization of a shard search operation. + * This will be getting the resource tracking information from the {@link + * org.opensearch.tasks.TaskResourceTrackingService}. + */ +public class RTFPerformanceAnalyzerSearchListener + implements SearchOperationListener, SearchListener { + + private static final Logger LOG = + LogManager.getLogger(RTFPerformanceAnalyzerSearchListener.class); + private static final String SHARD_FETCH_PHASE = "shard_fetch"; + private static final String SHARD_QUERY_PHASE = "shard_query"; + public static final String QUERY_START_TIME = "query_start_time"; + public static final String FETCH_START_TIME = "fetch_start_time"; + public static final String QUERY_TASK_ID = "query_task_id"; + private final ThreadLocal> threadLocal; + private static final SearchListener NO_OP_SEARCH_LISTENER = new NoOpSearchListener(); + + private final PerformanceAnalyzerController controller; + private final Histogram cpuUtilizationHistogram; + private final Histogram heapUsedHistogram; + private final int numProcessors; + + public RTFPerformanceAnalyzerSearchListener(final PerformanceAnalyzerController controller) { + this.controller = controller; + this.cpuUtilizationHistogram = + createCPUUtilizationHistogram(OpenSearchResources.INSTANCE.getMetricsRegistry()); + this.heapUsedHistogram = + createHeapUsedHistogram(OpenSearchResources.INSTANCE.getMetricsRegistry()); + this.threadLocal = ThreadLocal.withInitial(() -> new HashMap()); + this.numProcessors = Runtime.getRuntime().availableProcessors(); + } + + private Histogram createCPUUtilizationHistogram(MetricsRegistry metricsRegistry) { + if (metricsRegistry != null) { + return metricsRegistry.createHistogram( + RTFMetrics.OSMetrics.CPU_UTILIZATION.toString(), + "CPU Utilization per shard for a search phase", + RTFMetrics.MetricUnits.RATE.toString()); + } else { + LOG.debug("MetricsRegistry is null"); + return null; + } + } + + private Histogram createHeapUsedHistogram(MetricsRegistry metricsRegistry) { + if (metricsRegistry != null) { + return metricsRegistry.createHistogram( + RTFMetrics.OSMetrics.HEAP_ALLOCATED.toString(), + "Heap used per shard for a search phase", + RTFMetrics.MetricUnits.BYTE.toString()); + } else { + LOG.debug("MetricsRegistry is null"); + return null; + } + } + + @Override + public String toString() { + return RTFPerformanceAnalyzerSearchListener.class.getSimpleName(); + } + + @VisibleForTesting + SearchListener getSearchListener() { + return isSearchListenerEnabled() ? this : NO_OP_SEARCH_LISTENER; + } + + private boolean isSearchListenerEnabled() { + return OpenSearchResources.INSTANCE.getMetricsRegistry() != null + && controller.isPerformanceAnalyzerEnabled() + && (controller.getCollectorsRunModeValue() == Util.CollectorMode.DUAL.getValue() + || controller.getCollectorsRunModeValue() + == Util.CollectorMode.TELEMETRY.getValue()); + } + + @Override + public void onPreQueryPhase(SearchContext searchContext) { + try { + getSearchListener().preQueryPhase(searchContext); + } catch (Exception ex) { + LOG.error(ex); + StatsCollector.instance().logException(OPENSEARCH_REQUEST_INTERCEPTOR_ERROR); + } + } + + @Override + public void onQueryPhase(SearchContext searchContext, long tookInNanos) { + try { + getSearchListener().queryPhase(searchContext, tookInNanos); + } catch (Exception ex) { + LOG.error(ex); + StatsCollector.instance().logException(OPENSEARCH_REQUEST_INTERCEPTOR_ERROR); + } + } + + @Override + public void onFailedQueryPhase(SearchContext searchContext) { + try { + getSearchListener().failedQueryPhase(searchContext); + } catch (Exception ex) { + LOG.error(ex); + StatsCollector.instance().logException(OPENSEARCH_REQUEST_INTERCEPTOR_ERROR); + } + } + + @Override + public void onPreFetchPhase(SearchContext searchContext) { + try { + getSearchListener().preFetchPhase(searchContext); + } catch (Exception ex) { + LOG.error(ex); + StatsCollector.instance().logException(OPENSEARCH_REQUEST_INTERCEPTOR_ERROR); + } + } + + @Override + public void onFetchPhase(SearchContext searchContext, long tookInNanos) { + try { + getSearchListener().fetchPhase(searchContext, tookInNanos); + } catch (Exception ex) { + LOG.error(ex); + StatsCollector.instance().logException(OPENSEARCH_REQUEST_INTERCEPTOR_ERROR); + } + } + + @Override + public void onFailedFetchPhase(SearchContext searchContext) { + try { + getSearchListener().failedFetchPhase(searchContext); + } catch (Exception ex) { + LOG.error(ex); + StatsCollector.instance().logException(OPENSEARCH_REQUEST_INTERCEPTOR_ERROR); + } + } + + @Override + public void preQueryPhase(SearchContext searchContext) { + threadLocal.get().put(QUERY_START_TIME, System.nanoTime()); + threadLocal.get().put(QUERY_TASK_ID, searchContext.getTask().getId()); + } + + @Override + public void queryPhase(SearchContext searchContext, long tookInNanos) { + long queryStartTime = threadLocal.get().getOrDefault(QUERY_START_TIME, System.nanoTime()); + long queryTime = (System.nanoTime() - queryStartTime); + addResourceTrackingCompletionListener( + searchContext, queryStartTime, queryTime, SHARD_QUERY_PHASE, false); + } + + @Override + public void failedQueryPhase(SearchContext searchContext) { + long queryStartTime = threadLocal.get().getOrDefault(QUERY_START_TIME, System.nanoTime()); + long queryTime = (System.nanoTime() - queryStartTime); + addResourceTrackingCompletionListener( + searchContext, queryStartTime, queryTime, SHARD_QUERY_PHASE, true); + } + + @Override + public void preFetchPhase(SearchContext searchContext) { + threadLocal.get().put(FETCH_START_TIME, System.nanoTime()); + } + + @Override + public void fetchPhase(SearchContext searchContext, long tookInNanos) { + long fetchStartTime = threadLocal.get().getOrDefault(FETCH_START_TIME, System.nanoTime()); + long fetchTime = (System.nanoTime() - fetchStartTime); + addResourceTrackingCompletionListenerForFetchPhase( + searchContext, fetchStartTime, fetchTime, SHARD_FETCH_PHASE, false); + } + + @Override + public void failedFetchPhase(SearchContext searchContext) { + long fetchStartTime = threadLocal.get().getOrDefault(FETCH_START_TIME, System.nanoTime()); + long fetchTime = (System.nanoTime() - fetchStartTime); + addResourceTrackingCompletionListenerForFetchPhase( + searchContext, fetchStartTime, fetchTime, SHARD_FETCH_PHASE, true); + } + + private void addResourceTrackingCompletionListener( + SearchContext searchContext, + long startTime, + long queryTime, + String phase, + boolean isFailed) { + addCompletionListener(searchContext, startTime, queryTime, phase, isFailed); + } + + private void addResourceTrackingCompletionListenerForFetchPhase( + SearchContext searchContext, + long fetchStartTime, + long fetchTime, + String phase, + boolean isFailed) { + long startTime = fetchStartTime; + long queryTaskId = threadLocal.get().getOrDefault(QUERY_TASK_ID, -1l); + /** + * There are scenarios where both query and fetch phases run in the same task for an + * optimization. Adding a special handling for that case to divide the CPU usage between + * these 2 operations by their runTime. + */ + if (queryTaskId == searchContext.getTask().getId()) { + startTime = threadLocal.get().getOrDefault(QUERY_START_TIME, System.nanoTime()); + } + addCompletionListener(searchContext, startTime, fetchTime, phase, isFailed); + } + + private void addCompletionListener( + SearchContext searchContext, + long startTime, + long phaseTookTime, + String phase, + boolean isFailed) { + searchContext + .getTask() + .addResourceTrackingCompletionListener( + createListener(searchContext, startTime, phaseTookTime, phase, isFailed)); + } + + @VisibleForTesting + NotifyOnceListener createListener( + SearchContext searchContext, + long startTime, + long phaseTookTime, + String phase, + boolean isFailed) { + return new NotifyOnceListener() { + @Override + protected void innerOnResponse(Task task) { + LOG.debug("Updating the counter for task {}", task.getId()); + /** + * There are scenarios where cpuUsageTime consists of the total of CPU of multiple + * phases. In that case we are computing the cpuShareFactor by dividing the + * particular phaseTime and the total time till this calculation happen from the + * overall start time. + */ + long totalTime = System.nanoTime() - startTime; + double shareFactor = computeShareFactor(phaseTookTime, totalTime); + cpuUtilizationHistogram.record( + Utils.calculateCPUUtilization( + numProcessors, + totalTime, + task.getTotalResourceStats().getCpuTimeInNanos(), + shareFactor), + createTags()); + heapUsedHistogram.record( + Math.max(0, task.getTotalResourceStats().getMemoryInBytes() * shareFactor), + createTags()); + } + + private Tags createTags() { + return Tags.create() + .addTag( + RTFMetrics.CommonDimension.INDEX_NAME.toString(), + searchContext.request().shardId().getIndex().getName()) + .addTag( + RTFMetrics.CommonDimension.INDEX_UUID.toString(), + searchContext.request().shardId().getIndex().getUUID()) + .addTag( + RTFMetrics.CommonDimension.SHARD_ID.toString(), + searchContext.request().shardId().getId()) + .addTag(RTFMetrics.CommonDimension.OPERATION.toString(), phase) + .addTag(RTFMetrics.CommonDimension.FAILED.toString(), isFailed); + } + + @Override + protected void innerOnFailure(Exception e) { + LOG.error("Error is executing the the listener", e); + } + }; + } + + @VisibleForTesting + static double computeShareFactor(long phaseTookTime, long totalTime) { + return Math.min(1, ((double) phaseTookTime) / Math.max(1.0, totalTime)); + } +} diff --git a/src/main/java/org/opensearch/performanceanalyzer/transport/PerformanceAnalyzerTransportRequestHandler.java b/src/main/java/org/opensearch/performanceanalyzer/transport/PerformanceAnalyzerTransportRequestHandler.java index a4cd946e..64052ab1 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/transport/PerformanceAnalyzerTransportRequestHandler.java +++ b/src/main/java/org/opensearch/performanceanalyzer/transport/PerformanceAnalyzerTransportRequestHandler.java @@ -13,6 +13,7 @@ import org.opensearch.action.bulk.BulkShardRequest; import org.opensearch.action.support.replication.TransportReplicationAction.ConcreteShardRequest; import org.opensearch.performanceanalyzer.commons.collectors.StatsCollector; +import org.opensearch.performanceanalyzer.commons.util.Util; import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportChannel; @@ -45,7 +46,7 @@ public void messageReceived(T request, TransportChannel channel, Task task) thro @VisibleForTesting TransportChannel getChannel(T request, TransportChannel channel, Task task) { - if (!controller.isPerformanceAnalyzerEnabled()) { + if (!isCollectorEnabled()) { return channel; } @@ -56,6 +57,13 @@ TransportChannel getChannel(T request, TransportChannel channel, Task task) { } } + private boolean isCollectorEnabled() { + return controller.isPerformanceAnalyzerEnabled() + && (controller.getCollectorsRunModeValue() == Util.CollectorMode.DUAL.getValue() + || controller.getCollectorsRunModeValue() + == Util.CollectorMode.RCA.getValue()); + } + private TransportChannel getShardBulkChannel(T request, TransportChannel channel, Task task) { String className = request.getClass().getName(); boolean bPrimary = false; diff --git a/src/main/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportChannel.java b/src/main/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportChannel.java new file mode 100644 index 00000000..6eb64185 --- /dev/null +++ b/src/main/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportChannel.java @@ -0,0 +1,128 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.transport; + +import com.google.common.annotations.VisibleForTesting; +import com.sun.management.ThreadMXBean; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.transport.TransportResponse; +import org.opensearch.performanceanalyzer.commons.metrics.RTFMetrics; +import org.opensearch.performanceanalyzer.util.Utils; +import org.opensearch.telemetry.metrics.Histogram; +import org.opensearch.telemetry.metrics.tags.Tags; +import org.opensearch.transport.TransportChannel; + +/** + * {@link TransportChannel} implementation to override the sendResponse behavior to have handle of + * the {@link org.opensearch.action.bulk.BulkShardRequest} completion. + */ +public final class RTFPerformanceAnalyzerTransportChannel implements TransportChannel { + private static final Logger LOG = + LogManager.getLogger(RTFPerformanceAnalyzerTransportChannel.class); + + private static final ThreadMXBean threadMXBean = + (ThreadMXBean) ManagementFactory.getThreadMXBean(); + private static final String OPERATION_SHARD_BULK = "shardbulk"; + private static final String SHARD_ROLE_PRIMARY = "primary"; + private static final String SHARD_ROLE_REPLICA = "replica"; + + private long cpuStartTime; + private long operationStartTime; + + private Histogram cpuUtilizationHistogram; + + private TransportChannel original; + private String indexName; + private ShardId shardId; + private boolean primary; + + private long threadID; + private int numProcessors; + + void set( + TransportChannel original, + Histogram cpuUtilizationHistogram, + String indexName, + ShardId shardId, + boolean bPrimary) { + this.original = original; + this.cpuUtilizationHistogram = cpuUtilizationHistogram; + this.indexName = indexName; + this.shardId = shardId; + this.primary = bPrimary; + + this.operationStartTime = System.nanoTime(); + threadID = Thread.currentThread().getId(); + this.cpuStartTime = threadMXBean.getThreadCpuTime(threadID); + this.numProcessors = Runtime.getRuntime().availableProcessors(); + LOG.debug("Thread Name {}", Thread.currentThread().getName()); + } + + @Override + public String getProfileName() { + return "RTFPerformanceAnalyzerTransportChannelProfile"; + } + + @Override + public String getChannelType() { + return "RTFPerformanceAnalyzerTransportChannelType"; + } + + @Override + public void sendResponse(TransportResponse response) throws IOException { + emitMetrics(false); + original.sendResponse(response); + } + + @Override + public void sendResponse(Exception exception) throws IOException { + emitMetrics(true); + original.sendResponse(exception); + } + + private void emitMetrics(boolean isFailed) { + double cpuUtilization = calculateCPUUtilization(operationStartTime, cpuStartTime); + recordCPUUtilizationMetric(shardId, cpuUtilization, OPERATION_SHARD_BULK, isFailed); + } + + private double calculateCPUUtilization(long phaseStartTime, long phaseCPUStartTime) { + LOG.debug("Completion Thread Name {}", Thread.currentThread().getName()); + long totalCpuTime = + Math.max(0, (threadMXBean.getThreadCpuTime(threadID) - phaseCPUStartTime)); + return Utils.calculateCPUUtilization( + numProcessors, (System.nanoTime() - phaseStartTime), totalCpuTime, 1.0); + } + + @VisibleForTesting + void recordCPUUtilizationMetric( + ShardId shardId, double cpuUtilization, String operation, boolean isFailed) { + cpuUtilizationHistogram.record( + cpuUtilization, + Tags.create() + .addTag( + RTFMetrics.CommonDimension.INDEX_NAME.toString(), + shardId.getIndex().getName()) + .addTag( + RTFMetrics.CommonDimension.INDEX_UUID.toString(), + shardId.getIndex().getUUID()) + .addTag(RTFMetrics.CommonDimension.SHARD_ID.toString(), shardId.getId()) + .addTag(RTFMetrics.CommonDimension.OPERATION.toString(), operation) + .addTag(RTFMetrics.CommonDimension.FAILED.toString(), isFailed) + .addTag( + RTFMetrics.CommonDimension.SHARD_ROLE.toString(), + primary ? SHARD_ROLE_PRIMARY : SHARD_ROLE_REPLICA)); + } + + // This function is called from the security plugin using reflection. Do not + // remove this function without changing the security plugin. + public TransportChannel getInnerChannel() { + return this.original; + } +} diff --git a/src/main/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportInterceptor.java b/src/main/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportInterceptor.java new file mode 100644 index 00000000..5d2de2b6 --- /dev/null +++ b/src/main/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportInterceptor.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.transport; + +import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; +import org.opensearch.transport.TransportInterceptor; +import org.opensearch.transport.TransportRequest; +import org.opensearch.transport.TransportRequestHandler; + +/** + * Transport Interceptor to intercept the Indexing requests and populate the resource utilization + * metrics. + */ +public final class RTFPerformanceAnalyzerTransportInterceptor implements TransportInterceptor { + + private final PerformanceAnalyzerController controller; + + public RTFPerformanceAnalyzerTransportInterceptor( + final PerformanceAnalyzerController controller) { + this.controller = controller; + } + + @Override + public TransportRequestHandler interceptHandler( + String action, + String executor, + boolean forceExecution, + TransportRequestHandler actualHandler) { + return new RTFPerformanceAnalyzerTransportRequestHandler<>(actualHandler, controller); + } +} diff --git a/src/main/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportRequestHandler.java b/src/main/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportRequestHandler.java new file mode 100644 index 00000000..82a0abe6 --- /dev/null +++ b/src/main/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportRequestHandler.java @@ -0,0 +1,124 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.transport; + +import static org.opensearch.performanceanalyzer.commons.stats.metrics.StatExceptionCode.OPENSEARCH_REQUEST_INTERCEPTOR_ERROR; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.bulk.BulkShardRequest; +import org.opensearch.action.support.replication.TransportReplicationAction.ConcreteShardRequest; +import org.opensearch.performanceanalyzer.OpenSearchResources; +import org.opensearch.performanceanalyzer.commons.collectors.StatsCollector; +import org.opensearch.performanceanalyzer.commons.metrics.RTFMetrics; +import org.opensearch.performanceanalyzer.commons.util.Util; +import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; +import org.opensearch.tasks.Task; +import org.opensearch.telemetry.metrics.Histogram; +import org.opensearch.telemetry.metrics.MetricsRegistry; +import org.opensearch.transport.TransportChannel; +import org.opensearch.transport.TransportRequest; +import org.opensearch.transport.TransportRequestHandler; + +/** + * {@link TransportRequestHandler} implementation to intercept only the {@link BulkShardRequest} and + * skip other transport calls. + * + * @param {@link TransportRequest} + */ +public final class RTFPerformanceAnalyzerTransportRequestHandler + implements TransportRequestHandler { + private static final Logger LOG = + LogManager.getLogger(RTFPerformanceAnalyzerTransportRequestHandler.class); + private final PerformanceAnalyzerController controller; + private TransportRequestHandler actualHandler; + private boolean logOnce = false; + private final Histogram cpuUtilizationHistogram; + + RTFPerformanceAnalyzerTransportRequestHandler( + TransportRequestHandler actualHandler, PerformanceAnalyzerController controller) { + this.actualHandler = actualHandler; + this.controller = controller; + this.cpuUtilizationHistogram = createCPUUtilizationHistogram(); + } + + private Histogram createCPUUtilizationHistogram() { + MetricsRegistry metricsRegistry = OpenSearchResources.INSTANCE.getMetricsRegistry(); + if (metricsRegistry != null) { + return metricsRegistry.createHistogram( + RTFMetrics.OSMetrics.CPU_UTILIZATION.toString(), + "CPU Utilization per shard for an operation", + RTFMetrics.MetricUnits.RATE.toString()); + } else { + return null; + } + } + + @Override + public void messageReceived(T request, TransportChannel channel, Task task) throws Exception { + actualHandler.messageReceived(request, getChannel(request, channel, task), task); + } + + @VisibleForTesting + TransportChannel getChannel(T request, TransportChannel channel, Task task) { + if (!isCollectorEnabled()) { + return channel; + } + + if (request instanceof ConcreteShardRequest) { + return getShardBulkChannel(request, channel, task); + } else { + return channel; + } + } + + private boolean isCollectorEnabled() { + return OpenSearchResources.INSTANCE.getMetricsRegistry() != null + && controller.isPerformanceAnalyzerEnabled() + && (controller.getCollectorsRunModeValue() == Util.CollectorMode.DUAL.getValue() + || controller.getCollectorsRunModeValue() + == Util.CollectorMode.TELEMETRY.getValue()); + } + + private TransportChannel getShardBulkChannel(T request, TransportChannel channel, Task task) { + String className = request.getClass().getName(); + boolean bPrimary = false; + + if (className.equals( + "org.opensearch.action.support.replication.TransportReplicationAction$ConcreteShardRequest")) { + bPrimary = true; + } else if (className.equals( + "org.opensearch.action.support.replication.TransportReplicationAction$ConcreteReplicaRequest")) { + bPrimary = false; + } else { + return channel; + } + + TransportRequest transportRequest = ((ConcreteShardRequest) request).getRequest(); + + if (!(transportRequest instanceof BulkShardRequest)) { + return channel; + } + + BulkShardRequest bsr = (BulkShardRequest) transportRequest; + RTFPerformanceAnalyzerTransportChannel rtfPerformanceAnalyzerTransportChannel = + new RTFPerformanceAnalyzerTransportChannel(); + + try { + rtfPerformanceAnalyzerTransportChannel.set( + channel, cpuUtilizationHistogram, bsr.index(), bsr.shardId(), bPrimary); + } catch (Exception ex) { + if (!logOnce) { + LOG.error(ex); + logOnce = true; + } + StatsCollector.instance().logException(OPENSEARCH_REQUEST_INTERCEPTOR_ERROR); + } + + return rtfPerformanceAnalyzerTransportChannel; + } +} diff --git a/src/main/java/org/opensearch/performanceanalyzer/util/Utils.java b/src/main/java/org/opensearch/performanceanalyzer/util/Utils.java index e4963ebc..ebf440eb 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/util/Utils.java +++ b/src/main/java/org/opensearch/performanceanalyzer/util/Utils.java @@ -8,6 +8,8 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.Iterator; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.action.admin.indices.stats.CommonStatsFlags; import org.opensearch.action.admin.indices.stats.IndexShardStats; @@ -27,6 +29,7 @@ import org.opensearch.performanceanalyzer.commons.stats.ServiceMetrics; public class Utils { + private static final Logger LOG = LogManager.getLogger(Utils.class); public static void configureMetrics() { ServiceMetrics.initStatsReporter(); @@ -108,4 +111,38 @@ public static HashMap getShards() { IndexShardState.RECOVERING, IndexShardState.POST_RECOVERY, IndexShardState.STARTED); + + /** + * CPU Utilization is the time spend in CPU cycles divide by the total time cpu available time. + * Total cpu available time would be the multiplication of num of processors and the process + * time. It also takes into account the cpuShareFactor in case some adjustments are needed. + * + * @param numProcessors + * @param totalOperationTime + * @param cpuUsageTime + * @param cpuShareFactor + * @return + */ + public static double calculateCPUUtilization( + int numProcessors, long totalOperationTime, long cpuUsageTime, double cpuShareFactor) { + LOG.debug( + "Performance Analyzer CPUUtilization calculation with numProcessors: {}", + numProcessors); + LOG.debug( + "Performance Analyzer CPUUtilization calculation with cpuShareFactor {}", + cpuShareFactor); + LOG.debug( + "Performance Analyzer CPUUtilization calculation with totalCpuTime {}", + cpuUsageTime); + LOG.debug( + "Performance Analyzer CPUUtilization calculation with totalOperationTime {}", + totalOperationTime); + if (totalOperationTime == 0l || cpuUsageTime == 0l || numProcessors == 0) { + return 0.0d; + } + double totalAvailableCPUTime = Double.valueOf(totalOperationTime * numProcessors); + double cpuUtil = cpuShareFactor * (cpuUsageTime / totalAvailableCPUTime); + LOG.debug("Performance Analyzer CPUUtilization calculation with cpuUtil {}", cpuUtil); + return cpuUtil; + } } diff --git a/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPluginTests.java b/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPluginTests.java index fc5f95fa..554eca27 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPluginTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPluginTests.java @@ -22,7 +22,6 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.support.ActionFilter; import org.opensearch.client.node.NodeClient; -import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; @@ -90,12 +89,7 @@ public void setup() { } catch (Exception e) { e.printStackTrace(); } - clusterService = - new ClusterService( - settings, - clusterSettings, - threadPool, - new ClusterManagerMetrics(metricsRegistry)); + clusterService = new ClusterService(settings, clusterSettings, threadPool); identityService = new IdentityService(Settings.EMPTY, List.of()); restController = new RestController( @@ -131,7 +125,7 @@ public void testGetActions() { @Test public void testGetTransportInterceptors() { List list = plugin.getTransportInterceptors(null, null); - assertEquals(1, list.size()); + assertEquals(2, list.size()); assertEquals(PerformanceAnalyzerTransportInterceptor.class, list.get(0).getClass()); } diff --git a/src/test/java/org/opensearch/performanceanalyzer/listener/PerformanceAnalyzerSearchListenerTests.java b/src/test/java/org/opensearch/performanceanalyzer/listener/PerformanceAnalyzerSearchListenerTests.java index cfa8bf58..f7204783 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/listener/PerformanceAnalyzerSearchListenerTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/listener/PerformanceAnalyzerSearchListenerTests.java @@ -25,6 +25,7 @@ import org.opensearch.performanceanalyzer.commons.metrics.MetricsConfiguration; import org.opensearch.performanceanalyzer.commons.metrics.PerformanceAnalyzerMetrics; import org.opensearch.performanceanalyzer.commons.stats.metrics.StatExceptionCode; +import org.opensearch.performanceanalyzer.commons.util.Util; import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; import org.opensearch.performanceanalyzer.util.TestUtil; import org.opensearch.performanceanalyzer.util.Utils; @@ -69,6 +70,21 @@ public void init() { TestUtil.readEvents(); } + @Test + public void tesSearchListener() { + Mockito.when(controller.getCollectorsRunModeValue()) + .thenReturn(Util.CollectorMode.TELEMETRY.getValue()); + assertTrue(searchListener.getSearchListener() instanceof NoOpSearchListener); + + Mockito.when(controller.getCollectorsRunModeValue()) + .thenReturn(Util.CollectorMode.RCA.getValue()); + assertTrue(searchListener.getSearchListener() instanceof PerformanceAnalyzerSearchListener); + + Mockito.when(controller.getCollectorsRunModeValue()) + .thenReturn(Util.CollectorMode.DUAL.getValue()); + assertTrue(searchListener.getSearchListener() instanceof PerformanceAnalyzerSearchListener); + } + @Test public void testGetMetricsPath() { String expectedPath = diff --git a/src/test/java/org/opensearch/performanceanalyzer/listener/RTFPerformanceAnalyzerSearchListenerTests.java b/src/test/java/org/opensearch/performanceanalyzer/listener/RTFPerformanceAnalyzerSearchListenerTests.java new file mode 100644 index 00000000..16aba4bc --- /dev/null +++ b/src/test/java/org/opensearch/performanceanalyzer/listener/RTFPerformanceAnalyzerSearchListenerTests.java @@ -0,0 +1,175 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.listener; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.MockitoAnnotations.initMocks; + +import org.apache.commons.lang3.SystemUtils; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.opensearch.action.search.SearchShardTask; +import org.opensearch.core.action.NotifyOnceListener; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.tasks.resourcetracker.TaskResourceUsage; +import org.opensearch.performanceanalyzer.OpenSearchResources; +import org.opensearch.performanceanalyzer.commons.util.Util; +import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.internal.ShardSearchRequest; +import org.opensearch.tasks.Task; +import org.opensearch.telemetry.metrics.Histogram; +import org.opensearch.telemetry.metrics.MetricsRegistry; +import org.opensearch.telemetry.metrics.tags.Tags; + +public class RTFPerformanceAnalyzerSearchListenerTests { + + private RTFPerformanceAnalyzerSearchListener searchListener; + + @Mock private SearchContext searchContext; + @Mock private ShardSearchRequest shardSearchRequest; + @Mock private ShardId shardId; + @Mock private PerformanceAnalyzerController controller; + @Mock private SearchShardTask task; + @Mock private MetricsRegistry metricsRegistry; + @Mock private Histogram cpuUtilizationHistogram; + @Mock private Histogram heapUsedHistogram; + @Mock private Index index; + + @Mock private TaskResourceUsage taskResourceUsage; + + @BeforeClass + public static void setup() { + // this test only runs in Linux system + // as some of the static members of the ThreadList class are specific to Linux + org.junit.Assume.assumeTrue(SystemUtils.IS_OS_LINUX); + } + + @Before + public void init() { + initMocks(this); + OpenSearchResources.INSTANCE.setMetricsRegistry(metricsRegistry); + Mockito.when(controller.isPerformanceAnalyzerEnabled()).thenReturn(true); + Mockito.when( + metricsRegistry.createHistogram( + Mockito.eq("cpu_utilization"), + Mockito.anyString(), + Mockito.eq("rate"))) + .thenReturn(cpuUtilizationHistogram); + Mockito.when( + metricsRegistry.createHistogram( + Mockito.eq("heap_allocated"), Mockito.anyString(), Mockito.eq("B"))) + .thenReturn(heapUsedHistogram); + searchListener = new RTFPerformanceAnalyzerSearchListener(controller); + assertEquals( + RTFPerformanceAnalyzerSearchListener.class.getSimpleName(), + searchListener.toString()); + } + + @Test + public void tesSearchListener() { + Mockito.when(controller.getCollectorsRunModeValue()) + .thenReturn(Util.CollectorMode.RCA.getValue()); + assertTrue(searchListener.getSearchListener() instanceof NoOpSearchListener); + + Mockito.when(controller.getCollectorsRunModeValue()) + .thenReturn(Util.CollectorMode.TELEMETRY.getValue()); + assertTrue( + searchListener.getSearchListener() instanceof RTFPerformanceAnalyzerSearchListener); + + Mockito.when(controller.getCollectorsRunModeValue()) + .thenReturn(Util.CollectorMode.DUAL.getValue()); + assertTrue( + searchListener.getSearchListener() instanceof RTFPerformanceAnalyzerSearchListener); + } + + @Test + public void testQueryPhase() { + initializeValidSearchContext(true); + Mockito.when(controller.getCollectorsRunModeValue()) + .thenReturn(Util.CollectorMode.TELEMETRY.getValue()); + searchListener.preQueryPhase(searchContext); + searchListener.queryPhase(searchContext, 0l); + Mockito.verify(task).addResourceTrackingCompletionListener(Mockito.any()); + } + + @Test + public void testQueryPhaseFailed() { + initializeValidSearchContext(true); + Mockito.when(controller.getCollectorsRunModeValue()) + .thenReturn(Util.CollectorMode.TELEMETRY.getValue()); + searchListener.preQueryPhase(searchContext); + searchListener.failedQueryPhase(searchContext); + Mockito.verify(task).addResourceTrackingCompletionListener(Mockito.any()); + } + + @Test + public void testFetchPhase() { + initializeValidSearchContext(true); + Mockito.when(controller.getCollectorsRunModeValue()) + .thenReturn(Util.CollectorMode.TELEMETRY.getValue()); + searchListener.preFetchPhase(searchContext); + searchListener.fetchPhase(searchContext, 0l); + Mockito.verify(task).addResourceTrackingCompletionListener(Mockito.any()); + } + + @Test + public void testFetchPhaseFailed() { + initializeValidSearchContext(true); + Mockito.when(controller.getCollectorsRunModeValue()) + .thenReturn(Util.CollectorMode.TELEMETRY.getValue()); + searchListener.preFetchPhase(searchContext); + searchListener.failedFetchPhase(searchContext); + Mockito.verify(task).addResourceTrackingCompletionListener(Mockito.any()); + } + + @Test + public void testOperationShareFactor() { + assertEquals( + Double.valueOf(10.0 / 15), + RTFPerformanceAnalyzerSearchListener.computeShareFactor(10, 15), + 0); + assertEquals( + Double.valueOf(1), + RTFPerformanceAnalyzerSearchListener.computeShareFactor(15, 10), + 0); + } + + @Test + public void testTaskCompletionListener() { + initializeValidSearchContext(true); + RTFPerformanceAnalyzerSearchListener rtfSearchListener = + new RTFPerformanceAnalyzerSearchListener(controller); + + Mockito.when(shardId.getIndex()).thenReturn(index); + Mockito.when(index.getName()).thenReturn("myTestIndex"); + Mockito.when(index.getUUID()).thenReturn("abc-def"); + Mockito.when(task.getTotalResourceStats()).thenReturn(taskResourceUsage); + Mockito.when(taskResourceUsage.getCpuTimeInNanos()).thenReturn(10l); + + NotifyOnceListener taskCompletionListener = + rtfSearchListener.createListener(searchContext, 0l, 0l, "test", false); + taskCompletionListener.onResponse(task); + Mockito.verify(cpuUtilizationHistogram) + .record(Mockito.anyDouble(), Mockito.any(Tags.class)); + Mockito.verify(heapUsedHistogram).record(Mockito.anyDouble(), Mockito.any(Tags.class)); + } + + private void initializeValidSearchContext(boolean isValid) { + if (isValid) { + Mockito.when(searchContext.request()).thenReturn(shardSearchRequest); + Mockito.when(searchContext.getTask()).thenReturn(task); + Mockito.when(shardSearchRequest.shardId()).thenReturn(shardId); + } else { + Mockito.when(searchContext.request()).thenReturn(null); + } + } +} diff --git a/src/test/java/org/opensearch/performanceanalyzer/transport/PerformanceAnalyzerTransportRequestHandlerTests.java b/src/test/java/org/opensearch/performanceanalyzer/transport/PerformanceAnalyzerTransportRequestHandlerTests.java index 56dc289a..099390fa 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/transport/PerformanceAnalyzerTransportRequestHandlerTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/transport/PerformanceAnalyzerTransportRequestHandlerTests.java @@ -5,6 +5,7 @@ package org.opensearch.performanceanalyzer.transport; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.verify; import static org.mockito.MockitoAnnotations.initMocks; @@ -18,6 +19,7 @@ import org.opensearch.action.bulk.BulkShardRequest; import org.opensearch.action.support.replication.TransportReplicationAction.ConcreteShardRequest; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.performanceanalyzer.commons.util.Util; import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; import org.opensearch.performanceanalyzer.util.Utils; import org.opensearch.tasks.Task; @@ -67,4 +69,30 @@ public void testGetChannel() { TransportChannel actualChannel = handler.getChannel(concreteShardRequest, channel, task); assertTrue(actualChannel instanceof PerformanceAnalyzerTransportChannel); } + + @Test + public void testGetChannelIfRCAModeIsDisabled() { + Mockito.when(controller.getCollectorsRunModeValue()) + .thenReturn(Util.CollectorMode.TELEMETRY.getValue()); + concreteShardRequest = new ConcreteShardRequest(bulkShardRequest, "id", 1); + handler.getChannel(concreteShardRequest, channel, task); + + Mockito.when(bulkShardRequest.shardId()).thenReturn(shardId); + Mockito.when(bulkShardRequest.items()).thenReturn(new BulkItemRequest[1]); + TransportChannel actualChannel = handler.getChannel(concreteShardRequest, channel, task); + assertEquals(channel, actualChannel); + } + + @Test + public void testGetChannelIfDualModeIsEnabled() { + Mockito.when(controller.getCollectorsRunModeValue()) + .thenReturn(Util.CollectorMode.DUAL.getValue()); + concreteShardRequest = new ConcreteShardRequest(bulkShardRequest, "id", 1); + handler.getChannel(concreteShardRequest, channel, task); + + Mockito.when(bulkShardRequest.shardId()).thenReturn(shardId); + Mockito.when(bulkShardRequest.items()).thenReturn(new BulkItemRequest[1]); + TransportChannel actualChannel = handler.getChannel(concreteShardRequest, channel, task); + assertTrue(actualChannel instanceof PerformanceAnalyzerTransportChannel); + } } diff --git a/src/test/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportChannelTests.java b/src/test/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportChannelTests.java new file mode 100644 index 00000000..aa4e425b --- /dev/null +++ b/src/test/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportChannelTests.java @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.transport; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyDouble; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.MockitoAnnotations.initMocks; + +import java.io.IOException; +import org.apache.commons.lang3.SystemUtils; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.transport.TransportResponse; +import org.opensearch.performanceanalyzer.util.Utils; +import org.opensearch.telemetry.metrics.Histogram; +import org.opensearch.telemetry.metrics.tags.Tags; +import org.opensearch.transport.TransportChannel; + +public class RTFPerformanceAnalyzerTransportChannelTests { + private RTFPerformanceAnalyzerTransportChannel channel; + + @Mock private TransportChannel originalChannel; + @Mock private TransportResponse response; + @Mock private Histogram cpuUtilizationHistogram; + private ShardId shardId; + @Mock private ShardId mockedShardId; + @Mock private Index index; + + @Before + public void init() { + // this test only runs in Linux system + // as some of the static members of the ThreadList class are specific to Linux + org.junit.Assume.assumeTrue(SystemUtils.IS_OS_LINUX); + Utils.configureMetrics(); + initMocks(this); + String indexName = "testIndex"; + shardId = new ShardId(new Index(indexName, "uuid"), 1); + channel = new RTFPerformanceAnalyzerTransportChannel(); + channel.set(originalChannel, cpuUtilizationHistogram, indexName, shardId, false); + assertEquals("RTFPerformanceAnalyzerTransportChannelProfile", channel.getProfileName()); + assertEquals("RTFPerformanceAnalyzerTransportChannelType", channel.getChannelType()); + assertEquals(originalChannel, channel.getInnerChannel()); + } + + @Test + public void testResponse() throws IOException { + channel.sendResponse(response); + verify(originalChannel).sendResponse(response); + verify(cpuUtilizationHistogram, times(1)).record(anyDouble(), any(Tags.class)); + } + + @Test + public void testResponseWithException() throws IOException { + Exception exception = new Exception("dummy exception"); + channel.sendResponse(exception); + verify(originalChannel).sendResponse(exception); + verify(cpuUtilizationHistogram, times(1)).record(anyDouble(), any(Tags.class)); + } + + @Test + public void testRecordCPUUtilizationMetric() { + RTFPerformanceAnalyzerTransportChannel channel = + new RTFPerformanceAnalyzerTransportChannel(); + channel.set(originalChannel, cpuUtilizationHistogram, "testIndex", mockedShardId, false); + Mockito.when(mockedShardId.getIndex()).thenReturn(index); + Mockito.when(index.getName()).thenReturn("myTestIndex"); + Mockito.when(index.getUUID()).thenReturn("abc-def"); + channel.recordCPUUtilizationMetric(mockedShardId, 10l, "bulkShard", false); + Mockito.verify(cpuUtilizationHistogram) + .record(Mockito.anyDouble(), Mockito.any(Tags.class)); + } +} diff --git a/src/test/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportRequestHandlerTests.java b/src/test/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportRequestHandlerTests.java new file mode 100644 index 00000000..9a0f9d0e --- /dev/null +++ b/src/test/java/org/opensearch/performanceanalyzer/transport/RTFPerformanceAnalyzerTransportRequestHandlerTests.java @@ -0,0 +1,120 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.transport; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.verify; +import static org.mockito.MockitoAnnotations.initMocks; + +import org.apache.commons.lang3.SystemUtils; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.opensearch.action.bulk.BulkItemRequest; +import org.opensearch.action.bulk.BulkShardRequest; +import org.opensearch.action.support.replication.TransportReplicationAction.ConcreteShardRequest; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.performanceanalyzer.OpenSearchResources; +import org.opensearch.performanceanalyzer.commons.util.Util; +import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; +import org.opensearch.performanceanalyzer.util.Utils; +import org.opensearch.tasks.Task; +import org.opensearch.telemetry.metrics.MetricsRegistry; +import org.opensearch.transport.TransportChannel; +import org.opensearch.transport.TransportRequest; +import org.opensearch.transport.TransportRequestHandler; + +@SuppressWarnings("unchecked") +public class RTFPerformanceAnalyzerTransportRequestHandlerTests { + private RTFPerformanceAnalyzerTransportRequestHandler handler; + private ConcreteShardRequest concreteShardRequest; + + @Mock private TransportRequestHandler transportRequestHandler; + @Mock private PerformanceAnalyzerController controller; + @Mock private TransportChannel channel; + @Mock private TransportRequest request; + @Mock private BulkShardRequest bulkShardRequest; + @Mock private Task task; + @Mock private ShardId shardId; + @Mock private MetricsRegistry metricsRegistry; + + @Before + public void init() { + // this test only runs in Linux system + // as some of the static members of the ThreadList class are specific to Linux + org.junit.Assume.assumeTrue(SystemUtils.IS_OS_LINUX); + Utils.configureMetrics(); + initMocks(this); + handler = + new RTFPerformanceAnalyzerTransportRequestHandler( + transportRequestHandler, controller); + Mockito.when(controller.isPerformanceAnalyzerEnabled()).thenReturn(true); + } + + @Test + public void testMessageReceived() throws Exception { + handler.messageReceived(request, channel, task); + verify(transportRequestHandler).messageReceived(request, channel, task); + } + + @Test + public void testGetChannel() { + OpenSearchResources.INSTANCE.setMetricsRegistry(metricsRegistry); + Mockito.when(controller.getCollectorsRunModeValue()) + .thenReturn(Util.CollectorMode.TELEMETRY.getValue()); + concreteShardRequest = new ConcreteShardRequest(bulkShardRequest, "id", 1); + handler.getChannel(concreteShardRequest, channel, task); + + Mockito.when(bulkShardRequest.shardId()).thenReturn(shardId); + Mockito.when(bulkShardRequest.items()).thenReturn(new BulkItemRequest[1]); + TransportChannel actualChannel = handler.getChannel(concreteShardRequest, channel, task); + assertTrue(actualChannel instanceof RTFPerformanceAnalyzerTransportChannel); + } + + @Test + public void testGetChannelTelemetryIsDisabled() { + OpenSearchResources.INSTANCE.setMetricsRegistry(metricsRegistry); + Mockito.when(controller.getCollectorsRunModeValue()) + .thenReturn(Util.CollectorMode.RCA.getValue()); + concreteShardRequest = new ConcreteShardRequest(bulkShardRequest, "id", 1); + handler.getChannel(concreteShardRequest, channel, task); + + Mockito.when(bulkShardRequest.shardId()).thenReturn(shardId); + Mockito.when(bulkShardRequest.items()).thenReturn(new BulkItemRequest[1]); + TransportChannel actualChannel = handler.getChannel(concreteShardRequest, channel, task); + assertEquals(channel, actualChannel); + } + + @Test + public void testGetChannelDualMode() { + OpenSearchResources.INSTANCE.setMetricsRegistry(metricsRegistry); + Mockito.when(controller.getCollectorsRunModeValue()) + .thenReturn(Util.CollectorMode.DUAL.getValue()); + concreteShardRequest = new ConcreteShardRequest(bulkShardRequest, "id", 1); + handler.getChannel(concreteShardRequest, channel, task); + + Mockito.when(bulkShardRequest.shardId()).thenReturn(shardId); + Mockito.when(bulkShardRequest.items()).thenReturn(new BulkItemRequest[1]); + TransportChannel actualChannel = handler.getChannel(concreteShardRequest, channel, task); + assertTrue(actualChannel instanceof RTFPerformanceAnalyzerTransportChannel); + } + + @Test + public void testGetChannelMetricRegistryIsNull() { + OpenSearchResources.INSTANCE.setMetricsRegistry(null); + Mockito.when(controller.getCollectorsRunModeValue()) + .thenReturn(Util.CollectorMode.RCA.getValue()); + concreteShardRequest = new ConcreteShardRequest(bulkShardRequest, "id", 1); + handler.getChannel(concreteShardRequest, channel, task); + + Mockito.when(bulkShardRequest.shardId()).thenReturn(shardId); + Mockito.when(bulkShardRequest.items()).thenReturn(new BulkItemRequest[1]); + TransportChannel actualChannel = handler.getChannel(concreteShardRequest, channel, task); + assertEquals(channel, actualChannel); + } +} diff --git a/src/test/java/org/opensearch/performanceanalyzer/util/UtilsTests.java b/src/test/java/org/opensearch/performanceanalyzer/util/UtilsTests.java new file mode 100644 index 00000000..b1e2490e --- /dev/null +++ b/src/test/java/org/opensearch/performanceanalyzer/util/UtilsTests.java @@ -0,0 +1,32 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.util; + +import org.junit.Assert; +import org.junit.Test; + +public class UtilsTests { + + @Test + public void testCPUUtilization() { + Assert.assertEquals(0.5, Utils.calculateCPUUtilization(2, 5, 5, 1.0), 0.0); + Assert.assertEquals(1.0, Utils.calculateCPUUtilization(1, 5, 5, 1.0), 0.0); + Assert.assertEquals( + Double.valueOf(10 / 15.0), Utils.calculateCPUUtilization(3, 5, 10, 1.0), 0.0); + Assert.assertEquals( + Double.valueOf(0.50 * (20 / 30.0)), + Utils.calculateCPUUtilization(3, 10, 20, 0.5d), + 0.0); + } + + @Test + public void testCPUUtilizationZeroValue() { + Assert.assertEquals(0.0, Utils.calculateCPUUtilization(2, 5, 0, 1.0), 0.0); + Assert.assertEquals(0.0, Utils.calculateCPUUtilization(2, 0, 5, 1.0), 0.0); + Assert.assertEquals(0.0, Utils.calculateCPUUtilization(0, 5, 5, 1.0), 0.0); + Assert.assertEquals(0.0, Utils.calculateCPUUtilization(0, 5, 5, 0.0), 0.0); + } +} From 948b54a7f11820f2e5753036545a190d4b4a554b Mon Sep 17 00:00:00 2001 From: Atharva Sharma <60044988+atharvasharma61@users.noreply.github.com> Date: Tue, 23 Jul 2024 21:10:02 +0530 Subject: [PATCH 09/21] [Feature] adds index_uuid as a tag in node stats all shard metrics (#680) * Adds index_uuid as a dimension for node stats col Signed-off-by: Atharva Sharma * updates gauge usage and added index_uuid as tag Signed-off-by: Atharva Sharma * removed incomplete collector Signed-off-by: Atharva Sharma * fixed UTs after gauge changes Signed-off-by: Atharva Sharma * addressed comments Signed-off-by: Atharva Sharma * reverted gauge related changes Signed-off-by: Atharva Sharma --------- Signed-off-by: Atharva Sharma --- ...RTFNodeStatsAllShardsMetricsCollector.java | 30 +++++++++++-------- ...deStatsAllShardsMetricsCollectorTests.java | 4 +-- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFNodeStatsAllShardsMetricsCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFNodeStatsAllShardsMetricsCollector.java index 63033dba..6b32b3c5 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFNodeStatsAllShardsMetricsCollector.java +++ b/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFNodeStatsAllShardsMetricsCollector.java @@ -143,8 +143,7 @@ configOverridesWrapper, getCollectorName())) { // Populating value for the first run. recordMetrics( new NodeStatsMetricsAllShardsPerCollectionStatus(currentShardStats), - shardId.getIndexName(), - String.valueOf(shardId.id())); + shardId); continue; } ShardStats prevShardStats = prevPerShardStats.get(shardId); @@ -153,15 +152,14 @@ configOverridesWrapper, getCollectorName())) { // run. recordMetrics( new NodeStatsMetricsAllShardsPerCollectionStatus(currentShardStats), - shardId.getIndexName(), - String.valueOf(shardId.id())); + shardId); continue; } NodeStatsMetricsAllShardsPerCollectionStatus prevValue = new NodeStatsMetricsAllShardsPerCollectionStatus(prevShardStats); NodeStatsMetricsAllShardsPerCollectionStatus currValue = new NodeStatsMetricsAllShardsPerCollectionStatus(currentShardStats); - populateDiffMetricValue(prevValue, currValue, shardId.getIndexName(), shardId.id()); + populateDiffMetricValue(prevValue, currValue, shardId); } } @@ -243,13 +241,20 @@ public void populatePerShardStats(IndicesService indicesService) { } private void recordMetrics( - NodeStatsMetricsAllShardsPerCollectionStatus metrics, - String indexName, - String shardId) { + NodeStatsMetricsAllShardsPerCollectionStatus metrics, ShardId shardId) { Tags nodeStatsMetricsTag = Tags.create() - .addTag(RTFMetrics.CommonDimension.INDEX_NAME.toString(), indexName) - .addTag(RTFMetrics.CommonDimension.SHARD_ID.toString(), shardId); + .addTag( + RTFMetrics.CommonDimension.INDEX_NAME.toString(), + shardId.getIndexName()) + .addTag( + RTFMetrics.CommonDimension.SHARD_ID.toString(), + String.valueOf(shardId.getId())); + + if (shardId.getIndex() != null) { + nodeStatsMetricsTag.addTag( + RTFMetrics.CommonDimension.INDEX_UUID.toString(), shardId.getIndex().getUUID()); + } cacheQueryMissMetrics.add(metrics.getQueryCacheMissCount(), nodeStatsMetricsTag); cacheQuerySizeMetrics.add(metrics.getQueryCacheInBytes(), nodeStatsMetricsTag); @@ -267,8 +272,7 @@ private void recordMetrics( public void populateDiffMetricValue( NodeStatsMetricsAllShardsPerCollectionStatus prevValue, NodeStatsMetricsAllShardsPerCollectionStatus currValue, - String indexName, - int shardId) { + ShardId shardId) { NodeStatsMetricsAllShardsPerCollectionStatus metrics = new NodeStatsMetricsAllShardsPerCollectionStatus( @@ -289,7 +293,7 @@ public void populateDiffMetricValue( 0), currValue.requestCacheInBytes); - recordMetrics(metrics, indexName, String.valueOf(shardId)); + recordMetrics(metrics, shardId); } public static class NodeStatsMetricsAllShardsPerCollectionStatus extends MetricStatus { diff --git a/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFNodeStatsAllShardsMetricsCollectorTests.java b/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFNodeStatsAllShardsMetricsCollectorTests.java index 089a685b..918c766a 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFNodeStatsAllShardsMetricsCollectorTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFNodeStatsAllShardsMetricsCollectorTests.java @@ -120,11 +120,11 @@ public void testCollectMetrics() throws IOException { createIndex(TEST_INDEX); rtfNodeStatsAllShardsMetricsCollector.collectMetrics(startTimeInMills); verify(rtfNodeStatsAllShardsMetricsCollector, never()) - .populateDiffMetricValue(any(), any(), anyString(), anyInt()); + .populateDiffMetricValue(any(), any(), any()); startTimeInMills += 500; rtfNodeStatsAllShardsMetricsCollector.collectMetrics(startTimeInMills); verify(rtfNodeStatsAllShardsMetricsCollector, times(1)) - .populateDiffMetricValue(any(), any(), anyString(), anyInt()); + .populateDiffMetricValue(any(), any(), any()); verify(cacheFieldDataEvictionCounter, atLeastOnce()).add(anyDouble(), any()); verify(cacheFieldDataSizeCounter, atLeastOnce()).add(anyDouble(), any()); verify(cacheQueryMissCounter, atLeastOnce()).add(anyDouble(), any()); From f6a53e61476637f766586abd06242b478eec677b Mon Sep 17 00:00:00 2001 From: Hector Fernandez Date: Fri, 26 Jul 2024 13:39:05 +0200 Subject: [PATCH 10/21] Bump bouncycastle from 1.74 to 1.78.1 (#656) Signed-off-by: hectorj2f --- build.gradle | 4 ++-- licenses/bcpkix-jdk15to18-1.78.1.jar.sha1 | 1 + licenses/bcpkix-jdk18on-1.74.jar.sha1 | 1 - licenses/bcprov-jdk15to18-1.74.jar.sha1 | 1 - licenses/bcprov-jdk15to18-1.78.1.jar.sha1 | 1 + 5 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 licenses/bcpkix-jdk15to18-1.78.1.jar.sha1 delete mode 100644 licenses/bcpkix-jdk18on-1.74.jar.sha1 delete mode 100644 licenses/bcprov-jdk15to18-1.74.jar.sha1 create mode 100644 licenses/bcprov-jdk15to18-1.78.1.jar.sha1 diff --git a/build.gradle b/build.gradle index 0b8aac4e..a00b603c 100644 --- a/build.gradle +++ b/build.gradle @@ -301,8 +301,8 @@ dependencies { implementation 'org.jooq:jooq:3.10.8' implementation 'org.apache.commons:commons-lang3:3.9' - implementation 'org.bouncycastle:bcprov-jdk15to18:1.74' - implementation 'org.bouncycastle:bcpkix-jdk18on:1.74' + implementation 'org.bouncycastle:bcprov-jdk15to18:1.78.1' + implementation 'org.bouncycastle:bcpkix-jdk18on:1.78.1' implementation "org.opensearch:performance-analyzer-commons:${paCommonsVersion}" implementation "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}" implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonDataBindVersion}" diff --git a/licenses/bcpkix-jdk15to18-1.78.1.jar.sha1 b/licenses/bcpkix-jdk15to18-1.78.1.jar.sha1 new file mode 100644 index 00000000..3a1d2a1e --- /dev/null +++ b/licenses/bcpkix-jdk15to18-1.78.1.jar.sha1 @@ -0,0 +1 @@ +5884ee847542641d04abfbfdeca3446d0300670b diff --git a/licenses/bcpkix-jdk18on-1.74.jar.sha1 b/licenses/bcpkix-jdk18on-1.74.jar.sha1 deleted file mode 100644 index 59ea623a..00000000 --- a/licenses/bcpkix-jdk18on-1.74.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a197fb87f0697c1925e7248865ee84516fdb6d9c \ No newline at end of file diff --git a/licenses/bcprov-jdk15to18-1.74.jar.sha1 b/licenses/bcprov-jdk15to18-1.74.jar.sha1 deleted file mode 100644 index d0d132e6..00000000 --- a/licenses/bcprov-jdk15to18-1.74.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b1a91d0d102042e4ec02084384e1b52b73f125ab \ No newline at end of file diff --git a/licenses/bcprov-jdk15to18-1.78.1.jar.sha1 b/licenses/bcprov-jdk15to18-1.78.1.jar.sha1 new file mode 100644 index 00000000..393c2246 --- /dev/null +++ b/licenses/bcprov-jdk15to18-1.78.1.jar.sha1 @@ -0,0 +1 @@ +83bfa8229f7127d933161aefb281e54a9ffcf9f4 From f45b165ca079f68d81d4027025e15f0c4cbaafb9 Mon Sep 17 00:00:00 2001 From: Gagan Juneja Date: Mon, 29 Jul 2024 23:57:38 +0530 Subject: [PATCH 11/21] Bump PA commons to 1.5.0 (#698) Signed-off-by: Gagan Juneja Co-authored-by: Gagan Juneja --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a00b603c..0f9f3957 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,7 @@ buildscript { // The PA Commons (https://github.com/opensearch-project/performance-analyzer-commons) // is a library dependency with hardcoded versioning in PA and RCA repos. - paCommonsVersion = "1.4.0" + paCommonsVersion = "1.5.0" // 3.0.0-SNAPSHOT -> 3.0.0.0-SNAPSHOT version_tokens = opensearch_version.tokenize('-') From df8fae2a399765365f6caab3f3379fc86c1c3d21 Mon Sep 17 00:00:00 2001 From: Gagan Juneja Date: Tue, 30 Jul 2024 08:50:09 +0530 Subject: [PATCH 12/21] Adds release notes for 2.16.0 (#701) * Adds release notes for 2.16.0 Signed-off-by: Gagan Juneja * Adds release notes for 2.16.0 Signed-off-by: Gagan Juneja --------- Signed-off-by: Gagan Juneja Co-authored-by: Gagan Juneja --- ...rch-performance-analyzer.release-notes-2.16.0.0.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 release-notes/opensearch-performance-analyzer.release-notes-2.16.0.0.md diff --git a/release-notes/opensearch-performance-analyzer.release-notes-2.16.0.0.md b/release-notes/opensearch-performance-analyzer.release-notes-2.16.0.0.md new file mode 100644 index 00000000..e1b3636e --- /dev/null +++ b/release-notes/opensearch-performance-analyzer.release-notes-2.16.0.0.md @@ -0,0 +1,11 @@ +## Version 2.16.0 Release Notes + +Compatible with OpenSearch 2.16.0 + +### Infrastructure +* Bump bouncycastle from 1.74 to 1.78.1 [#656](https://github.com/opensearch-project/performance-analyzer/pull/656) +* Bump PA to use 1.5.0 PA commons lib [#698](https://github.com/opensearch-project/performance-analyzer/pull/698) + +### Feature +* Adds index_uuid as a tag in node stats all shard metrics [#680](https://github.com/opensearch-project/performance-analyzer/pull/680) +* Adds the listener for resource utilization metrics [#687](https://github.com/opensearch-project/performance-analyzer/pull/687) From a6074228e7192d121330ada847cf6395f414d796 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 22 Aug 2024 01:26:21 -0400 Subject: [PATCH 13/21] Update PULL_REQUEST_TEMPLATE to include an API spec change in the checklist. (#679) Signed-off-by: dblock --- .github/PULL_REQUEST_TEMPLATE.md | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d8180108..7c184b90 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,22 +1,16 @@ -**Is your feature request related to a problem? Please provide an existing Issue # , or describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] +### Description +[Describe what this change achieves] -**Describe the solution you are proposing** -A clear and concise description of what you want to happen. - -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. - -**Additional context** -Add any other context or screenshots about the feature request here. +### Related Issues +Resolves #[Issue number to be closed when this PR is merged] + ### Check List -- [ ] Backport Labels added. - [ ] New functionality includes testing. - - [ ] All tests pass - [ ] New functionality has been documented. - - [ ] New functionality has javadoc added -- [ ] Commits are signed per the DCO using --signoff +- [ ] API changes companion pull request [created](https://github.com/opensearch-project/opensearch-api-specification/blob/main/DEVELOPER_GUIDE.md). +- [ ] Commits are signed per the DCO using `--signoff`. +- [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose). By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/performance-analyzer/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). From 2624469183e77c701b55fe0f36f740bb766f8e58 Mon Sep 17 00:00:00 2001 From: Atharva Sharma <60044988+atharvasharma61@users.noreply.github.com> Date: Tue, 3 Sep 2024 21:36:46 +0530 Subject: [PATCH 14/21] Added cacheConfig Collector (#690) --- .../PerformanceAnalyzerPlugin.java | 4 + .../RTFCacheConfigMetricsCollector.java | 181 ++++++++++++++++++ .../performanceanalyzer/util/Utils.java | 8 +- .../RTFCacheConfigMetricsCollectorTests.java | 67 +++++++ 4 files changed, 256 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFCacheConfigMetricsCollector.java create mode 100644 src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFCacheConfigMetricsCollectorTests.java diff --git a/src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java b/src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java index 06f19fa2..84a09648 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java +++ b/src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java @@ -55,6 +55,7 @@ import org.opensearch.performanceanalyzer.collectors.ShardIndexingPressureMetricsCollector; import org.opensearch.performanceanalyzer.collectors.ShardStateCollector; import org.opensearch.performanceanalyzer.collectors.ThreadPoolMetricsCollector; +import org.opensearch.performanceanalyzer.collectors.telemetry.RTFCacheConfigMetricsCollector; import org.opensearch.performanceanalyzer.collectors.telemetry.RTFDisksCollector; import org.opensearch.performanceanalyzer.collectors.telemetry.RTFHeapMetricsCollector; import org.opensearch.performanceanalyzer.collectors.telemetry.RTFNodeStatsAllShardsMetricsCollector; @@ -235,6 +236,9 @@ private void scheduleTelemetryCollectors() { scheduledMetricCollectorsExecutor.addScheduledMetricCollector( new RTFNodeStatsAllShardsMetricsCollector( performanceAnalyzerController, configOverridesWrapper)); + scheduledMetricCollectorsExecutor.addScheduledMetricCollector( + new RTFCacheConfigMetricsCollector( + performanceAnalyzerController, configOverridesWrapper)); } private void scheduleRcaCollectors() { diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFCacheConfigMetricsCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFCacheConfigMetricsCollector.java new file mode 100644 index 00000000..8a5e3e5c --- /dev/null +++ b/src/main/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFCacheConfigMetricsCollector.java @@ -0,0 +1,181 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.collectors.telemetry; + +import static org.opensearch.performanceanalyzer.commons.metrics.AllMetrics.CacheType.FIELD_DATA_CACHE; +import static org.opensearch.performanceanalyzer.commons.metrics.AllMetrics.CacheType.SHARD_REQUEST_CACHE; +import static org.opensearch.performanceanalyzer.commons.stats.decisionmaker.DecisionMakerConsts.CACHE_MAX_WEIGHT; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.security.AccessController; +import java.security.PrivilegedAction; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.cache.Cache; +import org.opensearch.indices.IndicesService; +import org.opensearch.performanceanalyzer.OpenSearchResources; +import org.opensearch.performanceanalyzer.commons.collectors.MetricStatus; +import org.opensearch.performanceanalyzer.commons.collectors.PerformanceAnalyzerMetricsCollector; +import org.opensearch.performanceanalyzer.commons.collectors.TelemetryCollector; +import org.opensearch.performanceanalyzer.commons.config.overrides.ConfigOverridesWrapper; +import org.opensearch.performanceanalyzer.commons.metrics.AllMetrics; +import org.opensearch.performanceanalyzer.commons.metrics.MetricsConfiguration; +import org.opensearch.performanceanalyzer.commons.metrics.RTFMetrics; +import org.opensearch.performanceanalyzer.commons.stats.metrics.StatExceptionCode; +import org.opensearch.performanceanalyzer.commons.stats.metrics.StatMetrics; +import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; +import org.opensearch.telemetry.metrics.MetricsRegistry; +import org.opensearch.telemetry.metrics.tags.Tags; + +public class RTFCacheConfigMetricsCollector extends PerformanceAnalyzerMetricsCollector + implements TelemetryCollector { + private MetricsRegistry metricsRegistry; + private static final Logger LOG = LogManager.getLogger(RTFCacheConfigMetricsCollector.class); + private PerformanceAnalyzerController performanceAnalyzerController; + private ConfigOverridesWrapper configOverridesWrapper; + + public RTFCacheConfigMetricsCollector( + PerformanceAnalyzerController performanceAnalyzerController, + ConfigOverridesWrapper configOverridesWrapper) { + super( + MetricsConfiguration.CONFIG_MAP.get(RTFCacheConfigMetricsCollector.class) + .samplingInterval, + "RTFCacheConfigMetricsCollector", + StatMetrics.RTF_CACHE_CONFIG_METRICS_COLLECTOR_EXECUTION_TIME, + StatExceptionCode.RTF_CACHE_CONFIG_METRICS_COLLECTOR_ERROR); + this.performanceAnalyzerController = performanceAnalyzerController; + this.configOverridesWrapper = configOverridesWrapper; + } + + @Override + public void collectMetrics(long l) { + if (performanceAnalyzerController.isCollectorDisabled( + configOverridesWrapper, getCollectorName())) { + LOG.info("RTFCacheConfigMetricsCollector is disabled. Skipping collection."); + return; + } + + metricsRegistry = OpenSearchResources.INSTANCE.getMetricsRegistry(); + if (metricsRegistry == null) { + LOG.error("could not get the instance of MetricsRegistry class"); + return; + } + + IndicesService indicesService = OpenSearchResources.INSTANCE.getIndicesService(); + if (indicesService == null) { + LOG.error("could not get the instance of indicesService class"); + return; + } + + LOG.debug("Executing collect metrics for RTFCacheConfigMetricsCollector"); + CacheMaxSizeStatus fieldDataCacheMaxSizeStatus = + AccessController.doPrivileged( + (PrivilegedAction) + () -> { + try { + Cache fieldDataCache = + indicesService + .getIndicesFieldDataCache() + .getCache(); + long fieldDataMaxSize = + (Long) + FieldUtils.readField( + fieldDataCache, + CACHE_MAX_WEIGHT, + true); + return new CacheMaxSizeStatus( + FIELD_DATA_CACHE.toString(), fieldDataMaxSize); + } catch (Exception e) { + LOG.debug( + "Error occurred while fetching fieldDataCacheMaxSizeStatus: " + + e.getMessage()); + return null; + } + }); + + CacheMaxSizeStatus shardRequestCacheMaxSizeStatus = + AccessController.doPrivileged( + (PrivilegedAction) + () -> { + try { + Object reqCache = + FieldUtils.readField( + indicesService, + "indicesRequestCache", + true); + Object openSearchOnHeapCache = + FieldUtils.readField(reqCache, "cache", true); + Cache requestCache = + (Cache) + FieldUtils.readField( + openSearchOnHeapCache, + "cache", + true); + Long requestCacheMaxSize = + (Long) + FieldUtils.readField( + requestCache, + CACHE_MAX_WEIGHT, + true); + return new CacheMaxSizeStatus( + SHARD_REQUEST_CACHE.toString(), + requestCacheMaxSize); + } catch (Exception e) { + LOG.debug( + "Error occurred while fetching shardRequestCacheMaxSizeStatus: " + + e.getMessage()); + return null; + } + }); + + if (fieldDataCacheMaxSizeStatus != null + && fieldDataCacheMaxSizeStatus.getCacheMaxSize() > 0) { + recordMetrics(fieldDataCacheMaxSizeStatus); + } + + if (shardRequestCacheMaxSizeStatus != null + && shardRequestCacheMaxSizeStatus.getCacheMaxSize() > 0) { + recordMetrics(shardRequestCacheMaxSizeStatus); + } + } + + private void recordMetrics(CacheMaxSizeStatus cacheMaxSizeStatus) { + metricsRegistry.createGauge( + RTFMetrics.CacheConfigValue.Constants.CACHE_MAX_SIZE_VALUE, + "Cache Max Size metrics", + RTFMetrics.MetricUnits.BYTE.toString(), + () -> (double) cacheMaxSizeStatus.getCacheMaxSize(), + Tags.create() + .addTag( + RTFMetrics.CacheConfigDimension.Constants.TYPE_VALUE, + cacheMaxSizeStatus.getCacheType())); + } + + static class CacheMaxSizeStatus extends MetricStatus { + + private final String cacheType; + + @JsonInclude(JsonInclude.Include.NON_NULL) + private final Long cacheMaxSize; + + CacheMaxSizeStatus(String cacheType, Long cacheMaxSize) { + this.cacheType = cacheType; + this.cacheMaxSize = cacheMaxSize; + } + + @JsonProperty(AllMetrics.CacheConfigDimension.Constants.TYPE_VALUE) + public String getCacheType() { + return cacheType; + } + + @JsonProperty(AllMetrics.CacheConfigValue.Constants.CACHE_MAX_SIZE_VALUE) + public long getCacheMaxSize() { + return cacheMaxSize; + } + } +} diff --git a/src/main/java/org/opensearch/performanceanalyzer/util/Utils.java b/src/main/java/org/opensearch/performanceanalyzer/util/Utils.java index ebf440eb..8245c38b 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/util/Utils.java +++ b/src/main/java/org/opensearch/performanceanalyzer/util/Utils.java @@ -21,10 +21,7 @@ import org.opensearch.indices.IndicesService; import org.opensearch.performanceanalyzer.OpenSearchResources; import org.opensearch.performanceanalyzer.collectors.*; -import org.opensearch.performanceanalyzer.collectors.telemetry.RTFDisksCollector; -import org.opensearch.performanceanalyzer.collectors.telemetry.RTFHeapMetricsCollector; -import org.opensearch.performanceanalyzer.collectors.telemetry.RTFNodeStatsAllShardsMetricsCollector; -import org.opensearch.performanceanalyzer.collectors.telemetry.RTFThreadPoolMetricsCollector; +import org.opensearch.performanceanalyzer.collectors.telemetry.*; import org.opensearch.performanceanalyzer.commons.metrics.MetricsConfiguration; import org.opensearch.performanceanalyzer.commons.stats.ServiceMetrics; @@ -54,6 +51,9 @@ public static void configureMetrics() { MetricsConfiguration.CONFIG_MAP.put(RTFHeapMetricsCollector.class, cdefault); MetricsConfiguration.CONFIG_MAP.put(RTFNodeStatsAllShardsMetricsCollector.class, cdefault); MetricsConfiguration.CONFIG_MAP.put(RTFThreadPoolMetricsCollector.class, cdefault); + MetricsConfiguration.CONFIG_MAP.put( + RTFCacheConfigMetricsCollector.class, + new MetricsConfiguration.MetricConfig(60000, 0)); } // These methods are utility functions for the Node Stat Metrics Collectors. These methods are diff --git a/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFCacheConfigMetricsCollectorTests.java b/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFCacheConfigMetricsCollectorTests.java new file mode 100644 index 00000000..00d5d871 --- /dev/null +++ b/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFCacheConfigMetricsCollectorTests.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.collectors.telemetry; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyDouble; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.opensearch.indices.IndicesService; +import org.opensearch.performanceanalyzer.OpenSearchResources; +import org.opensearch.performanceanalyzer.commons.config.overrides.ConfigOverridesWrapper; +import org.opensearch.performanceanalyzer.commons.metrics.MetricsConfiguration; +import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; +import org.opensearch.telemetry.metrics.Histogram; +import org.opensearch.telemetry.metrics.MetricsRegistry; +import org.opensearch.test.OpenSearchSingleNodeTestCase; + +public class RTFCacheConfigMetricsCollectorTests extends OpenSearchSingleNodeTestCase { + private static final String TEST_INDEX = "test"; + private RTFCacheConfigMetricsCollector rtfCacheConfigMetricsCollector; + private static MetricsRegistry metricsRegistry; + private static Histogram testHistogram; + private long startTimeInMills = 1153721339; + + @Before + public void init() { + MetricsConfiguration.CONFIG_MAP.put( + RTFCacheConfigMetricsCollector.class, MetricsConfiguration.cdefault); + metricsRegistry = mock(MetricsRegistry.class); + testHistogram = mock(Histogram.class); + OpenSearchResources.INSTANCE.setMetricsRegistry(metricsRegistry); + when(metricsRegistry.createHistogram(anyString(), anyString(), anyString())) + .thenReturn(testHistogram); + IndicesService indicesService = getInstanceFromNode(IndicesService.class); + OpenSearchResources.INSTANCE.setIndicesService(indicesService); + ConfigOverridesWrapper mockWrapper = mock(ConfigOverridesWrapper.class); + PerformanceAnalyzerController mockController = mock(PerformanceAnalyzerController.class); + Mockito.when(mockController.isCollectorDisabled(any(), anyString())).thenReturn(false); + rtfCacheConfigMetricsCollector = + spy(new RTFCacheConfigMetricsCollector(mockController, mockWrapper)); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + } + + @Test + public void testCollectMetrics() throws IOException { + createIndex(TEST_INDEX); + rtfCacheConfigMetricsCollector.collectMetrics(startTimeInMills); + verify(testHistogram, atLeastOnce()).record(anyDouble(), any()); + } +} From e25c559eb866dcb3cca97d3a2459e435cf04b27f Mon Sep 17 00:00:00 2001 From: Atharva Sharma <60044988+atharvasharma61@users.noreply.github.com> Date: Wed, 4 Sep 2024 17:02:23 +0530 Subject: [PATCH 15/21] fixes the UT failures caused due to Identity Service (#714) Signed-off-by: Atharva Sharma --- .../performanceanalyzer/PerformanceAnalyzerPluginTests.java | 2 +- .../config/PerformanceAnalyzerClusterConfigActionTests.java | 2 +- .../config/PerformanceAnalyzerConfigActionTests.java | 2 +- .../PerformanceAnalyzerOverridesClusterConfigActionTests.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPluginTests.java b/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPluginTests.java index 554eca27..cf7dd703 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPluginTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPluginTests.java @@ -90,7 +90,7 @@ public void setup() { e.printStackTrace(); } clusterService = new ClusterService(settings, clusterSettings, threadPool); - identityService = new IdentityService(Settings.EMPTY, List.of()); + identityService = new IdentityService(Settings.EMPTY, threadPool, List.of()); restController = new RestController( Collections.emptySet(), diff --git a/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerClusterConfigActionTests.java b/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerClusterConfigActionTests.java index f0e082f5..8767291a 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerClusterConfigActionTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerClusterConfigActionTests.java @@ -71,7 +71,7 @@ public void init() { UsageService usageService = new UsageService(); threadPool = new TestThreadPool("test"); nodeClient = new NodeClient(Settings.EMPTY, threadPool); - identityService = new IdentityService(Settings.EMPTY, List.of()); + identityService = new IdentityService(Settings.EMPTY, threadPool, List.of()); restController = new RestController( Collections.emptySet(), diff --git a/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerConfigActionTests.java b/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerConfigActionTests.java index 1f2ae19f..c182fd35 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerConfigActionTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerConfigActionTests.java @@ -64,7 +64,7 @@ public void init() { UsageService usageService = new UsageService(); threadPool = new TestThreadPool("test"); nodeClient = new NodeClient(Settings.EMPTY, threadPool); - identityService = new IdentityService(Settings.EMPTY, List.of()); + identityService = new IdentityService(Settings.EMPTY, threadPool, List.of()); restController = new RestController( Collections.emptySet(), diff --git a/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerOverridesClusterConfigActionTests.java b/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerOverridesClusterConfigActionTests.java index 66c0a65a..16adf381 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerOverridesClusterConfigActionTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerOverridesClusterConfigActionTests.java @@ -65,7 +65,7 @@ public void init() { UsageService usageService = new UsageService(); threadPool = new TestThreadPool("test"); nodeClient = new NodeClient(Settings.EMPTY, threadPool); - identityService = new IdentityService(Settings.EMPTY, List.of()); + identityService = new IdentityService(Settings.EMPTY, threadPool, List.of()); restController = new RestController( Collections.emptySet(), From 7fd7bc76a0b213ff4bb5cb76f3b877d7ff82d6e6 Mon Sep 17 00:00:00 2001 From: Atharva Sharma <60044988+atharvasharma61@users.noreply.github.com> Date: Wed, 4 Sep 2024 17:03:18 +0530 Subject: [PATCH 16/21] fixes the cacheConfigRTFCollector UT (#713) Signed-off-by: Atharva Sharma --- .../telemetry/RTFCacheConfigMetricsCollectorTests.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFCacheConfigMetricsCollectorTests.java b/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFCacheConfigMetricsCollectorTests.java index 00d5d871..27d55e36 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFCacheConfigMetricsCollectorTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/collectors/telemetry/RTFCacheConfigMetricsCollectorTests.java @@ -6,13 +6,11 @@ package org.opensearch.performanceanalyzer.collectors.telemetry; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyDouble; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; import java.io.IOException; import org.junit.After; @@ -24,7 +22,6 @@ import org.opensearch.performanceanalyzer.commons.config.overrides.ConfigOverridesWrapper; import org.opensearch.performanceanalyzer.commons.metrics.MetricsConfiguration; import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; -import org.opensearch.telemetry.metrics.Histogram; import org.opensearch.telemetry.metrics.MetricsRegistry; import org.opensearch.test.OpenSearchSingleNodeTestCase; @@ -32,7 +29,6 @@ public class RTFCacheConfigMetricsCollectorTests extends OpenSearchSingleNodeTes private static final String TEST_INDEX = "test"; private RTFCacheConfigMetricsCollector rtfCacheConfigMetricsCollector; private static MetricsRegistry metricsRegistry; - private static Histogram testHistogram; private long startTimeInMills = 1153721339; @Before @@ -40,10 +36,7 @@ public void init() { MetricsConfiguration.CONFIG_MAP.put( RTFCacheConfigMetricsCollector.class, MetricsConfiguration.cdefault); metricsRegistry = mock(MetricsRegistry.class); - testHistogram = mock(Histogram.class); OpenSearchResources.INSTANCE.setMetricsRegistry(metricsRegistry); - when(metricsRegistry.createHistogram(anyString(), anyString(), anyString())) - .thenReturn(testHistogram); IndicesService indicesService = getInstanceFromNode(IndicesService.class); OpenSearchResources.INSTANCE.setIndicesService(indicesService); ConfigOverridesWrapper mockWrapper = mock(ConfigOverridesWrapper.class); @@ -62,6 +55,7 @@ public void tearDown() throws Exception { public void testCollectMetrics() throws IOException { createIndex(TEST_INDEX); rtfCacheConfigMetricsCollector.collectMetrics(startTimeInMills); - verify(testHistogram, atLeastOnce()).record(anyDouble(), any()); + verify(metricsRegistry, atLeastOnce()) + .createGauge(anyString(), anyString(), anyString(), any(), any()); } } From 5a2ac55dd4a67b3635f407bf0614684988bf5aca Mon Sep 17 00:00:00 2001 From: Atharva Sharma <60044988+atharvasharma61@users.noreply.github.com> Date: Wed, 4 Sep 2024 23:16:43 +0530 Subject: [PATCH 17/21] bump up PA commons to 1.6 (#712) Signed-off-by: Atharva Sharma --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 0f9f3957..4433815d 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,7 @@ buildscript { // The PA Commons (https://github.com/opensearch-project/performance-analyzer-commons) // is a library dependency with hardcoded versioning in PA and RCA repos. - paCommonsVersion = "1.5.0" + paCommonsVersion = "1.6.0" // 3.0.0-SNAPSHOT -> 3.0.0.0-SNAPSHOT version_tokens = opensearch_version.tokenize('-') From dc57bb4c62348abfc6d043e0ea20a8947a108029 Mon Sep 17 00:00:00 2001 From: Gagan Juneja Date: Fri, 6 Sep 2024 12:17:12 +0530 Subject: [PATCH 18/21] Adds release notes for 2.17.0 (#725) Signed-off-by: Gagan Juneja Co-authored-by: Gagan Juneja --- ...search-performance-analyzer.release-notes-2.17.0.0.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 release-notes/opensearch-performance-analyzer.release-notes-2.17.0.0.md diff --git a/release-notes/opensearch-performance-analyzer.release-notes-2.17.0.0.md b/release-notes/opensearch-performance-analyzer.release-notes-2.17.0.0.md new file mode 100644 index 00000000..969263bc --- /dev/null +++ b/release-notes/opensearch-performance-analyzer.release-notes-2.17.0.0.md @@ -0,0 +1,9 @@ +## Version 2.17.0 Release Notes + +Compatible with OpenSearch 2.17.0 + +### Infrastructure +* Bump PA to use 1.6.0 PA commons lib [#712](https://github.com/opensearch-project/performance-analyzer/pull/712) + +### Feature +* Added cacheConfig Collector [#690](https://github.com/opensearch-project/performance-analyzer/pull/690) From ca521d5206d52bc926947b4a9dbfa7f3b0c1d998 Mon Sep 17 00:00:00 2001 From: Gagan Juneja Date: Mon, 23 Sep 2024 14:18:40 +0530 Subject: [PATCH 19/21] Fixes failing docker issue (#733) Signed-off-by: Gagan Juneja Co-authored-by: Gagan Juneja --- .github/workflows/gradle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index ccc9aa2a..35fa767e 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -31,7 +31,7 @@ jobs: # Explicitly set the docker-compose program path so that our build scripts in RCA can run the program # This is necessary because of the Github Actions environment and the workingDir of the Gradle environment - name: Set docker-compose path - run: DOCKER_COMPOSE_LOCATION=$(which docker-compose) + run: echo "DOCKER_COMPOSE_LOCATION=$(which docker-compose)" >> $GITHUB_ENV # Set the vm.max_map_count system property to the minimum required to run OpenSearch - name: Set vm.max_map_count run: sudo sysctl -w vm.max_map_count=262144 From c643111d76f63eadd5fe1196844f96a84df530bb Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 23 Oct 2024 01:05:32 -0400 Subject: [PATCH 20/21] Remove IdentityService from RestController constructor (#736) * Remove IdentityService from RestController constructor Signed-off-by: Craig Perkins * Upgrade bwc version Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- build.gradle | 4 ++-- .../PerformanceAnalyzerPluginTests.java | 6 +----- .../PerformanceAnalyzerClusterConfigActionTests.java | 7 +------ .../config/PerformanceAnalyzerConfigActionTests.java | 7 +------ ...rformanceAnalyzerOverridesClusterConfigActionTests.java | 7 +------ 5 files changed, 6 insertions(+), 25 deletions(-) diff --git a/build.gradle b/build.gradle index 4433815d..9bd72dca 100644 --- a/build.gradle +++ b/build.gradle @@ -10,7 +10,7 @@ buildscript { isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") - default_bwc_version = System.getProperty("bwc.version", "2.12.0.0") + default_bwc_version = System.getProperty("bwc.version", "2.18.0.0") pa_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) baseName = "paBwcCluster" @@ -669,7 +669,7 @@ task integTestRemote(type: RestIntegTestTask) { testClusters { "${baseName}$i" { testDistribution = "ARCHIVE" - versions = ["2.12.0-SNAPSHOT", opensearch_version] + versions = ["2.18.0-SNAPSHOT", opensearch_version] numberOfNodes = 3 plugin(provider(new Callable() { @Override diff --git a/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPluginTests.java b/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPluginTests.java index cf7dd703..074267fb 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPluginTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPluginTests.java @@ -30,7 +30,6 @@ import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.env.Environment; import org.opensearch.env.TestEnvironment; -import org.opensearch.identity.IdentityService; import org.opensearch.indices.breaker.BreakerSettings; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.performanceanalyzer.action.PerformanceAnalyzerActionFilter; @@ -64,7 +63,6 @@ public class PerformanceAnalyzerPluginTests extends OpenSearchTestCase { private CircuitBreakerService circuitBreakerService; private ClusterService clusterService; private ClusterSettings clusterSettings; - private IdentityService identityService; private MetricsRegistry metricsRegistry; @@ -90,15 +88,13 @@ public void setup() { e.printStackTrace(); } clusterService = new ClusterService(settings, clusterSettings, threadPool); - identityService = new IdentityService(Settings.EMPTY, threadPool, List.of()); restController = new RestController( Collections.emptySet(), null, nodeClient, circuitBreakerService, - usageService, - identityService); + usageService); } @After diff --git a/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerClusterConfigActionTests.java b/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerClusterConfigActionTests.java index 8767291a..89ae282a 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerClusterConfigActionTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerClusterConfigActionTests.java @@ -12,7 +12,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.List; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -28,7 +27,6 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.identity.IdentityService; import org.opensearch.indices.breaker.BreakerSettings; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; @@ -54,7 +52,6 @@ public class PerformanceAnalyzerClusterConfigActionTests { private PerformanceAnalyzerClusterSettingHandler clusterSettingHandler; private NodeStatsSettingHandler nodeStatsSettingHandler; private PerformanceAnalyzerCollectorsSettingHandler performanceAnalyzerCollectorsSettingHandler; - private IdentityService identityService; @Mock private PerformanceAnalyzerController controller; @Mock private ClusterSettingsManager clusterSettingsManager; @@ -71,15 +68,13 @@ public void init() { UsageService usageService = new UsageService(); threadPool = new TestThreadPool("test"); nodeClient = new NodeClient(Settings.EMPTY, threadPool); - identityService = new IdentityService(Settings.EMPTY, threadPool, List.of()); restController = new RestController( Collections.emptySet(), null, nodeClient, circuitBreakerService, - usageService, - identityService); + usageService); clusterSettingHandler = new PerformanceAnalyzerClusterSettingHandler(controller, clusterSettingsManager); nodeStatsSettingHandler = new NodeStatsSettingHandler(controller, clusterSettingsManager); diff --git a/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerConfigActionTests.java b/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerConfigActionTests.java index c182fd35..77ff05a9 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerConfigActionTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerConfigActionTests.java @@ -12,7 +12,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.List; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -29,7 +28,6 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.identity.IdentityService; import org.opensearch.indices.breaker.BreakerSettings; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; @@ -48,7 +46,6 @@ public class PerformanceAnalyzerConfigActionTests { private NodeClient nodeClient; private CircuitBreakerService circuitBreakerService; private ClusterSettings clusterSettings; - private IdentityService identityService; @Mock private PerformanceAnalyzerController controller; @@ -64,15 +61,13 @@ public void init() { UsageService usageService = new UsageService(); threadPool = new TestThreadPool("test"); nodeClient = new NodeClient(Settings.EMPTY, threadPool); - identityService = new IdentityService(Settings.EMPTY, threadPool, List.of()); restController = new RestController( Collections.emptySet(), null, nodeClient, circuitBreakerService, - usageService, - identityService); + usageService); configAction = new PerformanceAnalyzerConfigAction(restController, controller); restController.registerHandler(configAction); diff --git a/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerOverridesClusterConfigActionTests.java b/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerOverridesClusterConfigActionTests.java index 16adf381..e5e43658 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerOverridesClusterConfigActionTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/http_action/config/PerformanceAnalyzerOverridesClusterConfigActionTests.java @@ -13,7 +13,6 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; -import java.util.List; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -27,7 +26,6 @@ import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.identity.IdentityService; import org.opensearch.indices.breaker.BreakerSettings; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.performanceanalyzer.commons.config.overrides.ConfigOverridesWrapper; @@ -48,7 +46,6 @@ public class PerformanceAnalyzerOverridesClusterConfigActionTests { private NodeClient nodeClient; private CircuitBreakerService circuitBreakerService; private ClusterSettings clusterSettings; - private IdentityService identityService; @Mock private ConfigOverridesClusterSettingHandler configOverridesClusterSettingHandler; @Mock private ConfigOverridesWrapper overridesWrapper; @@ -65,15 +62,13 @@ public void init() { UsageService usageService = new UsageService(); threadPool = new TestThreadPool("test"); nodeClient = new NodeClient(Settings.EMPTY, threadPool); - identityService = new IdentityService(Settings.EMPTY, threadPool, List.of()); restController = new RestController( Collections.emptySet(), null, nodeClient, circuitBreakerService, - usageService, - identityService); + usageService); configAction = new PerformanceAnalyzerOverridesClusterConfigAction( Settings.EMPTY, From 77fdbaa703fae748340728206228f81e4d5ec3c9 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 23 Oct 2024 09:46:53 -0400 Subject: [PATCH 21/21] Fix test report and code coverage upload (#739) * Fix code coverage upload Signed-off-by: Craig Perkins * Update path Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- .github/workflows/gradle.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 35fa767e..b579c9ab 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -55,10 +55,14 @@ jobs: working-directory: ./tmp/performance-analyzer run: ./gradlew jacocoTestReport - name: Upload coverage report - working-directory: ./tmp/performance-analyzer - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - run: bash <(curl -s https://codecov.io/bash) -f ./build/reports/jacoco/test/jacocoTestReport.xml + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + - uses: actions/upload-artifact@v3 + with: + name: linux-JDK${{ matrix.java }}-reports + path: | + ./tmp/performance-analyzer/build/reports/ - name: Run Integration Tests working-directory: ./tmp/performance-analyzer run: ./gradlew integTest -Dtests.enableIT -Dtests.useDockerCluster