From 90d1f491e1d51fc4f2cfe228c1a162be5fc1d989 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 7 Jul 2022 12:59:29 -0700 Subject: [PATCH 001/215] fixed ktlint errors Signed-off-by: Richard --- .../alerting/TriggerServiceTests.kt | 40 ++++++++++++++++--- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/TriggerServiceTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/TriggerServiceTests.kt index d4b7d63a4..1fe13970d 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/TriggerServiceTests.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/TriggerServiceTests.kt @@ -37,9 +37,18 @@ class TriggerServiceTests : OpenSearchTestCase() { val trigger = randomBucketLevelTrigger(bucketSelector = bucketSelectorExtAggregationBuilder) val monitor = randomBucketLevelMonitor(triggers = listOf(trigger)) - val inputResultsStr = "{\"_shards\":{\"total\":1,\"failed\":0,\"successful\":1,\"skipped\":0},\"hits\":{\"hits\":[{\"_index\":\"sample-http-responses\",\"_type\":\"http\",\"_source\":{\"status_code\":100,\"http_4xx\":0,\"http_3xx\":0,\"http_5xx\":0,\"http_2xx\":0,\"timestamp\":100000,\"http_1xx\":1},\"_id\":1,\"_score\":1}],\"total\":{\"value\":4,\"relation\":\"eq\"},\"max_score\":1},\"took\":37,\"timed_out\":false,\"aggregations\":{\"status_code\":{\"doc_count_error_upper_bound\":0,\"sum_other_doc_count\":0,\"buckets\":[{\"doc_count\":2,\"key\":100},{\"doc_count\":1,\"key\":102},{\"doc_count\":1,\"key\":201}]},\"${trigger.id}\":{\"parent_bucket_path\":\"status_code\",\"bucket_indices\":[0,1,2]}}}" - - val parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputResultsStr) + val inputResultsStr = "{\"_shards\":{\"total\":1,\"failed\":0,\"successful\":1,\"skipped\":0},\"hits\":{\"hits\":[{\"_index\":" + + "\"sample-http-responses\",\"_type\":\"http\",\"_source\":{\"status_code\":100,\"http_4xx\":0,\"http_3xx\":0," + + "\"http_5xx\":0,\"http_2xx\":0,\"timestamp\":100000,\"http_1xx\":1},\"_id\":1,\"_score\":1}]," + + "\"total\":{\"value\":4,\"relation\":\"eq\"},\"max_score\":1},\"took\":37,\"timed_out\":false," + + "\"aggregations\":{\"status_code\":{\"doc_count_error_upper_bound\":0,\"sum_other_doc_count\":0," + + "\"buckets\":[{\"doc_count\":2,\"key\":100},{\"doc_count\":1,\"key\":102},{\"doc_count\":1,\"key\":201}]}," + + "\"${trigger.id}\":{\"parent_bucket_path\":\"status_code\",\"bucket_indices\":[0,1,2]}}}" + + val parser = XContentType.JSON.xContent().createParser( + NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + inputResultsStr + ) val inputResults = parser.map() @@ -60,9 +69,28 @@ class TriggerServiceTests : OpenSearchTestCase() { val trigger = randomBucketLevelTrigger(bucketSelector = bucketSelectorExtAggregationBuilder) val monitor = randomBucketLevelMonitor(triggers = listOf(trigger)) - val inputResultsStr = "{\"_shards\":{\"total\":1, \"failed\":0, \"successful\":1, \"skipped\":0}, \"hits\":{\"hits\":[{\"_index\":\"sample-http-responses\", \"_type\":\"http\", \"_source\":{\"status_code\":100, \"http_4xx\":0, \"http_3xx\":0, \"http_5xx\":0, \"http_2xx\":0, \"timestamp\":100000, \"http_1xx\":1}, \"_id\":1, \"_score\":1.0}, {\"_index\":\"sample-http-responses\", \"_type\":\"http\", \"_source\":{\"status_code\":102, \"http_4xx\":0, \"http_3xx\":0, \"http_5xx\":0, \"http_2xx\":0, \"timestamp\":160000, \"http_1xx\":1}, \"_id\":2, \"_score\":1.0}, {\"_index\":\"sample-http-responses\", \"_type\":\"http\", \"_source\":{\"status_code\":100, \"http_4xx\":0, \"http_3xx\":0, \"http_5xx\":0, \"http_2xx\":0, \"timestamp\":220000, \"http_1xx\":1}, \"_id\":4, \"_score\":1.0}, {\"_index\":\"sample-http-responses\", \"_type\":\"http\", \"_source\":{\"status_code\":201, \"http_4xx\":0, \"http_3xx\":0, \"http_5xx\":0, \"http_2xx\":1, \"timestamp\":280000, \"http_1xx\":0}, \"_id\":5, \"_score\":1.0}], \"total\":{\"value\":4, \"relation\":\"eq\"}, \"max_score\":1.0}, \"took\":15, \"timed_out\":false, \"aggregations\":{\"${trigger.id}\":{\"parent_bucket_path\":\"status_code\", \"bucket_indices\":[0, 1, 2]}, \"status_code\":{\"buckets\":[{\"doc_count\":2, \"key\":{\"status_code\":100}}, {\"doc_count\":1, \"key\":{\"status_code\":102}}, {\"doc_count\":1, \"key\":{\"status_code\":201}}], \"after_key\":{\"status_code\":201}}}}" - - val parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputResultsStr) + val inputResultsStr = "{\"_shards\":{\"total\":1, \"failed\":0, \"successful\":1, \"skipped\":0}, " + + "\"hits\":{\"hits\":[{\"_index\":\"sample-http-responses\", \"_type\":\"http\", " + + "\"_source\":{\"status_code\":100, \"http_4xx\":0, \"http_3xx\":0, \"http_5xx\":0, " + + "\"http_2xx\":0, \"timestamp\":100000, \"http_1xx\":1}, \"_id\":1, \"_score\":1.0}, " + + "{\"_index\":\"sample-http-responses\", \"_type\":\"http\", \"_source\":{\"status_code\":102, " + + "\"http_4xx\":0, \"http_3xx\":0, \"http_5xx\":0, \"http_2xx\":0, \"timestamp\":160000, \"http_1xx\":1}, " + + "\"_id\":2, \"_score\":1.0}, {\"_index\":\"sample-http-responses\", " + + "\"_type\":\"http\", \"_source\":{\"status_code\":100, " + + "\"http_4xx\":0, \"http_3xx\":0, \"http_5xx\":0, \"http_2xx\":0, " + + "\"timestamp\":220000, \"http_1xx\":1}, \"_id\":4, \"_score\":1.0}, {\"_index\":\"sample-http-responses\", " + + "\"_type\":\"http\", \"_source\":{\"status_code\":201, \"http_4xx\":0, \"http_3xx\":0, \"http_5xx\":0, \"http_2xx\":1, " + + "\"timestamp\":280000, \"http_1xx\":0}, \"_id\":5, \"_score\":1.0}], \"total\":{\"value\":4, \"relation\":\"eq\"}, " + + "\"max_score\":1.0}, \"took\":15, \"timed_out\":false, " + + "\"aggregations\":{\"${trigger.id}\":{\"parent_bucket_path\":\"status_code\", \"bucket_indices\":[0, 1, 2]}, " + + "\"status_code\":{\"buckets\":[{\"doc_count\":2, \"key\":{\"status_code\":100}}, " + + "{\"doc_count\":1, \"key\":{\"status_code\":102}}, {\"doc_count\":1, \"key\":{\"status_code\":201}}], " + + "\"after_key\":{\"status_code\":201}}}}" + + val parser = XContentType.JSON.xContent().createParser( + NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + inputResultsStr + ) val inputResults = parser.map() From 09858a50d39c59311e7d10a41260dda572bc6ada Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 7 Jul 2022 13:57:16 -0700 Subject: [PATCH 002/215] create visualization index commit Signed-off-by: Richard --- .../org/opensearch/alerting/AlertingPlugin.kt | 41 ++++++++++++++++++- .../util/ClusterMetricsVisualizationIndex.kt | 41 +++++++++++++++++++ .../mappings/metrics-visualizations.json | 19 +++++++++ 3 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt create mode 100644 core/src/main/resources/mappings/metrics-visualizations.json diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 89ed19c23..745966f77 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -7,6 +7,7 @@ package org.opensearch.alerting import org.opensearch.action.ActionRequest import org.opensearch.action.ActionResponse +import org.opensearch.action.support.WriteRequest import org.opensearch.alerting.action.AcknowledgeAlertAction import org.opensearch.alerting.action.DeleteMonitorAction import org.opensearch.alerting.action.ExecuteMonitorAction @@ -17,6 +18,7 @@ import org.opensearch.alerting.action.GetEmailGroupAction import org.opensearch.alerting.action.GetFindingsAction import org.opensearch.alerting.action.GetMonitorAction import org.opensearch.alerting.action.IndexMonitorAction +import org.opensearch.alerting.action.IndexMonitorRequest import org.opensearch.alerting.action.SearchEmailAccountAction import org.opensearch.alerting.action.SearchEmailGroupAction import org.opensearch.alerting.action.SearchMonitorAction @@ -27,6 +29,7 @@ import org.opensearch.alerting.core.ScheduledJobIndices import org.opensearch.alerting.core.action.node.ScheduledJobsStatsAction import org.opensearch.alerting.core.action.node.ScheduledJobsStatsTransportAction import org.opensearch.alerting.core.model.ClusterMetricsInput +import org.opensearch.alerting.core.model.CronSchedule import org.opensearch.alerting.core.model.DocLevelMonitorInput import org.opensearch.alerting.core.model.ScheduledJob import org.opensearch.alerting.core.model.SearchInput @@ -69,6 +72,7 @@ import org.opensearch.alerting.transport.TransportIndexMonitorAction import org.opensearch.alerting.transport.TransportSearchEmailAccountAction import org.opensearch.alerting.transport.TransportSearchEmailGroupAction import org.opensearch.alerting.transport.TransportSearchMonitorAction +import org.opensearch.alerting.util.ClusterMetricsVisualizationIndex import org.opensearch.alerting.util.DocLevelMonitorQueries import org.opensearch.alerting.util.destinationmigration.DestinationMigrationCoordinator import org.opensearch.client.Client @@ -83,7 +87,10 @@ import org.opensearch.common.settings.Setting import org.opensearch.common.settings.Settings import org.opensearch.common.settings.SettingsFilter import org.opensearch.common.xcontent.NamedXContentRegistry +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentType import org.opensearch.env.Environment import org.opensearch.env.NodeEnvironment import org.opensearch.index.IndexModule @@ -98,10 +105,13 @@ import org.opensearch.plugins.SearchPlugin import org.opensearch.repositories.RepositoriesService import org.opensearch.rest.RestController import org.opensearch.rest.RestHandler +import org.opensearch.rest.RestRequest import org.opensearch.script.ScriptContext import org.opensearch.script.ScriptService import org.opensearch.threadpool.ThreadPool import org.opensearch.watcher.ResourceWatcherService +import java.time.Instant +import java.time.ZoneId import java.util.function.Supplier /** @@ -212,10 +222,36 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R nodeEnvironment: NodeEnvironment, namedWriteableRegistry: NamedWriteableRegistry, indexNameExpressionResolver: IndexNameExpressionResolver, - repositoriesServiceSupplier: Supplier + repositoriesServiceSupplier: Supplier, + ): Collection { // Need to figure out how to use the OpenSearch DI classes rather than handwiring things here. val settings = environment.settings() + val cronSchedule = CronSchedule("*/15 * * * *", ZoneId.of("US/Pacific")) + val monitor = Monitor( + id = "", + version = 0L, + name = "yepclock", + enabled = true, + user = null, + schedule = cronSchedule, + lastUpdateTime = Instant.now(), + enabledTime = Instant.now(), + monitorType = Monitor.MonitorType.CLUSTER_METRICS_MONITOR, + schemaVersion = 0, + inputs = mutableListOf(), + triggers = mutableListOf(), + uiMetadata = mutableMapOf() + ) + val monitorRequest = IndexMonitorRequest( + monitorId = "", + seqNo = 0L, + primaryTerm = 0L, + refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, + RestRequest.Method.POST, monitor + ) + val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() + response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) alertIndices = AlertIndices(settings, client, threadPool, clusterService) runner = MonitorRunnerService .registerClusterService(clusterService) @@ -236,6 +272,9 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R scheduler = JobScheduler(threadPool, runner) sweeper = JobSweeper(environment.settings(), client, clusterService, threadPool, xContentRegistry, scheduler, ALERTING_JOB_TYPES) destinationMigrationCoordinator = DestinationMigrationCoordinator(client, clusterService, threadPool, scheduledJobIndices) + // create a cluster metrics visualization index upon initialization of alerting plugin + ClusterMetricsVisualizationIndex(client, clusterService) + // Create Monitor using Monitor.kt, and then feed the request to client in some way this.threadPool = threadPool this.clusterService = clusterService return listOf(sweeper, scheduler, runner, scheduledJobIndices, docLevelMonitorQueries, destinationMigrationCoordinator) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt new file mode 100644 index 000000000..bce7133c7 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -0,0 +1,41 @@ +package org.opensearch.alerting.util + +import org.apache.logging.log4j.LogManager +import org.opensearch.action.admin.indices.create.CreateIndexRequest +import org.opensearch.action.admin.indices.create.CreateIndexResponse +import org.opensearch.client.Client +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.settings.Settings + +private val log = LogManager.getLogger(ClusterMetricsVisualizationIndex::class.java) + +class ClusterMetricsVisualizationIndex(private val client: Client, private val clusterService: ClusterService) { + /** The index name pattern for all cluster metric visualizations indices */ + val CLUSTER_METRIC_VISUALIZATION_INDEX = ".opendistro-alerting-cluster-metrics" + + companion object { + @JvmStatic + fun clusterMetricsVisualizationsMappings(): String { + return ClusterMetricsVisualizationIndex::class.java.classLoader.getResource("mappings/metrics-visualizations.json").readText() + } + } + + init { + if (!clusterMetricsVisualizationIndexExists()) { + val indexRequest = CreateIndexRequest(CLUSTER_METRIC_VISUALIZATION_INDEX) + .mapping(clusterMetricsVisualizationsMappings()) + .settings( + Settings.builder().put("index.hidden", true) + .build() + ) + val createIndexResponse: CreateIndexResponse = client.admin().indices().create(indexRequest).get() + createIndexResponse.isAcknowledged + } + } + // potentially want to use suspendUntil, tomorrow do similar to ClusterMetricsInput.kt class + + fun clusterMetricsVisualizationIndexExists(): Boolean { + val clusterState = clusterService.state() + return clusterState.routingTable.hasIndex(CLUSTER_METRIC_VISUALIZATION_INDEX) + } +} diff --git a/core/src/main/resources/mappings/metrics-visualizations.json b/core/src/main/resources/mappings/metrics-visualizations.json new file mode 100644 index 000000000..f11efdbf0 --- /dev/null +++ b/core/src/main/resources/mappings/metrics-visualizations.json @@ -0,0 +1,19 @@ +{ + "_meta": { + "schema_version": 1 + }, + "properties": { + "metric":{ + "type": "nested", + "properties": { + "date": { + "type": "text", + "format": "basic_date_time" + }, + "metric_value": { + "type": "text" + } + } + } + } +} \ No newline at end of file From f641e804efb521791ed94f473d4a34d0bae40dce Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 7 Jul 2022 14:48:22 -0700 Subject: [PATCH 003/215] create visualization index commit Signed-off-by: Richard --- .../org/opensearch/alerting/AlertingPlugin.kt | 11 +++++---- .../util/ClusterMetricsVisualizationIndex.kt | 24 +++++++++++++++++-- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 745966f77..118d09517 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -5,6 +5,7 @@ package org.opensearch.alerting +import org.apache.logging.log4j.LogManager import org.opensearch.action.ActionRequest import org.opensearch.action.ActionResponse import org.opensearch.action.support.WriteRequest @@ -120,6 +121,7 @@ import java.util.function.Supplier * It also adds [Monitor.XCONTENT_REGISTRY], [SearchInput.XCONTENT_REGISTRY], [QueryLevelTrigger.XCONTENT_REGISTRY], * [BucketLevelTrigger.XCONTENT_REGISTRY], [ClusterMetricsInput.XCONTENT_REGISTRY] to the [NamedXContentRegistry] so that we are able to deserialize the custom named objects. */ +private val log = LogManager.getLogger(AlertingPlugin::class.java) internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, ReloadablePlugin, SearchPlugin, PercolatorPluginExt() { override fun getContextWhitelists(): Map, List> { @@ -250,8 +252,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, RestRequest.Method.POST, monitor ) - val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() - response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) + alertIndices = AlertIndices(settings, client, threadPool, clusterService) runner = MonitorRunnerService .registerClusterService(clusterService) @@ -272,11 +273,13 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R scheduler = JobScheduler(threadPool, runner) sweeper = JobSweeper(environment.settings(), client, clusterService, threadPool, xContentRegistry, scheduler, ALERTING_JOB_TYPES) destinationMigrationCoordinator = DestinationMigrationCoordinator(client, clusterService, threadPool, scheduledJobIndices) - // create a cluster metrics visualization index upon initialization of alerting plugin - ClusterMetricsVisualizationIndex(client, clusterService) // Create Monitor using Monitor.kt, and then feed the request to client in some way this.threadPool = threadPool this.clusterService = clusterService + // create a cluster metrics visualization index upon initialization of alerting plugin + ClusterMetricsVisualizationIndex(client, clusterService) + val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() + response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) return listOf(sweeper, scheduler, runner, scheduledJobIndices, docLevelMonitorQueries, destinationMigrationCoordinator) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index bce7133c7..7d194f552 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -5,6 +5,7 @@ import org.opensearch.action.admin.indices.create.CreateIndexRequest import org.opensearch.action.admin.indices.create.CreateIndexResponse import org.opensearch.client.Client import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.settings.Setting import org.opensearch.common.settings.Settings private val log = LogManager.getLogger(ClusterMetricsVisualizationIndex::class.java) @@ -12,6 +13,7 @@ private val log = LogManager.getLogger(ClusterMetricsVisualizationIndex::class.j class ClusterMetricsVisualizationIndex(private val client: Client, private val clusterService: ClusterService) { /** The index name pattern for all cluster metric visualizations indices */ val CLUSTER_METRIC_VISUALIZATION_INDEX = ".opendistro-alerting-cluster-metrics" + val METRICS_HISTORY_ENABLED = Setting.boolSetting("opendistro.alerting.metrics_history_enabled", true) companion object { @JvmStatic @@ -31,9 +33,27 @@ class ClusterMetricsVisualizationIndex(private val client: Client, private val c val createIndexResponse: CreateIndexResponse = client.admin().indices().create(indexRequest).get() createIndexResponse.isAcknowledged } +// clusterService.addListener(this) +// clusterService.clusterSettings.addSettingsUpdateConsumer(AlertingSettings.ALERT_HISTORY_ENABLED) { alertHistoryEnabled = it } +// clusterService.clusterSettings.addSettingsUpdateConsumer(AlertingSettings.ALERT_HISTORY_MAX_DOCS) { alertHistoryMaxDocs = it } +// clusterService.clusterSettings.addSettingsUpdateConsumer(AlertingSettings.ALERT_HISTORY_INDEX_MAX_AGE) { alertHistoryMaxAge = it } +// clusterService.clusterSettings.addSettingsUpdateConsumer(AlertingSettings.ALERT_HISTORY_ROLLOVER_PERIOD) { +// alertHistoryRolloverPeriod = it +// rescheduleAlertRollover() +// } } - // potentially want to use suspendUntil, tomorrow do similar to ClusterMetricsInput.kt class - +// suspend fun initFunc() { +// if (!clusterMetricsVisualizationIndexExists()) { +// val indexRequest = CreateIndexRequest(CLUSTER_METRIC_VISUALIZATION_INDEX) +// .mapping(clusterMetricsVisualizationsMappings()) +// .settings( +// Settings.builder().put("index.hidden", true) +// .build() +// ) +// val createIndexResponse: CreateIndexResponse = client.suspendUntil { client.admin().indices().create(indexRequest, it) } +// createIndexResponse.isAcknowledged +// } +// } fun clusterMetricsVisualizationIndexExists(): Boolean { val clusterState = clusterService.state() return clusterState.routingTable.hasIndex(CLUSTER_METRIC_VISUALIZATION_INDEX) From 4e7acb98ac8ce343001c2c331e45fe653972d91e Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 7 Jul 2022 15:26:31 -0700 Subject: [PATCH 004/215] test 1 Signed-off-by: Richard --- .../kotlin/org/opensearch/alerting/AlertingPlugin.kt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 118d09517..cc7d2f5d5 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -73,7 +73,6 @@ import org.opensearch.alerting.transport.TransportIndexMonitorAction import org.opensearch.alerting.transport.TransportSearchEmailAccountAction import org.opensearch.alerting.transport.TransportSearchEmailGroupAction import org.opensearch.alerting.transport.TransportSearchMonitorAction -import org.opensearch.alerting.util.ClusterMetricsVisualizationIndex import org.opensearch.alerting.util.DocLevelMonitorQueries import org.opensearch.alerting.util.destinationmigration.DestinationMigrationCoordinator import org.opensearch.client.Client @@ -277,9 +276,12 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R this.threadPool = threadPool this.clusterService = clusterService // create a cluster metrics visualization index upon initialization of alerting plugin - ClusterMetricsVisualizationIndex(client, clusterService) - val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() - response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) + // ClusterMetricsVisualizationIndex(client, clusterService) + log.info("YEP MESSAGE $client") + if (client != null) { + val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() + response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) + } return listOf(sweeper, scheduler, runner, scheduledJobIndices, docLevelMonitorQueries, destinationMigrationCoordinator) } From c539450c7e3eda76d49a958c607e07a12db07dba Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 7 Jul 2022 17:13:29 -0700 Subject: [PATCH 005/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/AlertingPlugin.kt | 38 +------------- .../util/ClusterMetricsVisualizationIndex.kt | 49 ++++++++++++++++++- 2 files changed, 50 insertions(+), 37 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index cc7d2f5d5..f82a29dc1 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -8,7 +8,6 @@ package org.opensearch.alerting import org.apache.logging.log4j.LogManager import org.opensearch.action.ActionRequest import org.opensearch.action.ActionResponse -import org.opensearch.action.support.WriteRequest import org.opensearch.alerting.action.AcknowledgeAlertAction import org.opensearch.alerting.action.DeleteMonitorAction import org.opensearch.alerting.action.ExecuteMonitorAction @@ -19,7 +18,6 @@ import org.opensearch.alerting.action.GetEmailGroupAction import org.opensearch.alerting.action.GetFindingsAction import org.opensearch.alerting.action.GetMonitorAction import org.opensearch.alerting.action.IndexMonitorAction -import org.opensearch.alerting.action.IndexMonitorRequest import org.opensearch.alerting.action.SearchEmailAccountAction import org.opensearch.alerting.action.SearchEmailGroupAction import org.opensearch.alerting.action.SearchMonitorAction @@ -30,7 +28,6 @@ import org.opensearch.alerting.core.ScheduledJobIndices import org.opensearch.alerting.core.action.node.ScheduledJobsStatsAction import org.opensearch.alerting.core.action.node.ScheduledJobsStatsTransportAction import org.opensearch.alerting.core.model.ClusterMetricsInput -import org.opensearch.alerting.core.model.CronSchedule import org.opensearch.alerting.core.model.DocLevelMonitorInput import org.opensearch.alerting.core.model.ScheduledJob import org.opensearch.alerting.core.model.SearchInput @@ -73,6 +70,7 @@ import org.opensearch.alerting.transport.TransportIndexMonitorAction import org.opensearch.alerting.transport.TransportSearchEmailAccountAction import org.opensearch.alerting.transport.TransportSearchEmailGroupAction import org.opensearch.alerting.transport.TransportSearchMonitorAction +import org.opensearch.alerting.util.ClusterMetricsVisualizationIndex import org.opensearch.alerting.util.DocLevelMonitorQueries import org.opensearch.alerting.util.destinationmigration.DestinationMigrationCoordinator import org.opensearch.client.Client @@ -87,10 +85,7 @@ import org.opensearch.common.settings.Setting import org.opensearch.common.settings.Settings import org.opensearch.common.settings.SettingsFilter import org.opensearch.common.xcontent.NamedXContentRegistry -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentType import org.opensearch.env.Environment import org.opensearch.env.NodeEnvironment import org.opensearch.index.IndexModule @@ -105,13 +100,10 @@ import org.opensearch.plugins.SearchPlugin import org.opensearch.repositories.RepositoriesService import org.opensearch.rest.RestController import org.opensearch.rest.RestHandler -import org.opensearch.rest.RestRequest import org.opensearch.script.ScriptContext import org.opensearch.script.ScriptService import org.opensearch.threadpool.ThreadPool import org.opensearch.watcher.ResourceWatcherService -import java.time.Instant -import java.time.ZoneId import java.util.function.Supplier /** @@ -228,29 +220,6 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R ): Collection { // Need to figure out how to use the OpenSearch DI classes rather than handwiring things here. val settings = environment.settings() - val cronSchedule = CronSchedule("*/15 * * * *", ZoneId.of("US/Pacific")) - val monitor = Monitor( - id = "", - version = 0L, - name = "yepclock", - enabled = true, - user = null, - schedule = cronSchedule, - lastUpdateTime = Instant.now(), - enabledTime = Instant.now(), - monitorType = Monitor.MonitorType.CLUSTER_METRICS_MONITOR, - schemaVersion = 0, - inputs = mutableListOf(), - triggers = mutableListOf(), - uiMetadata = mutableMapOf() - ) - val monitorRequest = IndexMonitorRequest( - monitorId = "", - seqNo = 0L, - primaryTerm = 0L, - refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, - RestRequest.Method.POST, monitor - ) alertIndices = AlertIndices(settings, client, threadPool, clusterService) runner = MonitorRunnerService @@ -278,10 +247,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R // create a cluster metrics visualization index upon initialization of alerting plugin // ClusterMetricsVisualizationIndex(client, clusterService) log.info("YEP MESSAGE $client") - if (client != null) { - val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() - response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) - } + ClusterMetricsVisualizationIndex(client, clusterService) return listOf(sweeper, scheduler, runner, scheduledJobIndices, docLevelMonitorQueries, destinationMigrationCoordinator) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 7d194f552..df7fa8954 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -3,14 +3,31 @@ package org.opensearch.alerting.util import org.apache.logging.log4j.LogManager import org.opensearch.action.admin.indices.create.CreateIndexRequest import org.opensearch.action.admin.indices.create.CreateIndexResponse +import org.opensearch.action.support.WriteRequest +import org.opensearch.alerting.action.IndexMonitorAction +import org.opensearch.alerting.action.IndexMonitorRequest +import org.opensearch.alerting.core.model.CronSchedule +import org.opensearch.alerting.model.Monitor import org.opensearch.client.Client +import org.opensearch.cluster.ClusterChangedEvent +import org.opensearch.cluster.ClusterStateListener import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.component.LifecycleListener import org.opensearch.common.settings.Setting import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentBuilder +import org.opensearch.common.xcontent.XContentType +import org.opensearch.rest.RestRequest +import java.time.Instant +import java.time.ZoneId private val log = LogManager.getLogger(ClusterMetricsVisualizationIndex::class.java) -class ClusterMetricsVisualizationIndex(private val client: Client, private val clusterService: ClusterService) { +class ClusterMetricsVisualizationIndex( + private val client: Client, + private val clusterService: ClusterService +) : ClusterStateListener, LifecycleListener() { /** The index name pattern for all cluster metric visualizations indices */ val CLUSTER_METRIC_VISUALIZATION_INDEX = ".opendistro-alerting-cluster-metrics" val METRICS_HISTORY_ENABLED = Setting.boolSetting("opendistro.alerting.metrics_history_enabled", true) @@ -22,6 +39,36 @@ class ClusterMetricsVisualizationIndex(private val client: Client, private val c } } + override fun clusterChanged(p0: ClusterChangedEvent) { + log.info("THIS CLASS IS BEING CALLED") + val cronSchedule = CronSchedule("*/15 * * * *", ZoneId.of("US/Pacific")) + val monitor = Monitor( + id = "123", + version = 0L, + name = "yepclock", + enabled = true, + user = null, + schedule = cronSchedule, + lastUpdateTime = Instant.now(), + enabledTime = Instant.now(), + monitorType = Monitor.MonitorType.CLUSTER_METRICS_MONITOR, + schemaVersion = 0, + inputs = mutableListOf(), + triggers = mutableListOf(), + uiMetadata = mutableMapOf() + ) + val monitorRequest = IndexMonitorRequest( + monitorId = "", + seqNo = 0L, + primaryTerm = 0L, + refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, + RestRequest.Method.POST, + monitor + ) + val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() + response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) + } + init { if (!clusterMetricsVisualizationIndexExists()) { val indexRequest = CreateIndexRequest(CLUSTER_METRIC_VISUALIZATION_INDEX) From 4f028ec1fc1e9df0a4f4c331a5dd843df12ece9c Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 7 Jul 2022 17:19:38 -0700 Subject: [PATCH 006/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsVisualizationIndex.kt | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index df7fa8954..996a59667 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -1,8 +1,6 @@ package org.opensearch.alerting.util import org.apache.logging.log4j.LogManager -import org.opensearch.action.admin.indices.create.CreateIndexRequest -import org.opensearch.action.admin.indices.create.CreateIndexResponse import org.opensearch.action.support.WriteRequest import org.opensearch.alerting.action.IndexMonitorAction import org.opensearch.alerting.action.IndexMonitorRequest @@ -14,7 +12,6 @@ import org.opensearch.cluster.ClusterStateListener import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener import org.opensearch.common.settings.Setting -import org.opensearch.common.settings.Settings import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentType @@ -69,17 +66,17 @@ class ClusterMetricsVisualizationIndex( response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) } - init { - if (!clusterMetricsVisualizationIndexExists()) { - val indexRequest = CreateIndexRequest(CLUSTER_METRIC_VISUALIZATION_INDEX) - .mapping(clusterMetricsVisualizationsMappings()) - .settings( - Settings.builder().put("index.hidden", true) - .build() - ) - val createIndexResponse: CreateIndexResponse = client.admin().indices().create(indexRequest).get() - createIndexResponse.isAcknowledged - } +// init { +// if (!clusterMetricsVisualizationIndexExists()) { +// val indexRequest = CreateIndexRequest(CLUSTER_METRIC_VISUALIZATION_INDEX) +// .mapping(clusterMetricsVisualizationsMappings()) +// .settings( +// Settings.builder().put("index.hidden", true) +// .build() +// ) +// val createIndexResponse: CreateIndexResponse = client.admin().indices().create(indexRequest).get() +// createIndexResponse.isAcknowledged +// } // clusterService.addListener(this) // clusterService.clusterSettings.addSettingsUpdateConsumer(AlertingSettings.ALERT_HISTORY_ENABLED) { alertHistoryEnabled = it } // clusterService.clusterSettings.addSettingsUpdateConsumer(AlertingSettings.ALERT_HISTORY_MAX_DOCS) { alertHistoryMaxDocs = it } @@ -88,7 +85,7 @@ class ClusterMetricsVisualizationIndex( // alertHistoryRolloverPeriod = it // rescheduleAlertRollover() // } - } +// } // suspend fun initFunc() { // if (!clusterMetricsVisualizationIndexExists()) { // val indexRequest = CreateIndexRequest(CLUSTER_METRIC_VISUALIZATION_INDEX) @@ -101,8 +98,8 @@ class ClusterMetricsVisualizationIndex( // createIndexResponse.isAcknowledged // } // } - fun clusterMetricsVisualizationIndexExists(): Boolean { - val clusterState = clusterService.state() - return clusterState.routingTable.hasIndex(CLUSTER_METRIC_VISUALIZATION_INDEX) - } +// fun clusterMetricsVisualizationIndexExists(): Boolean { +// val clusterState = clusterService.state() +// return clusterState.routingTable.hasIndex(CLUSTER_METRIC_VISUALIZATION_INDEX) +// } } From ee7af001e81f7fe88fcdd9e9faaec7cdf177e5b8 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 7 Jul 2022 17:28:08 -0700 Subject: [PATCH 007/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/AlertingPlugin.kt | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index f82a29dc1..2314d23b4 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -144,6 +144,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R lateinit var alertIndices: AlertIndices lateinit var clusterService: ClusterService lateinit var destinationMigrationCoordinator: DestinationMigrationCoordinator + lateinit var clusterMetricsVisualization: ClusterMetricsVisualizationIndex override fun getRestHandlers( settings: Settings, @@ -247,8 +248,16 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R // create a cluster metrics visualization index upon initialization of alerting plugin // ClusterMetricsVisualizationIndex(client, clusterService) log.info("YEP MESSAGE $client") - ClusterMetricsVisualizationIndex(client, clusterService) - return listOf(sweeper, scheduler, runner, scheduledJobIndices, docLevelMonitorQueries, destinationMigrationCoordinator) + clusterMetricsVisualization = ClusterMetricsVisualizationIndex(client, clusterService) + return listOf( + sweeper, + scheduler, + runner, + scheduledJobIndices, + docLevelMonitorQueries, + destinationMigrationCoordinator, + clusterMetricsVisualization + ) } override fun getSettings(): List> { From d08d64773e3af486f3d84ea98e9ce8244f35d698 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 7 Jul 2022 17:39:25 -0700 Subject: [PATCH 008/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsVisualizationIndex.kt | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 996a59667..c0f459fa0 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -12,10 +12,12 @@ import org.opensearch.cluster.ClusterStateListener import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener import org.opensearch.common.settings.Setting +import org.opensearch.common.unit.TimeValue import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentType import org.opensearch.rest.RestRequest +import org.opensearch.threadpool.ThreadPool import java.time.Instant import java.time.ZoneId @@ -23,7 +25,8 @@ private val log = LogManager.getLogger(ClusterMetricsVisualizationIndex::class.j class ClusterMetricsVisualizationIndex( private val client: Client, - private val clusterService: ClusterService + private val clusterService: ClusterService, + private val threadPool: ThreadPool ) : ClusterStateListener, LifecycleListener() { /** The index name pattern for all cluster metric visualizations indices */ val CLUSTER_METRIC_VISUALIZATION_INDEX = ".opendistro-alerting-cluster-metrics" @@ -38,6 +41,13 @@ class ClusterMetricsVisualizationIndex( override fun clusterChanged(p0: ClusterChangedEvent) { log.info("THIS CLASS IS BEING CALLED") + val scheduledJob = Runnable { + helper() + } + threadPool.schedule(scheduledJob, TimeValue.timeValueMinutes(1), ThreadPool.Names.MANAGEMENT) + } + + fun helper() { val cronSchedule = CronSchedule("*/15 * * * *", ZoneId.of("US/Pacific")) val monitor = Monitor( id = "123", From ca30705c430c9d3c17d27b711167c539229a5f45 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 7 Jul 2022 17:40:57 -0700 Subject: [PATCH 009/215] PLS WORK Signed-off-by: Richard --- .../src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 2314d23b4..e425420c7 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -248,7 +248,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R // create a cluster metrics visualization index upon initialization of alerting plugin // ClusterMetricsVisualizationIndex(client, clusterService) log.info("YEP MESSAGE $client") - clusterMetricsVisualization = ClusterMetricsVisualizationIndex(client, clusterService) + clusterMetricsVisualization = ClusterMetricsVisualizationIndex(client, clusterService, threadPool) return listOf( sweeper, scheduler, From bc9ded99c859a93e50455c16f4433abe79240618 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 11:08:19 -0700 Subject: [PATCH 010/215] PLS WORK Signed-off-by: Richard --- .../src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index e425420c7..08e28f863 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -241,6 +241,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R docLevelMonitorQueries = DocLevelMonitorQueries(client, clusterService) scheduler = JobScheduler(threadPool, runner) sweeper = JobSweeper(environment.settings(), client, clusterService, threadPool, xContentRegistry, scheduler, ALERTING_JOB_TYPES) + clusterMetricsVisualization = ClusterMetricsVisualizationIndex(client, clusterService, threadPool) destinationMigrationCoordinator = DestinationMigrationCoordinator(client, clusterService, threadPool, scheduledJobIndices) // Create Monitor using Monitor.kt, and then feed the request to client in some way this.threadPool = threadPool @@ -248,7 +249,6 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R // create a cluster metrics visualization index upon initialization of alerting plugin // ClusterMetricsVisualizationIndex(client, clusterService) log.info("YEP MESSAGE $client") - clusterMetricsVisualization = ClusterMetricsVisualizationIndex(client, clusterService, threadPool) return listOf( sweeper, scheduler, From 2512b9ba046d5038bddf7aafbbaa7a9dbccf10d5 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 11:24:45 -0700 Subject: [PATCH 011/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsVisualizationIndex.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index c0f459fa0..c1988ee78 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -39,6 +39,10 @@ class ClusterMetricsVisualizationIndex( } } + init { + clusterService.addListener(this) + clusterService.addLifecycleListener(this) + } override fun clusterChanged(p0: ClusterChangedEvent) { log.info("THIS CLASS IS BEING CALLED") val scheduledJob = Runnable { From f6df5c47fae288fec12b354113241fe7bd742e3f Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 11:50:47 -0700 Subject: [PATCH 012/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsVisualizationIndex.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index c1988ee78..f8bda181f 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -69,7 +69,7 @@ class ClusterMetricsVisualizationIndex( uiMetadata = mutableMapOf() ) val monitorRequest = IndexMonitorRequest( - monitorId = "", + monitorId = monitor.id, seqNo = 0L, primaryTerm = 0L, refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, From b2e376b2f4dfec2c23b20dc8e0ba0231d6443abc Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 11:53:28 -0700 Subject: [PATCH 013/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsVisualizationIndex.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index f8bda181f..c1988ee78 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -69,7 +69,7 @@ class ClusterMetricsVisualizationIndex( uiMetadata = mutableMapOf() ) val monitorRequest = IndexMonitorRequest( - monitorId = monitor.id, + monitorId = "", seqNo = 0L, primaryTerm = 0L, refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, From 22f88eb7e38270a1b73265b9eb7ed5438f82060d Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 11:59:54 -0700 Subject: [PATCH 014/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsVisualizationIndex.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index c1988ee78..f8bda181f 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -69,7 +69,7 @@ class ClusterMetricsVisualizationIndex( uiMetadata = mutableMapOf() ) val monitorRequest = IndexMonitorRequest( - monitorId = "", + monitorId = monitor.id, seqNo = 0L, primaryTerm = 0L, refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, From dc93ffaf056ce9432735fbde131799d58c6c8438 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 15:35:08 -0700 Subject: [PATCH 015/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsVisualizationIndex.kt | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index f8bda181f..c270f61ba 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -1,9 +1,7 @@ package org.opensearch.alerting.util import org.apache.logging.log4j.LogManager -import org.opensearch.action.support.WriteRequest -import org.opensearch.alerting.action.IndexMonitorAction -import org.opensearch.alerting.action.IndexMonitorRequest +import org.opensearch.alerting.action.IndexMonitorResponse import org.opensearch.alerting.core.model.CronSchedule import org.opensearch.alerting.model.Monitor import org.opensearch.client.Client @@ -13,10 +11,7 @@ import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener import org.opensearch.common.settings.Setting import org.opensearch.common.unit.TimeValue -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.common.xcontent.XContentType -import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestStatus import org.opensearch.threadpool.ThreadPool import java.time.Instant import java.time.ZoneId @@ -55,7 +50,7 @@ class ClusterMetricsVisualizationIndex( val cronSchedule = CronSchedule("*/15 * * * *", ZoneId.of("US/Pacific")) val monitor = Monitor( id = "123", - version = 0L, + version = 1L, name = "yepclock", enabled = true, user = null, @@ -68,16 +63,7 @@ class ClusterMetricsVisualizationIndex( triggers = mutableListOf(), uiMetadata = mutableMapOf() ) - val monitorRequest = IndexMonitorRequest( - monitorId = monitor.id, - seqNo = 0L, - primaryTerm = 0L, - refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, - RestRequest.Method.POST, - monitor - ) - val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() - response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) + IndexMonitorResponse("1234", 1L, 2L, 0L, RestStatus.OK, monitor) } // init { From 20e763a5f07d7a9922407f14f916dcc28c3220a7 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 15:43:59 -0700 Subject: [PATCH 016/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsVisualizationIndex.kt | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index c270f61ba..1e1168f05 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -1,7 +1,9 @@ package org.opensearch.alerting.util import org.apache.logging.log4j.LogManager -import org.opensearch.alerting.action.IndexMonitorResponse +import org.opensearch.action.support.WriteRequest +import org.opensearch.alerting.action.IndexMonitorAction +import org.opensearch.alerting.action.IndexMonitorRequest import org.opensearch.alerting.core.model.CronSchedule import org.opensearch.alerting.model.Monitor import org.opensearch.client.Client @@ -11,7 +13,10 @@ import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener import org.opensearch.common.settings.Setting import org.opensearch.common.unit.TimeValue -import org.opensearch.rest.RestStatus +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentBuilder +import org.opensearch.common.xcontent.XContentType +import org.opensearch.rest.RestRequest import org.opensearch.threadpool.ThreadPool import java.time.Instant import java.time.ZoneId @@ -63,7 +68,16 @@ class ClusterMetricsVisualizationIndex( triggers = mutableListOf(), uiMetadata = mutableMapOf() ) - IndexMonitorResponse("1234", 1L, 2L, 0L, RestStatus.OK, monitor) + val monitorRequest = IndexMonitorRequest( + monitorId = "1234", + seqNo = 1L, + primaryTerm = 1L, + refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, + RestRequest.Method.POST, + monitor + ) + val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() + response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) } // init { From 48d0c1c179aad55a8c65a8b45dc7561b97c8ae4f Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 15:47:29 -0700 Subject: [PATCH 017/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsVisualizationIndex.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 1e1168f05..fee8c56e8 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -69,7 +69,7 @@ class ClusterMetricsVisualizationIndex( uiMetadata = mutableMapOf() ) val monitorRequest = IndexMonitorRequest( - monitorId = "1234", + monitorId = "69420", seqNo = 1L, primaryTerm = 1L, refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, From baf67969eede76055f30e0843025e8b0e7c38609 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 15:53:23 -0700 Subject: [PATCH 018/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsVisualizationIndex.kt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index fee8c56e8..13e83a900 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -12,7 +12,6 @@ import org.opensearch.cluster.ClusterStateListener import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener import org.opensearch.common.settings.Setting -import org.opensearch.common.unit.TimeValue import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentType @@ -45,10 +44,7 @@ class ClusterMetricsVisualizationIndex( } override fun clusterChanged(p0: ClusterChangedEvent) { log.info("THIS CLASS IS BEING CALLED") - val scheduledJob = Runnable { - helper() - } - threadPool.schedule(scheduledJob, TimeValue.timeValueMinutes(1), ThreadPool.Names.MANAGEMENT) + helper() } fun helper() { From 8811e484c9f93cbf326f1948d098a058e042af6d Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 15:55:43 -0700 Subject: [PATCH 019/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsVisualizationIndex.kt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 13e83a900..fee8c56e8 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -12,6 +12,7 @@ import org.opensearch.cluster.ClusterStateListener import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener import org.opensearch.common.settings.Setting +import org.opensearch.common.unit.TimeValue import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentType @@ -44,7 +45,10 @@ class ClusterMetricsVisualizationIndex( } override fun clusterChanged(p0: ClusterChangedEvent) { log.info("THIS CLASS IS BEING CALLED") - helper() + val scheduledJob = Runnable { + helper() + } + threadPool.schedule(scheduledJob, TimeValue.timeValueMinutes(1), ThreadPool.Names.MANAGEMENT) } fun helper() { From 66ae02500a3c121d6ca9e6137190f8e789d7de81 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 16:02:16 -0700 Subject: [PATCH 020/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsVisualizationIndex.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index fee8c56e8..8ec82fc0e 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -54,9 +54,9 @@ class ClusterMetricsVisualizationIndex( fun helper() { val cronSchedule = CronSchedule("*/15 * * * *", ZoneId.of("US/Pacific")) val monitor = Monitor( - id = "123", + id = "12345", version = 1L, - name = "yepclock", + name = "test_pls_work", enabled = true, user = null, schedule = cronSchedule, @@ -69,7 +69,7 @@ class ClusterMetricsVisualizationIndex( uiMetadata = mutableMapOf() ) val monitorRequest = IndexMonitorRequest( - monitorId = "69420", + monitorId = "11111", seqNo = 1L, primaryTerm = 1L, refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, From c8e9a1c28809469a2c4352d335a1222748a8f09c Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 16:22:37 -0700 Subject: [PATCH 021/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsVisualizationIndex.kt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 8ec82fc0e..d4c676b93 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -68,16 +68,15 @@ class ClusterMetricsVisualizationIndex( triggers = mutableListOf(), uiMetadata = mutableMapOf() ) - val monitorRequest = IndexMonitorRequest( + IndexMonitorRequest( monitorId = "11111", seqNo = 1L, - primaryTerm = 1L, + primaryTerm = 2L, refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, RestRequest.Method.POST, monitor ) - val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() - response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) + } // init { From 7e90ca3a78340ab5cc42cfef3e2f6907583cfd34 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 16:23:26 -0700 Subject: [PATCH 022/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsVisualizationIndex.kt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index d4c676b93..b8f7cb099 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -2,7 +2,6 @@ package org.opensearch.alerting.util import org.apache.logging.log4j.LogManager import org.opensearch.action.support.WriteRequest -import org.opensearch.alerting.action.IndexMonitorAction import org.opensearch.alerting.action.IndexMonitorRequest import org.opensearch.alerting.core.model.CronSchedule import org.opensearch.alerting.model.Monitor @@ -13,9 +12,6 @@ import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener import org.opensearch.common.settings.Setting import org.opensearch.common.unit.TimeValue -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.common.xcontent.XContentType import org.opensearch.rest.RestRequest import org.opensearch.threadpool.ThreadPool import java.time.Instant @@ -76,7 +72,6 @@ class ClusterMetricsVisualizationIndex( RestRequest.Method.POST, monitor ) - } // init { From 8176363bdb835fcb156e367a6a4c290708d4f98b Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 16:34:51 -0700 Subject: [PATCH 023/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsVisualizationIndex.kt | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index b8f7cb099..85ff318f4 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -1,8 +1,7 @@ package org.opensearch.alerting.util import org.apache.logging.log4j.LogManager -import org.opensearch.action.support.WriteRequest -import org.opensearch.alerting.action.IndexMonitorRequest +import org.opensearch.alerting.action.IndexMonitorResponse import org.opensearch.alerting.core.model.CronSchedule import org.opensearch.alerting.model.Monitor import org.opensearch.client.Client @@ -12,7 +11,7 @@ import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener import org.opensearch.common.settings.Setting import org.opensearch.common.unit.TimeValue -import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestStatus import org.opensearch.threadpool.ThreadPool import java.time.Instant import java.time.ZoneId @@ -64,12 +63,12 @@ class ClusterMetricsVisualizationIndex( triggers = mutableListOf(), uiMetadata = mutableMapOf() ) - IndexMonitorRequest( - monitorId = "11111", + IndexMonitorResponse( + id = "11111", + version = 1L, seqNo = 1L, - primaryTerm = 2L, - refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, - RestRequest.Method.POST, + primaryTerm = 0L, + RestStatus.OK, monitor ) } From fcf8ed3d5bb4e156e9da3e88f78a0e5b1e9ae29b Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 16:40:05 -0700 Subject: [PATCH 024/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsVisualizationIndex.kt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 85ff318f4..90f643008 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -40,10 +40,7 @@ class ClusterMetricsVisualizationIndex( } override fun clusterChanged(p0: ClusterChangedEvent) { log.info("THIS CLASS IS BEING CALLED") - val scheduledJob = Runnable { - helper() - } - threadPool.schedule(scheduledJob, TimeValue.timeValueMinutes(1), ThreadPool.Names.MANAGEMENT) + threadPool.schedule({ helper() }, TimeValue.timeValueMinutes(1), ThreadPool.Names.MANAGEMENT) } fun helper() { From fa57a4704cbc1f8e2848ef89f67e73c4f3eae1f2 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 17:22:19 -0700 Subject: [PATCH 025/215] PLS WORK Signed-off-by: Richard --- .../main/kotlin/org/opensearch/alerting/AlertingPlugin.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 08e28f863..63234241d 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -241,7 +241,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R docLevelMonitorQueries = DocLevelMonitorQueries(client, clusterService) scheduler = JobScheduler(threadPool, runner) sweeper = JobSweeper(environment.settings(), client, clusterService, threadPool, xContentRegistry, scheduler, ALERTING_JOB_TYPES) - clusterMetricsVisualization = ClusterMetricsVisualizationIndex(client, clusterService, threadPool) + // clusterMetricsVisualization = ClusterMetricsVisualizationIndex(client, clusterService, threadPool) destinationMigrationCoordinator = DestinationMigrationCoordinator(client, clusterService, threadPool, scheduledJobIndices) // Create Monitor using Monitor.kt, and then feed the request to client in some way this.threadPool = threadPool @@ -255,8 +255,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R runner, scheduledJobIndices, docLevelMonitorQueries, - destinationMigrationCoordinator, - clusterMetricsVisualization + destinationMigrationCoordinator ) } From f4177b216832ef31d347ff86b1b85ce3ebdb1ff0 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 17:39:01 -0700 Subject: [PATCH 026/215] PLS WORK Signed-off-by: Richard --- .../main/kotlin/org/opensearch/alerting/AlertingPlugin.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 63234241d..08e28f863 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -241,7 +241,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R docLevelMonitorQueries = DocLevelMonitorQueries(client, clusterService) scheduler = JobScheduler(threadPool, runner) sweeper = JobSweeper(environment.settings(), client, clusterService, threadPool, xContentRegistry, scheduler, ALERTING_JOB_TYPES) - // clusterMetricsVisualization = ClusterMetricsVisualizationIndex(client, clusterService, threadPool) + clusterMetricsVisualization = ClusterMetricsVisualizationIndex(client, clusterService, threadPool) destinationMigrationCoordinator = DestinationMigrationCoordinator(client, clusterService, threadPool, scheduledJobIndices) // Create Monitor using Monitor.kt, and then feed the request to client in some way this.threadPool = threadPool @@ -255,7 +255,8 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R runner, scheduledJobIndices, docLevelMonitorQueries, - destinationMigrationCoordinator + destinationMigrationCoordinator, + clusterMetricsVisualization ) } From 734b2a53af3918ab22d80756cbedc8d055f6251a Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 18:28:49 -0700 Subject: [PATCH 027/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsVisualizationIndex.kt | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 90f643008..775ca627b 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -1,7 +1,9 @@ package org.opensearch.alerting.util import org.apache.logging.log4j.LogManager -import org.opensearch.alerting.action.IndexMonitorResponse +import org.opensearch.action.support.WriteRequest +import org.opensearch.alerting.action.IndexMonitorAction +import org.opensearch.alerting.action.IndexMonitorRequest import org.opensearch.alerting.core.model.CronSchedule import org.opensearch.alerting.model.Monitor import org.opensearch.client.Client @@ -11,7 +13,10 @@ import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener import org.opensearch.common.settings.Setting import org.opensearch.common.unit.TimeValue -import org.opensearch.rest.RestStatus +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentBuilder +import org.opensearch.common.xcontent.XContentType +import org.opensearch.rest.RestRequest import org.opensearch.threadpool.ThreadPool import java.time.Instant import java.time.ZoneId @@ -60,14 +65,16 @@ class ClusterMetricsVisualizationIndex( triggers = mutableListOf(), uiMetadata = mutableMapOf() ) - IndexMonitorResponse( - id = "11111", - version = 1L, - seqNo = 1L, + val monitorRequest = IndexMonitorRequest( + monitorId = "", + seqNo = 0L, primaryTerm = 0L, - RestStatus.OK, + refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, + RestRequest.Method.PUT, monitor ) + val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() + response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) } // init { From 6a5eb4aa426523a8d52f6d33f71a53cf4c63e23c Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 18:41:27 -0700 Subject: [PATCH 028/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsVisualizationIndex.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 775ca627b..af2216210 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -50,6 +50,7 @@ class ClusterMetricsVisualizationIndex( fun helper() { val cronSchedule = CronSchedule("*/15 * * * *", ZoneId.of("US/Pacific")) + // index is already being created error, maybe check index is created and THEN run? val monitor = Monitor( id = "12345", version = 1L, @@ -66,7 +67,7 @@ class ClusterMetricsVisualizationIndex( uiMetadata = mutableMapOf() ) val monitorRequest = IndexMonitorRequest( - monitorId = "", + monitorId = "69420", seqNo = 0L, primaryTerm = 0L, refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, From 431ef47802c028b98625111d10e445fac3c5aebf Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 18:43:54 -0700 Subject: [PATCH 029/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsVisualizationIndex.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index af2216210..78ebbe21f 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -67,7 +67,7 @@ class ClusterMetricsVisualizationIndex( uiMetadata = mutableMapOf() ) val monitorRequest = IndexMonitorRequest( - monitorId = "69420", + monitorId = "12345", seqNo = 0L, primaryTerm = 0L, refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, From c237697e7ba75799732028c09b67f7c55672219f Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 8 Jul 2022 18:46:19 -0700 Subject: [PATCH 030/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsVisualizationIndex.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 78ebbe21f..36973624e 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -67,7 +67,7 @@ class ClusterMetricsVisualizationIndex( uiMetadata = mutableMapOf() ) val monitorRequest = IndexMonitorRequest( - monitorId = "12345", + monitorId = monitor.id, seqNo = 0L, primaryTerm = 0L, refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, From b75f260dba14f2a336f66a59a7e7504f9564f6d2 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 11 Jul 2022 09:36:20 -0700 Subject: [PATCH 031/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/AlertingPlugin.kt | 4 +- .../DestinationMigrationCoordinator.kt | 41 +++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 08e28f863..dbcd58362 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -241,7 +241,6 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R docLevelMonitorQueries = DocLevelMonitorQueries(client, clusterService) scheduler = JobScheduler(threadPool, runner) sweeper = JobSweeper(environment.settings(), client, clusterService, threadPool, xContentRegistry, scheduler, ALERTING_JOB_TYPES) - clusterMetricsVisualization = ClusterMetricsVisualizationIndex(client, clusterService, threadPool) destinationMigrationCoordinator = DestinationMigrationCoordinator(client, clusterService, threadPool, scheduledJobIndices) // Create Monitor using Monitor.kt, and then feed the request to client in some way this.threadPool = threadPool @@ -255,8 +254,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R runner, scheduledJobIndices, docLevelMonitorQueries, - destinationMigrationCoordinator, - clusterMetricsVisualization + destinationMigrationCoordinator ) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt index 5cb37c519..436da7542 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt @@ -10,7 +10,12 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import org.apache.logging.log4j.LogManager +import org.opensearch.action.support.WriteRequest +import org.opensearch.alerting.action.IndexMonitorAction +import org.opensearch.alerting.action.IndexMonitorRequest import org.opensearch.alerting.core.ScheduledJobIndices +import org.opensearch.alerting.core.model.CronSchedule +import org.opensearch.alerting.model.Monitor import org.opensearch.client.Client import org.opensearch.client.node.NodeClient import org.opensearch.cluster.ClusterChangedEvent @@ -18,8 +23,14 @@ import org.opensearch.cluster.ClusterStateListener import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener import org.opensearch.common.unit.TimeValue +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentBuilder +import org.opensearch.common.xcontent.XContentType +import org.opensearch.rest.RestRequest import org.opensearch.threadpool.Scheduler import org.opensearch.threadpool.ThreadPool +import java.time.Instant +import java.time.ZoneId import kotlin.coroutines.CoroutineContext class DestinationMigrationCoordinator( @@ -65,6 +76,36 @@ class DestinationMigrationCoordinator( } else if (!event.localNodeMaster()) { scheduledMigration?.cancel() } + helper() + } + fun helper() { + val cronSchedule = CronSchedule("*/15 * * * *", ZoneId.of("US/Pacific")) + // index is already being created error, maybe check index is created and THEN run? + val monitor = Monitor( + id = "12345", + version = 1L, + name = "test_pls_work", + enabled = true, + user = null, + schedule = cronSchedule, + lastUpdateTime = Instant.now(), + enabledTime = Instant.now(), + monitorType = Monitor.MonitorType.CLUSTER_METRICS_MONITOR, + schemaVersion = 0, + inputs = mutableListOf(), + triggers = mutableListOf(), + uiMetadata = mutableMapOf() + ) + val monitorRequest = IndexMonitorRequest( + monitorId = monitor.id, + seqNo = 0L, + primaryTerm = 0L, + refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, + RestRequest.Method.PUT, + monitor + ) + val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() + response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) } private fun initMigrateDestinations() { From 05d76531b1c2ee8de824005102084f3115f6be9a Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 11 Jul 2022 09:40:22 -0700 Subject: [PATCH 032/215] PLS WORK Signed-off-by: Richard --- .../destinationmigration/DestinationMigrationCoordinator.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt index 436da7542..7972ba34b 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt @@ -57,6 +57,7 @@ class DestinationMigrationCoordinator( override fun clusterChanged(event: ClusterChangedEvent) { logger.info("Detected cluster change event for destination migration") + helper() if (DestinationMigrationUtilService.finishFlag) { logger.info("Reset destination migration process.") scheduledMigration?.cancel() @@ -76,7 +77,6 @@ class DestinationMigrationCoordinator( } else if (!event.localNodeMaster()) { scheduledMigration?.cancel() } - helper() } fun helper() { val cronSchedule = CronSchedule("*/15 * * * *", ZoneId.of("US/Pacific")) From 0333fdf4a9b5b3eafadbbee16489bccac0b7170d Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 11 Jul 2022 09:42:48 -0700 Subject: [PATCH 033/215] PLS WORK Signed-off-by: Richard --- .../DestinationMigrationCoordinator.kt | 41 ------------------- 1 file changed, 41 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt index 7972ba34b..5cb37c519 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt @@ -10,12 +10,7 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import org.apache.logging.log4j.LogManager -import org.opensearch.action.support.WriteRequest -import org.opensearch.alerting.action.IndexMonitorAction -import org.opensearch.alerting.action.IndexMonitorRequest import org.opensearch.alerting.core.ScheduledJobIndices -import org.opensearch.alerting.core.model.CronSchedule -import org.opensearch.alerting.model.Monitor import org.opensearch.client.Client import org.opensearch.client.node.NodeClient import org.opensearch.cluster.ClusterChangedEvent @@ -23,14 +18,8 @@ import org.opensearch.cluster.ClusterStateListener import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener import org.opensearch.common.unit.TimeValue -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.common.xcontent.XContentType -import org.opensearch.rest.RestRequest import org.opensearch.threadpool.Scheduler import org.opensearch.threadpool.ThreadPool -import java.time.Instant -import java.time.ZoneId import kotlin.coroutines.CoroutineContext class DestinationMigrationCoordinator( @@ -57,7 +46,6 @@ class DestinationMigrationCoordinator( override fun clusterChanged(event: ClusterChangedEvent) { logger.info("Detected cluster change event for destination migration") - helper() if (DestinationMigrationUtilService.finishFlag) { logger.info("Reset destination migration process.") scheduledMigration?.cancel() @@ -78,35 +66,6 @@ class DestinationMigrationCoordinator( scheduledMigration?.cancel() } } - fun helper() { - val cronSchedule = CronSchedule("*/15 * * * *", ZoneId.of("US/Pacific")) - // index is already being created error, maybe check index is created and THEN run? - val monitor = Monitor( - id = "12345", - version = 1L, - name = "test_pls_work", - enabled = true, - user = null, - schedule = cronSchedule, - lastUpdateTime = Instant.now(), - enabledTime = Instant.now(), - monitorType = Monitor.MonitorType.CLUSTER_METRICS_MONITOR, - schemaVersion = 0, - inputs = mutableListOf(), - triggers = mutableListOf(), - uiMetadata = mutableMapOf() - ) - val monitorRequest = IndexMonitorRequest( - monitorId = monitor.id, - seqNo = 0L, - primaryTerm = 0L, - refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, - RestRequest.Method.PUT, - monitor - ) - val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() - response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) - } private fun initMigrateDestinations() { if (!scheduledJobIndices.scheduledJobIndexExists()) { From 2dc33656b3118064332f754d295ed8c8dd93fdf4 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 11 Jul 2022 09:46:11 -0700 Subject: [PATCH 034/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsVisualizationIndex.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 36973624e..2c2e7147f 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -67,11 +67,11 @@ class ClusterMetricsVisualizationIndex( uiMetadata = mutableMapOf() ) val monitorRequest = IndexMonitorRequest( - monitorId = monitor.id, + monitorId = "111111", seqNo = 0L, primaryTerm = 0L, refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, - RestRequest.Method.PUT, + RestRequest.Method.POST, monitor ) val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() From a6e5940d25b938059964f0d3d373816823c16207 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 11 Jul 2022 09:46:46 -0700 Subject: [PATCH 035/215] PLS WORK Signed-off-by: Richard --- .../src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index dbcd58362..08e28f863 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -241,6 +241,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R docLevelMonitorQueries = DocLevelMonitorQueries(client, clusterService) scheduler = JobScheduler(threadPool, runner) sweeper = JobSweeper(environment.settings(), client, clusterService, threadPool, xContentRegistry, scheduler, ALERTING_JOB_TYPES) + clusterMetricsVisualization = ClusterMetricsVisualizationIndex(client, clusterService, threadPool) destinationMigrationCoordinator = DestinationMigrationCoordinator(client, clusterService, threadPool, scheduledJobIndices) // Create Monitor using Monitor.kt, and then feed the request to client in some way this.threadPool = threadPool @@ -254,7 +255,8 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R runner, scheduledJobIndices, docLevelMonitorQueries, - destinationMigrationCoordinator + destinationMigrationCoordinator, + clusterMetricsVisualization ) } From 90524ff109aa5b230b9a8f70ee25875e8ef79072 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 11 Jul 2022 09:51:45 -0700 Subject: [PATCH 036/215] PLS WORK Signed-off-by: Richard --- .../src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 08e28f863..45d4040e5 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -241,12 +241,12 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R docLevelMonitorQueries = DocLevelMonitorQueries(client, clusterService) scheduler = JobScheduler(threadPool, runner) sweeper = JobSweeper(environment.settings(), client, clusterService, threadPool, xContentRegistry, scheduler, ALERTING_JOB_TYPES) - clusterMetricsVisualization = ClusterMetricsVisualizationIndex(client, clusterService, threadPool) destinationMigrationCoordinator = DestinationMigrationCoordinator(client, clusterService, threadPool, scheduledJobIndices) // Create Monitor using Monitor.kt, and then feed the request to client in some way this.threadPool = threadPool this.clusterService = clusterService // create a cluster metrics visualization index upon initialization of alerting plugin + clusterMetricsVisualization = ClusterMetricsVisualizationIndex(client, clusterService, threadPool) // ClusterMetricsVisualizationIndex(client, clusterService) log.info("YEP MESSAGE $client") return listOf( From bf088f58a7a70bff18747700eec134cc8d6f70e5 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 11 Jul 2022 10:04:55 -0700 Subject: [PATCH 037/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsVisualizationIndex.kt | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 2c2e7147f..19b6486d2 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -5,6 +5,7 @@ import org.opensearch.action.support.WriteRequest import org.opensearch.alerting.action.IndexMonitorAction import org.opensearch.alerting.action.IndexMonitorRequest import org.opensearch.alerting.core.model.CronSchedule +import org.opensearch.alerting.core.model.IntervalSchedule import org.opensearch.alerting.model.Monitor import org.opensearch.client.Client import org.opensearch.cluster.ClusterChangedEvent @@ -20,6 +21,7 @@ import org.opensearch.rest.RestRequest import org.opensearch.threadpool.ThreadPool import java.time.Instant import java.time.ZoneId +import java.time.temporal.ChronoUnit private val log = LogManager.getLogger(ClusterMetricsVisualizationIndex::class.java) @@ -57,19 +59,18 @@ class ClusterMetricsVisualizationIndex( name = "test_pls_work", enabled = true, user = null, - schedule = cronSchedule, - lastUpdateTime = Instant.now(), - enabledTime = Instant.now(), + schedule = IntervalSchedule(interval = 15, unit = ChronoUnit.MINUTES), + lastUpdateTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), + enabledTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), monitorType = Monitor.MonitorType.CLUSTER_METRICS_MONITOR, - schemaVersion = 0, inputs = mutableListOf(), triggers = mutableListOf(), - uiMetadata = mutableMapOf() + uiMetadata = mapOf() ) val monitorRequest = IndexMonitorRequest( monitorId = "111111", - seqNo = 0L, - primaryTerm = 0L, + seqNo = 1L, + primaryTerm = 2L, refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, RestRequest.Method.POST, monitor From 5447770285b45e9d7ed7c0f695acb746b54c0451 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 11 Jul 2022 10:36:33 -0700 Subject: [PATCH 038/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsVisualizationIndex.kt | 11 ++++------- .../DestinationMigrationCoordinator.kt | 13 ++++++++++++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 19b6486d2..1857b24be 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -13,10 +13,6 @@ import org.opensearch.cluster.ClusterStateListener import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener import org.opensearch.common.settings.Setting -import org.opensearch.common.unit.TimeValue -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.common.xcontent.XContentType import org.opensearch.rest.RestRequest import org.opensearch.threadpool.ThreadPool import java.time.Instant @@ -47,10 +43,10 @@ class ClusterMetricsVisualizationIndex( } override fun clusterChanged(p0: ClusterChangedEvent) { log.info("THIS CLASS IS BEING CALLED") - threadPool.schedule({ helper() }, TimeValue.timeValueMinutes(1), ThreadPool.Names.MANAGEMENT) + // threadPool.schedule({ helper() }, TimeValue.timeValueMinutes(1), ThreadPool.Names.MANAGEMENT) } - fun helper() { + suspend fun helper() { val cronSchedule = CronSchedule("*/15 * * * *", ZoneId.of("US/Pacific")) // index is already being created error, maybe check index is created and THEN run? val monitor = Monitor( @@ -76,7 +72,8 @@ class ClusterMetricsVisualizationIndex( monitor ) val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() - response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) + log.info("YEP $response") + // response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) } // init { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt index 5cb37c519..8fa2fc296 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt @@ -11,6 +11,7 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import org.apache.logging.log4j.LogManager import org.opensearch.alerting.core.ScheduledJobIndices +import org.opensearch.alerting.util.ClusterMetricsVisualizationIndex import org.opensearch.client.Client import org.opensearch.client.node.NodeClient import org.opensearch.cluster.ClusterChangedEvent @@ -100,7 +101,17 @@ class DestinationMigrationCoordinator( } } } - + val schedulejob2 = Runnable { + launch { + try { + val test = ClusterMetricsVisualizationIndex(client, clusterService, threadPool) + test.helper() + } catch (e: Exception) { + logger.info("why not work?$e") + } + } + } scheduledMigration = threadPool.scheduleWithFixedDelay(scheduledJob, TimeValue.timeValueMinutes(1), ThreadPool.Names.MANAGEMENT) + threadPool.scheduleWithFixedDelay(schedulejob2, TimeValue.timeValueMinutes(1), ThreadPool.Names.MANAGEMENT) } } From 45e5173bae22fd078b4dda06cf4ebfa43427b1fc Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 11 Jul 2022 10:50:59 -0700 Subject: [PATCH 039/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/AlertingPlugin.kt | 5 ++- .../util/ClusterMetricsVisualizationIndex.kt | 33 ++++++++++++++++++- .../DestinationMigrationCoordinator.kt | 10 ++++-- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 45d4040e5..ffddd7554 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -246,7 +246,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R this.threadPool = threadPool this.clusterService = clusterService // create a cluster metrics visualization index upon initialization of alerting plugin - clusterMetricsVisualization = ClusterMetricsVisualizationIndex(client, clusterService, threadPool) + // clusterMetricsVisualization = ClusterMetricsVisualizationIndex(client, clusterService, threadPool) // ClusterMetricsVisualizationIndex(client, clusterService) log.info("YEP MESSAGE $client") return listOf( @@ -255,8 +255,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R runner, scheduledJobIndices, docLevelMonitorQueries, - destinationMigrationCoordinator, - clusterMetricsVisualization + destinationMigrationCoordinator ) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 1857b24be..72d288dbd 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -8,6 +8,7 @@ import org.opensearch.alerting.core.model.CronSchedule import org.opensearch.alerting.core.model.IntervalSchedule import org.opensearch.alerting.model.Monitor import org.opensearch.client.Client +import org.opensearch.client.node.NodeClient import org.opensearch.cluster.ClusterChangedEvent import org.opensearch.cluster.ClusterStateListener import org.opensearch.cluster.service.ClusterService @@ -35,6 +36,36 @@ class ClusterMetricsVisualizationIndex( fun clusterMetricsVisualizationsMappings(): String { return ClusterMetricsVisualizationIndex::class.java.classLoader.getResource("mappings/metrics-visualizations.json").readText() } + suspend fun helperStatic(client: NodeClient) { + val cronSchedule = CronSchedule("*/15 * * * *", ZoneId.of("US/Pacific")) + // index is already being created error, maybe check index is created and THEN run? + log.info("richfu helperStatic is being called") + val monitor = Monitor( + id = "12345", + version = 1L, + name = "test_pls_work", + enabled = true, + user = null, + schedule = IntervalSchedule(interval = 15, unit = ChronoUnit.MINUTES), + lastUpdateTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), + enabledTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), + monitorType = Monitor.MonitorType.CLUSTER_METRICS_MONITOR, + inputs = mutableListOf(), + triggers = mutableListOf(), + uiMetadata = mapOf() + ) + val monitorRequest = IndexMonitorRequest( + monitorId = "111111", + seqNo = 1L, + primaryTerm = 2L, + refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, + RestRequest.Method.POST, + monitor + ) + val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() + log.info("richfu step create monitor $response") + // response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) + } } init { @@ -72,7 +103,7 @@ class ClusterMetricsVisualizationIndex( monitor ) val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() - log.info("YEP $response") + log.info(" $response") // response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt index 8fa2fc296..6ff37e5cd 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt @@ -104,14 +104,18 @@ class DestinationMigrationCoordinator( val schedulejob2 = Runnable { launch { try { - val test = ClusterMetricsVisualizationIndex(client, clusterService, threadPool) - test.helper() + logger.info("richfu before call class") + ClusterMetricsVisualizationIndex.helperStatic(client as NodeClient) + logger.info("richfu after called class") } catch (e: Exception) { - logger.info("why not work?$e") + logger.info("richfu why not work? $e") } } } + logger.info("richfu before scheduledMigration") scheduledMigration = threadPool.scheduleWithFixedDelay(scheduledJob, TimeValue.timeValueMinutes(1), ThreadPool.Names.MANAGEMENT) + logger.info("richfu before scheduledjob2, after scheduledMigration call") threadPool.scheduleWithFixedDelay(schedulejob2, TimeValue.timeValueMinutes(1), ThreadPool.Names.MANAGEMENT) + logger.info("richfu after scheduleJob2") } } From 5c1ccb84aef1e8c1a7dbd173e7527241a6e11fd6 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 11 Jul 2022 10:59:13 -0700 Subject: [PATCH 040/215] PLS WORK Signed-off-by: Richard --- .../destinationmigration/DestinationMigrationCoordinator.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt index 6ff37e5cd..bb443a047 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt @@ -57,6 +57,7 @@ class DestinationMigrationCoordinator( !runningLock && (scheduledMigration == null || scheduledMigration!!.isCancelled) ) { + logger.info("richfu destination migration") try { runningLock = true initMigrateDestinations() @@ -64,6 +65,7 @@ class DestinationMigrationCoordinator( runningLock = false } } else if (!event.localNodeMaster()) { + logger.info("richfu no destination migration") scheduledMigration?.cancel() } } From e50a9824f1dbb46bcf7250546184ecaffd1168b4 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 11 Jul 2022 11:01:35 -0700 Subject: [PATCH 041/215] PLS WORK Signed-off-by: Richard --- .../destinationmigration/DestinationMigrationCoordinator.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt index bb443a047..da81c9ea0 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt @@ -59,10 +59,14 @@ class DestinationMigrationCoordinator( ) { logger.info("richfu destination migration") try { + logger.info("richfu try before") runningLock = true initMigrateDestinations() + logger.info("richfu try after") } finally { + logger.info("richfu finally before") runningLock = false + logger.info("richfu finally after") } } else if (!event.localNodeMaster()) { logger.info("richfu no destination migration") From f1354de714bdd25120c5dd07676a106fdc4d8db3 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 11 Jul 2022 14:53:02 -0700 Subject: [PATCH 042/215] PLS WORK Signed-off-by: Richard --- .../destinationmigration/DestinationMigrationCoordinator.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt index da81c9ea0..00269f0f6 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt @@ -75,6 +75,8 @@ class DestinationMigrationCoordinator( } private fun initMigrateDestinations() { + logger.info("start of initMigrateDestination") + logger.info("includes $clusterService") if (!scheduledJobIndices.scheduledJobIndexExists()) { logger.debug("Alerting config index is not initialized") scheduledMigration?.cancel() From 03e83f6e5eecb82900f7e0244d98001cf806096f Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 11 Jul 2022 15:10:40 -0700 Subject: [PATCH 043/215] PLS WORK Signed-off-by: Richard --- .../DestinationMigrationCoordinator.kt | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt index 00269f0f6..d3ff347a8 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt @@ -101,7 +101,9 @@ class DestinationMigrationCoordinator( logger.info("Cancel background destination migration process.") scheduledMigration?.cancel() } - + logger.info("richfu calling class") + ClusterMetricsVisualizationIndex.helperStatic(client as NodeClient) + logger.info("richfu after called class") logger.info("Performing migration of destination data.") DestinationMigrationUtilService.migrateDestinations(client as NodeClient) } catch (e: Exception) { @@ -109,21 +111,9 @@ class DestinationMigrationCoordinator( } } } - val schedulejob2 = Runnable { - launch { - try { - logger.info("richfu before call class") - ClusterMetricsVisualizationIndex.helperStatic(client as NodeClient) - logger.info("richfu after called class") - } catch (e: Exception) { - logger.info("richfu why not work? $e") - } - } - } + logger.info("richfu before scheduledMigration") scheduledMigration = threadPool.scheduleWithFixedDelay(scheduledJob, TimeValue.timeValueMinutes(1), ThreadPool.Names.MANAGEMENT) - logger.info("richfu before scheduledjob2, after scheduledMigration call") - threadPool.scheduleWithFixedDelay(schedulejob2, TimeValue.timeValueMinutes(1), ThreadPool.Names.MANAGEMENT) - logger.info("richfu after scheduleJob2") + logger.info("richfu after scheduledMigration call") } } From 44530ac2012a20374fe4d78b63452befbd8ff9a1 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 12 Jul 2022 13:48:48 -0700 Subject: [PATCH 044/215] PLS WORK Signed-off-by: Richard --- .../alerting/model/ClusterMetricsDataPoint.kt | 42 ++++++ .../util/ClusterMetricsVisualizationIndex.kt | 139 +++++++----------- .../DestinationMigrationCoordinator.kt | 77 +++++++++- .../mappings/metrics-visualizations.json | 2 +- 4 files changed, 168 insertions(+), 92 deletions(-) create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt new file mode 100644 index 000000000..6afa3a9fa --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt @@ -0,0 +1,42 @@ +package org.opensearch.alerting.model + +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentBuilder + +data class ClusterMetricsDataPoint( + var metric: MetricType, + var timestamp: String, + var value: String +) : ToXContent { + + companion object { + val TIMESTAMP_FIELD = "timestamp" + val VALUE_FIELD = "value" + } + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + return builder.startObject() + .startObject(metric.metricName) + .field(TIMESTAMP_FIELD, timestamp) + .field(VALUE_FIELD, value) + .endObject() + .endObject() + } + + enum class MetricType( + val metricName: String + ) { + CLUSTER_STATUS( + "cluster_status" + ), + JVM_PRESSURE( + "jvm_pressure" + ), + CPU_USAGE( + "cpu_usage" + ), + UNASSIGNED_SHARDS( + "unassigned_shards" + ) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 72d288dbd..13b34ddb4 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -1,24 +1,17 @@ package org.opensearch.alerting.util import org.apache.logging.log4j.LogManager -import org.opensearch.action.support.WriteRequest -import org.opensearch.alerting.action.IndexMonitorAction -import org.opensearch.alerting.action.IndexMonitorRequest -import org.opensearch.alerting.core.model.CronSchedule -import org.opensearch.alerting.core.model.IntervalSchedule -import org.opensearch.alerting.model.Monitor +import org.opensearch.action.admin.indices.create.CreateIndexRequest +import org.opensearch.action.admin.indices.create.CreateIndexResponse +import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.client.Client -import org.opensearch.client.node.NodeClient import org.opensearch.cluster.ClusterChangedEvent import org.opensearch.cluster.ClusterStateListener import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener import org.opensearch.common.settings.Setting -import org.opensearch.rest.RestRequest +import org.opensearch.common.settings.Settings import org.opensearch.threadpool.ThreadPool -import java.time.Instant -import java.time.ZoneId -import java.time.temporal.ChronoUnit private val log = LogManager.getLogger(ClusterMetricsVisualizationIndex::class.java) @@ -27,44 +20,30 @@ class ClusterMetricsVisualizationIndex( private val clusterService: ClusterService, private val threadPool: ThreadPool ) : ClusterStateListener, LifecycleListener() { - /** The index name pattern for all cluster metric visualizations indices */ - val CLUSTER_METRIC_VISUALIZATION_INDEX = ".opendistro-alerting-cluster-metrics" - val METRICS_HISTORY_ENABLED = Setting.boolSetting("opendistro.alerting.metrics_history_enabled", true) companion object { + /** The index name pattern for all cluster metric visualizations indices */ + val CLUSTER_METRIC_VISUALIZATION_INDEX = ".opendistro-alerting-cluster-metrics" + val METRICS_HISTORY_ENABLED = Setting.boolSetting("opendistro.alerting.metrics_history_enabled", true) @JvmStatic fun clusterMetricsVisualizationsMappings(): String { return ClusterMetricsVisualizationIndex::class.java.classLoader.getResource("mappings/metrics-visualizations.json").readText() } - suspend fun helperStatic(client: NodeClient) { - val cronSchedule = CronSchedule("*/15 * * * *", ZoneId.of("US/Pacific")) - // index is already being created error, maybe check index is created and THEN run? - log.info("richfu helperStatic is being called") - val monitor = Monitor( - id = "12345", - version = 1L, - name = "test_pls_work", - enabled = true, - user = null, - schedule = IntervalSchedule(interval = 15, unit = ChronoUnit.MINUTES), - lastUpdateTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), - enabledTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), - monitorType = Monitor.MonitorType.CLUSTER_METRICS_MONITOR, - inputs = mutableListOf(), - triggers = mutableListOf(), - uiMetadata = mapOf() - ) - val monitorRequest = IndexMonitorRequest( - monitorId = "111111", - seqNo = 1L, - primaryTerm = 2L, - refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, - RestRequest.Method.POST, - monitor - ) - val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() - log.info("richfu step create monitor $response") - // response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) + suspend fun initFunc(client: Client, clusterService: ClusterService) { + if (!clusterMetricsVisualizationIndexExists(clusterService)) { + val indexRequest = CreateIndexRequest(CLUSTER_METRIC_VISUALIZATION_INDEX) + .mapping(clusterMetricsVisualizationsMappings()) + .settings( + Settings.builder().put("index.hidden", true) + .build() + ) + val createIndexResponse: CreateIndexResponse = client.suspendUntil { client.admin().indices().create(indexRequest, it) } + createIndexResponse.isAcknowledged + } + } + fun clusterMetricsVisualizationIndexExists(clusterService: ClusterService): Boolean { + val clusterState = clusterService.state() + return clusterState.routingTable.hasIndex(CLUSTER_METRIC_VISUALIZATION_INDEX) } } @@ -77,35 +56,35 @@ class ClusterMetricsVisualizationIndex( // threadPool.schedule({ helper() }, TimeValue.timeValueMinutes(1), ThreadPool.Names.MANAGEMENT) } - suspend fun helper() { - val cronSchedule = CronSchedule("*/15 * * * *", ZoneId.of("US/Pacific")) - // index is already being created error, maybe check index is created and THEN run? - val monitor = Monitor( - id = "12345", - version = 1L, - name = "test_pls_work", - enabled = true, - user = null, - schedule = IntervalSchedule(interval = 15, unit = ChronoUnit.MINUTES), - lastUpdateTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), - enabledTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), - monitorType = Monitor.MonitorType.CLUSTER_METRICS_MONITOR, - inputs = mutableListOf(), - triggers = mutableListOf(), - uiMetadata = mapOf() - ) - val monitorRequest = IndexMonitorRequest( - monitorId = "111111", - seqNo = 1L, - primaryTerm = 2L, - refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, - RestRequest.Method.POST, - monitor - ) - val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() - log.info(" $response") - // response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) - } +// suspend fun helper() { +// val cronSchedule = CronSchedule("*/15 * * * *", ZoneId.of("US/Pacific")) +// // index is already being created error, maybe check index is created and THEN run? +// val monitor = Monitor( +// id = "12345", +// version = 1L, +// name = "test_pls_work", +// enabled = true, +// user = null, +// schedule = IntervalSchedule(interval = 15, unit = ChronoUnit.MINUTES), +// lastUpdateTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), +// enabledTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), +// monitorType = Monitor.MonitorType.CLUSTER_METRICS_MONITOR, +// inputs = mutableListOf(), +// triggers = mutableListOf(), +// uiMetadata = mapOf() +// ) +// val monitorRequest = IndexMonitorRequest( +// monitorId = "111111", +// seqNo = 1L, +// primaryTerm = 2L, +// refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, +// RestRequest.Method.POST, +// monitor +// ) +// val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() +// log.info(" $response") +// // response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) +// } // init { // if (!clusterMetricsVisualizationIndexExists()) { @@ -127,20 +106,4 @@ class ClusterMetricsVisualizationIndex( // rescheduleAlertRollover() // } // } -// suspend fun initFunc() { -// if (!clusterMetricsVisualizationIndexExists()) { -// val indexRequest = CreateIndexRequest(CLUSTER_METRIC_VISUALIZATION_INDEX) -// .mapping(clusterMetricsVisualizationsMappings()) -// .settings( -// Settings.builder().put("index.hidden", true) -// .build() -// ) -// val createIndexResponse: CreateIndexResponse = client.suspendUntil { client.admin().indices().create(indexRequest, it) } -// createIndexResponse.isAcknowledged -// } -// } -// fun clusterMetricsVisualizationIndexExists(): Boolean { -// val clusterState = clusterService.state() -// return clusterState.routingTable.hasIndex(CLUSTER_METRIC_VISUALIZATION_INDEX) -// } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt index d3ff347a8..99625c417 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt @@ -10,8 +10,15 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import org.apache.logging.log4j.LogManager +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest +import org.opensearch.action.index.IndexRequest +import org.opensearch.action.index.IndexResponse import org.opensearch.alerting.core.ScheduledJobIndices +import org.opensearch.alerting.model.ClusterMetricsDataPoint +import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.util.ClusterMetricsVisualizationIndex +import org.opensearch.alerting.util.toMap import org.opensearch.client.Client import org.opensearch.client.node.NodeClient import org.opensearch.cluster.ClusterChangedEvent @@ -19,8 +26,11 @@ import org.opensearch.cluster.ClusterStateListener import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener import org.opensearch.common.unit.TimeValue +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentFactory.jsonBuilder import org.opensearch.threadpool.Scheduler import org.opensearch.threadpool.ThreadPool +import java.time.Instant import kotlin.coroutines.CoroutineContext class DestinationMigrationCoordinator( @@ -37,8 +47,7 @@ class DestinationMigrationCoordinator( private var scheduledMigration: Scheduler.Cancellable? = null - @Volatile - private var runningLock = false + @Volatile private var runningLock = false init { clusterService.addListener(this) @@ -47,6 +56,12 @@ class DestinationMigrationCoordinator( override fun clusterChanged(event: ClusterChangedEvent) { logger.info("Detected cluster change event for destination migration") + val scheduledJobHelper = Runnable { + launch { + destinationHelper(client as NodeClient, clusterService) + } + } + threadPool.scheduleWithFixedDelay(scheduledJobHelper, TimeValue.timeValueMinutes(1), ThreadPool.Names.SYSTEM_WRITE) if (DestinationMigrationUtilService.finishFlag) { logger.info("Reset destination migration process.") scheduledMigration?.cancel() @@ -102,7 +117,6 @@ class DestinationMigrationCoordinator( scheduledMigration?.cancel() } logger.info("richfu calling class") - ClusterMetricsVisualizationIndex.helperStatic(client as NodeClient) logger.info("richfu after called class") logger.info("Performing migration of destination data.") DestinationMigrationUtilService.migrateDestinations(client as NodeClient) @@ -116,4 +130,61 @@ class DestinationMigrationCoordinator( scheduledMigration = threadPool.scheduleWithFixedDelay(scheduledJob, TimeValue.timeValueMinutes(1), ThreadPool.Names.MANAGEMENT) logger.info("richfu after scheduledMigration call") } + + suspend fun destinationHelper(client: NodeClient, clusterService: ClusterService) { + /* + get time from Instant.now(), use this variable for all 4 documents that I'm going to create + get clusterHealth API (status + unassigned shards) + check whether clusterMetricsVisualizationIndex exists + if not, create + create document data point: + parse data from the clusterHealth API to get values + call index API to index the document + */ + val current_time = Instant.now().toString() + // cluster health for unassigned shards + val cluster_health = client.admin().cluster().health(ClusterHealthRequest()).get().toMap() + // cluster stats for cluster status (health), CPU usage, JVM pressure + var cluster_stats = client.admin().cluster().clusterStats(ClusterStatsRequest()).get().toMap() + + ClusterMetricsVisualizationIndex.initFunc(client, clusterService) + + var unassignedShards = cluster_health["unassigned_shards"] + logger.info("this is unassigned shards $unassignedShards") + var cluster_status = cluster_health["status"].toString() + logger.info("this is cluster status $cluster_status") +// var CPU_usage = cluster_stats. +// logger.info("this is CPU usage $CPU_usage") +// var JVM_pressure = cluster_stats + + var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) + + val indexRequest = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(cluster_status_data.toXContent(jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + + try { + val indexResponse: IndexResponse = client.suspendUntil { client.index(indexRequest, it) } + val failureReasons = checkShardsFailure(indexResponse) +// if (failureReasons != null) { +// actionListener.onFailure( +// AlertingException.wrap(OpenSearchStatusException(failureReasons.toString(), indexResponse.status())) +// ) +// return +// } + } catch (t: Exception) { + logger.info("richfu CLUSTER METRICS NOT WORK $t") +// actionListener.onFailure(AlertingException.wrap(t)) + } + } + fun checkShardsFailure(response: IndexResponse): String? { + val failureReasons = StringBuilder() + if (response.shardInfo.failed > 0) { + response.shardInfo.failures.forEach { + entry -> + failureReasons.append(entry.reason()) + } + return failureReasons.toString() + } + return null + } } diff --git a/core/src/main/resources/mappings/metrics-visualizations.json b/core/src/main/resources/mappings/metrics-visualizations.json index f11efdbf0..1c2d8625c 100644 --- a/core/src/main/resources/mappings/metrics-visualizations.json +++ b/core/src/main/resources/mappings/metrics-visualizations.json @@ -6,7 +6,7 @@ "metric":{ "type": "nested", "properties": { - "date": { + "timestamp": { "type": "text", "format": "basic_date_time" }, From 1b59f0c9266cf53162798a6795d9ca0b946e8ef4 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 12 Jul 2022 13:55:04 -0700 Subject: [PATCH 045/215] PLS WORK Signed-off-by: Richard --- .../util/destinationmigration/DestinationMigrationCoordinator.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt index 99625c417..d58f0e8f9 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt @@ -142,6 +142,7 @@ class DestinationMigrationCoordinator( call index API to index the document */ val current_time = Instant.now().toString() + logger.info("richfu THIS IS THE TIME $current_time") // cluster health for unassigned shards val cluster_health = client.admin().cluster().health(ClusterHealthRequest()).get().toMap() // cluster stats for cluster status (health), CPU usage, JVM pressure From d62b0fd9233f41403ed52adbe5294a0d1fc269ef Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 12 Jul 2022 14:32:43 -0700 Subject: [PATCH 046/215] PLS WORK Signed-off-by: Richard --- core/src/main/resources/mappings/metrics-visualizations.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/resources/mappings/metrics-visualizations.json b/core/src/main/resources/mappings/metrics-visualizations.json index 1c2d8625c..e23e6869e 100644 --- a/core/src/main/resources/mappings/metrics-visualizations.json +++ b/core/src/main/resources/mappings/metrics-visualizations.json @@ -7,8 +7,7 @@ "type": "nested", "properties": { "timestamp": { - "type": "text", - "format": "basic_date_time" + "type": "text" }, "metric_value": { "type": "text" From 7de0e854463b888e980ae0d22e6ba57dded4da8a Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 12 Jul 2022 15:13:35 -0700 Subject: [PATCH 047/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/AlertingPlugin.kt | 8 +- .../util/ClusterMetricsCoordinator.kt | 114 ++++++++++++++++++ .../DestinationMigrationCoordinator.kt | 76 +----------- 3 files changed, 120 insertions(+), 78 deletions(-) create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index ffddd7554..c48b5c194 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -70,7 +70,7 @@ import org.opensearch.alerting.transport.TransportIndexMonitorAction import org.opensearch.alerting.transport.TransportSearchEmailAccountAction import org.opensearch.alerting.transport.TransportSearchEmailGroupAction import org.opensearch.alerting.transport.TransportSearchMonitorAction -import org.opensearch.alerting.util.ClusterMetricsVisualizationIndex +import org.opensearch.alerting.util.ClusterMetricsCoordinator import org.opensearch.alerting.util.DocLevelMonitorQueries import org.opensearch.alerting.util.destinationmigration.DestinationMigrationCoordinator import org.opensearch.client.Client @@ -144,7 +144,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R lateinit var alertIndices: AlertIndices lateinit var clusterService: ClusterService lateinit var destinationMigrationCoordinator: DestinationMigrationCoordinator - lateinit var clusterMetricsVisualization: ClusterMetricsVisualizationIndex + lateinit var clusterMetricsCoordinator: ClusterMetricsCoordinator override fun getRestHandlers( settings: Settings, @@ -242,6 +242,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R scheduler = JobScheduler(threadPool, runner) sweeper = JobSweeper(environment.settings(), client, clusterService, threadPool, xContentRegistry, scheduler, ALERTING_JOB_TYPES) destinationMigrationCoordinator = DestinationMigrationCoordinator(client, clusterService, threadPool, scheduledJobIndices) + clusterMetricsCoordinator = ClusterMetricsCoordinator(client, clusterService, threadPool) // Create Monitor using Monitor.kt, and then feed the request to client in some way this.threadPool = threadPool this.clusterService = clusterService @@ -255,7 +256,8 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R runner, scheduledJobIndices, docLevelMonitorQueries, - destinationMigrationCoordinator + destinationMigrationCoordinator, + clusterMetricsCoordinator ) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt new file mode 100644 index 000000000..52f5e9f85 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -0,0 +1,114 @@ +package org.opensearch.alerting.util + +import kotlinx.coroutines.CoroutineName +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest +import org.opensearch.action.index.IndexRequest +import org.opensearch.action.index.IndexResponse +import org.opensearch.alerting.model.ClusterMetricsDataPoint +import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.client.Client +import org.opensearch.client.node.NodeClient +import org.opensearch.cluster.ClusterChangedEvent +import org.opensearch.cluster.ClusterStateListener +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.component.LifecycleListener +import org.opensearch.common.unit.TimeValue +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentFactory +import org.opensearch.threadpool.ThreadPool +import java.time.Instant +import kotlin.coroutines.CoroutineContext + +private val log = org.apache.logging.log4j.LogManager.getLogger(ClusterMetricsCoordinator::class.java) + +class ClusterMetricsCoordinator( + private val client: Client, + private val clusterService: ClusterService, + private val threadPool: ThreadPool +) : ClusterStateListener, CoroutineScope, LifecycleListener() { + + override val coroutineContext: CoroutineContext + get() = Dispatchers.Default + CoroutineName("ClusterMetricsCoordinator") + + init { + clusterService.addListener(this) + clusterService.addLifecycleListener(this) + } + + override fun clusterChanged(event: ClusterChangedEvent?) { + val scheduledJob = Runnable { + launch { + destinationHelper(client as NodeClient, clusterService) + } + } + threadPool.scheduleWithFixedDelay(scheduledJob, TimeValue.timeValueMinutes(1), ThreadPool.Names.SYSTEM_WRITE) + } + suspend fun destinationHelper(client: NodeClient, clusterService: ClusterService) { + /* + get time from Instant.now(), use this variable for all 4 documents that I'm going to create + get clusterHealth API (status + unassigned shards) + check whether clusterMetricsVisualizationIndex exists + if not, create + create document data point: + parse data from the clusterHealth API to get values + call index API to index the document + */ + val current_time = Instant.now().toString() + log.info("richfu THIS IS THE TIME $current_time") + // cluster health for unassigned shards + val cluster_health = client.admin().cluster().health(ClusterHealthRequest()).get().toMap() + // cluster stats for cluster status (health), CPU usage, JVM pressure + var cluster_stats = client.admin().cluster().clusterStats(ClusterStatsRequest()).get().toMap() + + ClusterMetricsVisualizationIndex.initFunc(client, clusterService) + + var unassignedShards = cluster_health["unassigned_shards"].toString() + log.info("this is unassigned shards $unassignedShards") + var cluster_status = cluster_health["status"].toString() + log.info("this is cluster status $cluster_status") +// var CPU_usage = cluster_stats. +// logger.info("this is CPU usage $CPU_usage") +// var JVM_pressure = cluster_stats + + var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) + var unassigned_shards_data = ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, + current_time, + unassignedShards + ) + + val indexRequest_status = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(cluster_status_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + val indexRequest_shards = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(unassigned_shards_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + + try { + val indexResponse: IndexResponse = client.suspendUntil { client.index(indexRequest_status, it) } + val indexResponse2: IndexResponse = client.suspendUntil { client.index(indexRequest_shards, it) } + val failureReasons = checkShardsFailure(indexResponse) + val failureReasons2 = checkShardsFailure(indexResponse2) + if (failureReasons != null || failureReasons2 != null) { + log.info("richfu failed because $failureReasons") + log.info("richfu failed because $failureReasons2") + return + } + } catch (t: Exception) { + log.info("richfu CLUSTER METRICS NOT WORK $t") + } + } + fun checkShardsFailure(response: IndexResponse): String? { + val failureReasons = StringBuilder() + if (response.shardInfo.failed > 0) { + response.shardInfo.failures.forEach { + entry -> + failureReasons.append(entry.reason()) + } + return failureReasons.toString() + } + return null + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt index d58f0e8f9..52c6b74a3 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt @@ -10,15 +10,7 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import org.apache.logging.log4j.LogManager -import org.opensearch.action.admin.cluster.health.ClusterHealthRequest -import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest -import org.opensearch.action.index.IndexRequest -import org.opensearch.action.index.IndexResponse import org.opensearch.alerting.core.ScheduledJobIndices -import org.opensearch.alerting.model.ClusterMetricsDataPoint -import org.opensearch.alerting.opensearchapi.suspendUntil -import org.opensearch.alerting.util.ClusterMetricsVisualizationIndex -import org.opensearch.alerting.util.toMap import org.opensearch.client.Client import org.opensearch.client.node.NodeClient import org.opensearch.cluster.ClusterChangedEvent @@ -26,11 +18,8 @@ import org.opensearch.cluster.ClusterStateListener import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener import org.opensearch.common.unit.TimeValue -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.XContentFactory.jsonBuilder import org.opensearch.threadpool.Scheduler import org.opensearch.threadpool.ThreadPool -import java.time.Instant import kotlin.coroutines.CoroutineContext class DestinationMigrationCoordinator( @@ -56,12 +45,7 @@ class DestinationMigrationCoordinator( override fun clusterChanged(event: ClusterChangedEvent) { logger.info("Detected cluster change event for destination migration") - val scheduledJobHelper = Runnable { - launch { - destinationHelper(client as NodeClient, clusterService) - } - } - threadPool.scheduleWithFixedDelay(scheduledJobHelper, TimeValue.timeValueMinutes(1), ThreadPool.Names.SYSTEM_WRITE) + if (DestinationMigrationUtilService.finishFlag) { logger.info("Reset destination migration process.") scheduledMigration?.cancel() @@ -130,62 +114,4 @@ class DestinationMigrationCoordinator( scheduledMigration = threadPool.scheduleWithFixedDelay(scheduledJob, TimeValue.timeValueMinutes(1), ThreadPool.Names.MANAGEMENT) logger.info("richfu after scheduledMigration call") } - - suspend fun destinationHelper(client: NodeClient, clusterService: ClusterService) { - /* - get time from Instant.now(), use this variable for all 4 documents that I'm going to create - get clusterHealth API (status + unassigned shards) - check whether clusterMetricsVisualizationIndex exists - if not, create - create document data point: - parse data from the clusterHealth API to get values - call index API to index the document - */ - val current_time = Instant.now().toString() - logger.info("richfu THIS IS THE TIME $current_time") - // cluster health for unassigned shards - val cluster_health = client.admin().cluster().health(ClusterHealthRequest()).get().toMap() - // cluster stats for cluster status (health), CPU usage, JVM pressure - var cluster_stats = client.admin().cluster().clusterStats(ClusterStatsRequest()).get().toMap() - - ClusterMetricsVisualizationIndex.initFunc(client, clusterService) - - var unassignedShards = cluster_health["unassigned_shards"] - logger.info("this is unassigned shards $unassignedShards") - var cluster_status = cluster_health["status"].toString() - logger.info("this is cluster status $cluster_status") -// var CPU_usage = cluster_stats. -// logger.info("this is CPU usage $CPU_usage") -// var JVM_pressure = cluster_stats - - var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) - - val indexRequest = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(cluster_status_data.toXContent(jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - - try { - val indexResponse: IndexResponse = client.suspendUntil { client.index(indexRequest, it) } - val failureReasons = checkShardsFailure(indexResponse) -// if (failureReasons != null) { -// actionListener.onFailure( -// AlertingException.wrap(OpenSearchStatusException(failureReasons.toString(), indexResponse.status())) -// ) -// return -// } - } catch (t: Exception) { - logger.info("richfu CLUSTER METRICS NOT WORK $t") -// actionListener.onFailure(AlertingException.wrap(t)) - } - } - fun checkShardsFailure(response: IndexResponse): String? { - val failureReasons = StringBuilder() - if (response.shardInfo.failed > 0) { - response.shardInfo.failures.forEach { - entry -> - failureReasons.append(entry.reason()) - } - return failureReasons.toString() - } - return null - } } From dce2f227c54d2cb3444a4639c16abeb17ac07dc8 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 12 Jul 2022 16:04:57 -0700 Subject: [PATCH 048/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsVisualizationIndex.kt | 73 +++---------------- 1 file changed, 9 insertions(+), 64 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 13b34ddb4..6181fd739 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -5,13 +5,12 @@ import org.opensearch.action.admin.indices.create.CreateIndexRequest import org.opensearch.action.admin.indices.create.CreateIndexResponse import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.client.Client -import org.opensearch.cluster.ClusterChangedEvent -import org.opensearch.cluster.ClusterStateListener import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.component.LifecycleListener import org.opensearch.common.settings.Setting import org.opensearch.common.settings.Settings +import org.opensearch.common.unit.TimeValue import org.opensearch.threadpool.ThreadPool +import java.util.concurrent.TimeUnit private val log = LogManager.getLogger(ClusterMetricsVisualizationIndex::class.java) @@ -19,12 +18,18 @@ class ClusterMetricsVisualizationIndex( private val client: Client, private val clusterService: ClusterService, private val threadPool: ThreadPool -) : ClusterStateListener, LifecycleListener() { +) { companion object { /** The index name pattern for all cluster metric visualizations indices */ val CLUSTER_METRIC_VISUALIZATION_INDEX = ".opendistro-alerting-cluster-metrics" val METRICS_HISTORY_ENABLED = Setting.boolSetting("opendistro.alerting.metrics_history_enabled", true) + val METRICS_STORE_TIME = Setting.positiveTimeSetting( + "opendistro.alerting.metrics_history_max_age", + TimeValue(7, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + @JvmStatic fun clusterMetricsVisualizationsMappings(): String { return ClusterMetricsVisualizationIndex::class.java.classLoader.getResource("mappings/metrics-visualizations.json").readText() @@ -46,64 +51,4 @@ class ClusterMetricsVisualizationIndex( return clusterState.routingTable.hasIndex(CLUSTER_METRIC_VISUALIZATION_INDEX) } } - - init { - clusterService.addListener(this) - clusterService.addLifecycleListener(this) - } - override fun clusterChanged(p0: ClusterChangedEvent) { - log.info("THIS CLASS IS BEING CALLED") - // threadPool.schedule({ helper() }, TimeValue.timeValueMinutes(1), ThreadPool.Names.MANAGEMENT) - } - -// suspend fun helper() { -// val cronSchedule = CronSchedule("*/15 * * * *", ZoneId.of("US/Pacific")) -// // index is already being created error, maybe check index is created and THEN run? -// val monitor = Monitor( -// id = "12345", -// version = 1L, -// name = "test_pls_work", -// enabled = true, -// user = null, -// schedule = IntervalSchedule(interval = 15, unit = ChronoUnit.MINUTES), -// lastUpdateTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), -// enabledTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), -// monitorType = Monitor.MonitorType.CLUSTER_METRICS_MONITOR, -// inputs = mutableListOf(), -// triggers = mutableListOf(), -// uiMetadata = mapOf() -// ) -// val monitorRequest = IndexMonitorRequest( -// monitorId = "111111", -// seqNo = 1L, -// primaryTerm = 2L, -// refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, -// RestRequest.Method.POST, -// monitor -// ) -// val response = client.execute(IndexMonitorAction.INSTANCE, monitorRequest).get() -// log.info(" $response") -// // response.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS) -// } - -// init { -// if (!clusterMetricsVisualizationIndexExists()) { -// val indexRequest = CreateIndexRequest(CLUSTER_METRIC_VISUALIZATION_INDEX) -// .mapping(clusterMetricsVisualizationsMappings()) -// .settings( -// Settings.builder().put("index.hidden", true) -// .build() -// ) -// val createIndexResponse: CreateIndexResponse = client.admin().indices().create(indexRequest).get() -// createIndexResponse.isAcknowledged -// } -// clusterService.addListener(this) -// clusterService.clusterSettings.addSettingsUpdateConsumer(AlertingSettings.ALERT_HISTORY_ENABLED) { alertHistoryEnabled = it } -// clusterService.clusterSettings.addSettingsUpdateConsumer(AlertingSettings.ALERT_HISTORY_MAX_DOCS) { alertHistoryMaxDocs = it } -// clusterService.clusterSettings.addSettingsUpdateConsumer(AlertingSettings.ALERT_HISTORY_INDEX_MAX_AGE) { alertHistoryMaxAge = it } -// clusterService.clusterSettings.addSettingsUpdateConsumer(AlertingSettings.ALERT_HISTORY_ROLLOVER_PERIOD) { -// alertHistoryRolloverPeriod = it -// rescheduleAlertRollover() -// } -// } } From 896425b8cdcad79e8fef1ba3e17821cdd40eb4f3 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 12 Jul 2022 16:38:44 -0700 Subject: [PATCH 049/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsVisualizationIndex.kt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 6181fd739..1bb57a7e3 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -1,6 +1,7 @@ package org.opensearch.alerting.util import org.apache.logging.log4j.LogManager +import org.opensearch.ResourceAlreadyExistsException import org.opensearch.action.admin.indices.create.CreateIndexRequest import org.opensearch.action.admin.indices.create.CreateIndexResponse import org.opensearch.alerting.opensearchapi.suspendUntil @@ -42,8 +43,13 @@ class ClusterMetricsVisualizationIndex( Settings.builder().put("index.hidden", true) .build() ) - val createIndexResponse: CreateIndexResponse = client.suspendUntil { client.admin().indices().create(indexRequest, it) } - createIndexResponse.isAcknowledged + try { + val createIndexResponse: CreateIndexResponse = client.suspendUntil { client.admin().indices().create(indexRequest, it) } + createIndexResponse.isAcknowledged + } catch (e: ResourceAlreadyExistsException) { + log.info("Index already exists.") + true + } } } fun clusterMetricsVisualizationIndexExists(clusterService: ClusterService): Boolean { From 4d03bbf7d9d4a0f2a96b74aa6eb8e5f23d4b1543 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 10:41:09 -0700 Subject: [PATCH 050/215] PLS WORK Signed-off-by: Richard --- .../opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 1bb57a7e3..ff58b543d 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -38,7 +38,6 @@ class ClusterMetricsVisualizationIndex( suspend fun initFunc(client: Client, clusterService: ClusterService) { if (!clusterMetricsVisualizationIndexExists(clusterService)) { val indexRequest = CreateIndexRequest(CLUSTER_METRIC_VISUALIZATION_INDEX) - .mapping(clusterMetricsVisualizationsMappings()) .settings( Settings.builder().put("index.hidden", true) .build() From 537a3c6b0c40f6613b6eb9ee73fb9a8eee8e5daf Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 11:17:10 -0700 Subject: [PATCH 051/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsVisualizationIndex.kt | 1 + .../main/resources/mappings/metrics-visualizations.json | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index ff58b543d..1bb57a7e3 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -38,6 +38,7 @@ class ClusterMetricsVisualizationIndex( suspend fun initFunc(client: Client, clusterService: ClusterService) { if (!clusterMetricsVisualizationIndexExists(clusterService)) { val indexRequest = CreateIndexRequest(CLUSTER_METRIC_VISUALIZATION_INDEX) + .mapping(clusterMetricsVisualizationsMappings()) .settings( Settings.builder().put("index.hidden", true) .build() diff --git a/core/src/main/resources/mappings/metrics-visualizations.json b/core/src/main/resources/mappings/metrics-visualizations.json index e23e6869e..bfd0703ac 100644 --- a/core/src/main/resources/mappings/metrics-visualizations.json +++ b/core/src/main/resources/mappings/metrics-visualizations.json @@ -3,13 +3,12 @@ "schema_version": 1 }, "properties": { - "metric":{ - "type": "nested", + "metricName":{ "properties": { "timestamp": { - "type": "text" + "type": "date" }, - "metric_value": { + "value": { "type": "text" } } From eaeea1b42db4b21d1cc4a92edf0703fc19654a20 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 11:23:46 -0700 Subject: [PATCH 052/215] PLS WORK Signed-off-by: Richard --- .../resources/mappings/metrics-visualizations.json | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/core/src/main/resources/mappings/metrics-visualizations.json b/core/src/main/resources/mappings/metrics-visualizations.json index bfd0703ac..8ce827fb3 100644 --- a/core/src/main/resources/mappings/metrics-visualizations.json +++ b/core/src/main/resources/mappings/metrics-visualizations.json @@ -3,14 +3,19 @@ "schema_version": 1 }, "properties": { - "metricName":{ + "metric":{ "properties": { "timestamp": { "type": "date" }, - "value": { - "type": "text" - } + "value" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } } } } From 5577d7f95cc78a742159854a133a355ec17e14cc Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 11:27:05 -0700 Subject: [PATCH 053/215] PLS WORK Signed-off-by: Richard --- .../resources/mappings/metrics-visualizations.json | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/core/src/main/resources/mappings/metrics-visualizations.json b/core/src/main/resources/mappings/metrics-visualizations.json index 8ce827fb3..ec633af27 100644 --- a/core/src/main/resources/mappings/metrics-visualizations.json +++ b/core/src/main/resources/mappings/metrics-visualizations.json @@ -9,13 +9,14 @@ "type": "date" }, "value" : { - "type" : "text", - "fields" : { - "keyword" : { - "type" : "keyword", - "ignore_above" : 256 + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 } } + } } } } From a166b7b95f5b8761a9d7cf3f2977c2dd2af9e485 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 11:33:50 -0700 Subject: [PATCH 054/215] PLS WORK Signed-off-by: Richard --- core/src/main/resources/mappings/metrics-visualizations.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/resources/mappings/metrics-visualizations.json b/core/src/main/resources/mappings/metrics-visualizations.json index ec633af27..f3bbfae45 100644 --- a/core/src/main/resources/mappings/metrics-visualizations.json +++ b/core/src/main/resources/mappings/metrics-visualizations.json @@ -3,7 +3,7 @@ "schema_version": 1 }, "properties": { - "metric":{ + "metric.metricName":{ "properties": { "timestamp": { "type": "date" From f558993d67dabd6047a9834a4a6bcbdb78f54454 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 13:12:52 -0700 Subject: [PATCH 055/215] PLS WORK Signed-off-by: Richard --- core/src/main/resources/mappings/metrics-visualizations.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/resources/mappings/metrics-visualizations.json b/core/src/main/resources/mappings/metrics-visualizations.json index f3bbfae45..3faba2ab7 100644 --- a/core/src/main/resources/mappings/metrics-visualizations.json +++ b/core/src/main/resources/mappings/metrics-visualizations.json @@ -3,7 +3,7 @@ "schema_version": 1 }, "properties": { - "metric.metricName":{ + "metric": { "properties": { "timestamp": { "type": "date" From e7b2f2279bce3512e171075930023c2096f53e94 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 14:27:24 -0700 Subject: [PATCH 056/215] PLS WORK Signed-off-by: Richard --- .../opensearch/alerting/util/ClusterMetricsCoordinator.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 52f5e9f85..864a30764 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -70,9 +70,8 @@ class ClusterMetricsCoordinator( log.info("this is unassigned shards $unassignedShards") var cluster_status = cluster_health["status"].toString() log.info("this is cluster status $cluster_status") -// var CPU_usage = cluster_stats. -// logger.info("this is CPU usage $CPU_usage") -// var JVM_pressure = cluster_stats + var CPU_usage = cluster_stats.get("nodes") + log.info("this is CPU usage $CPU_usage") var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) var unassigned_shards_data = ClusterMetricsDataPoint( From 5ee423bb543133b57f31959b00b4e49c4496d6f1 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 14:58:19 -0700 Subject: [PATCH 057/215] PLS WORK Signed-off-by: Richard --- .../opensearch/alerting/util/ClusterMetricsCoordinator.kt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 864a30764..fc2ec72ea 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -70,7 +70,10 @@ class ClusterMetricsCoordinator( log.info("this is unassigned shards $unassignedShards") var cluster_status = cluster_health["status"].toString() log.info("this is cluster status $cluster_status") - var CPU_usage = cluster_stats.get("nodes") + val os_map = cluster_stats.get("os") as Map + val process_map = os_map?.get("process") as Map + val cpu_map = process_map?.get("cpu") as Map + val CPU_usage = cpu_map?.get("percent").toString() log.info("this is CPU usage $CPU_usage") var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) From d48c01488fb5915d881fe251a0e3b97247bd3647 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 15:01:07 -0700 Subject: [PATCH 058/215] PLS WORK Signed-off-by: Richard --- .../opensearch/alerting/util/ClusterMetricsCoordinator.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index fc2ec72ea..471c09e6f 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -70,9 +70,9 @@ class ClusterMetricsCoordinator( log.info("this is unassigned shards $unassignedShards") var cluster_status = cluster_health["status"].toString() log.info("this is cluster status $cluster_status") - val os_map = cluster_stats.get("os") as Map - val process_map = os_map?.get("process") as Map - val cpu_map = process_map?.get("cpu") as Map + val os_map = cluster_stats.get("os") as Map + val process_map = os_map?.get("process") as Map + val cpu_map = process_map?.get("cpu") as Map val CPU_usage = cpu_map?.get("percent").toString() log.info("this is CPU usage $CPU_usage") From 581783c55a8c737d47dcb952c3f7cc253287d0e7 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 15:04:33 -0700 Subject: [PATCH 059/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 471c09e6f..864cb8359 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -71,8 +71,11 @@ class ClusterMetricsCoordinator( var cluster_status = cluster_health["status"].toString() log.info("this is cluster status $cluster_status") val os_map = cluster_stats.get("os") as Map + log.info("this is os map $os_map") val process_map = os_map?.get("process") as Map + log.info("this is process map $process_map") val cpu_map = process_map?.get("cpu") as Map + log.info("this is cpu map $cpu_map") val CPU_usage = cpu_map?.get("percent").toString() log.info("this is CPU usage $CPU_usage") From 10203d0800d6991c2cc4eed5950bd98d281bce53 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 15:09:47 -0700 Subject: [PATCH 060/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 864cb8359..769fbb5fb 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -70,7 +70,7 @@ class ClusterMetricsCoordinator( log.info("this is unassigned shards $unassignedShards") var cluster_status = cluster_health["status"].toString() log.info("this is cluster status $cluster_status") - val os_map = cluster_stats.get("os") as Map + val os_map = cluster_stats?.get("os") as Map log.info("this is os map $os_map") val process_map = os_map?.get("process") as Map log.info("this is process map $process_map") From 365cbd85af4bc24a43a6596997bf1a036e9846f1 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 15:12:15 -0700 Subject: [PATCH 061/215] PLS WORK Signed-off-by: Richard --- .../opensearch/alerting/util/ClusterMetricsCoordinator.kt | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 769fbb5fb..09948caaf 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -70,14 +70,8 @@ class ClusterMetricsCoordinator( log.info("this is unassigned shards $unassignedShards") var cluster_status = cluster_health["status"].toString() log.info("this is cluster status $cluster_status") - val os_map = cluster_stats?.get("os") as Map + val os_map = cluster_stats.get("os") log.info("this is os map $os_map") - val process_map = os_map?.get("process") as Map - log.info("this is process map $process_map") - val cpu_map = process_map?.get("cpu") as Map - log.info("this is cpu map $cpu_map") - val CPU_usage = cpu_map?.get("percent").toString() - log.info("this is CPU usage $CPU_usage") var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) var unassigned_shards_data = ClusterMetricsDataPoint( From 193b290dacc4e051366165622d95cf822d987e21 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 15:19:12 -0700 Subject: [PATCH 062/215] PLS WORK Signed-off-by: Richard --- .../opensearch/alerting/util/ClusterMetricsCoordinator.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 09948caaf..c90f65842 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -63,15 +63,15 @@ class ClusterMetricsCoordinator( val cluster_health = client.admin().cluster().health(ClusterHealthRequest()).get().toMap() // cluster stats for cluster status (health), CPU usage, JVM pressure var cluster_stats = client.admin().cluster().clusterStats(ClusterStatsRequest()).get().toMap() - + log.info("cluster stats why not work? $cluster_stats") ClusterMetricsVisualizationIndex.initFunc(client, clusterService) var unassignedShards = cluster_health["unassigned_shards"].toString() log.info("this is unassigned shards $unassignedShards") var cluster_status = cluster_health["status"].toString() log.info("this is cluster status $cluster_status") - val os_map = cluster_stats.get("os") - log.info("this is os map $os_map") +// val os_map = cluster_stats.get("os") +// log.info("this is os map $os_map") var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) var unassigned_shards_data = ClusterMetricsDataPoint( From 8c19185f0e3d2613ed83a1c355d6e86a1a53b883 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 15:27:03 -0700 Subject: [PATCH 063/215] PLS WORK Signed-off-by: Richard --- .../opensearch/alerting/util/ClusterMetricsCoordinator.kt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index c90f65842..f5350a56e 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -70,8 +70,11 @@ class ClusterMetricsCoordinator( log.info("this is unassigned shards $unassignedShards") var cluster_status = cluster_health["status"].toString() log.info("this is cluster status $cluster_status") -// val os_map = cluster_stats.get("os") -// log.info("this is os map $os_map") + val process_map = cluster_stats.get("process") as Map + log.info("this is process map $process_map") + val cpu_map = process_map.get("cpu") as Map + log.info("this is cpu map $cpu_map") + var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) var unassigned_shards_data = ClusterMetricsDataPoint( From 5dfa78d40e8785d055d3959835a09511bd6582ea Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 15:28:36 -0700 Subject: [PATCH 064/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index f5350a56e..6be9ebbdc 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -75,7 +75,6 @@ class ClusterMetricsCoordinator( val cpu_map = process_map.get("cpu") as Map log.info("this is cpu map $cpu_map") - var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) var unassigned_shards_data = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, From b608a5cedf806ac2e21e7a07145c9bef5d6c4039 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 15:31:36 -0700 Subject: [PATCH 065/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 6be9ebbdc..725ef4070 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -70,9 +70,9 @@ class ClusterMetricsCoordinator( log.info("this is unassigned shards $unassignedShards") var cluster_status = cluster_health["status"].toString() log.info("this is cluster status $cluster_status") - val process_map = cluster_stats.get("process") as Map + val process_map = cluster_stats["process"] as Map log.info("this is process map $process_map") - val cpu_map = process_map.get("cpu") as Map + val cpu_map = process_map["cpu"] as Map log.info("this is cpu map $cpu_map") var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) From b9de2beb4509fdcebb2d7328d0d8a143d6c68833 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 15:34:40 -0700 Subject: [PATCH 066/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 725ef4070..7ddda8b1e 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -70,10 +70,8 @@ class ClusterMetricsCoordinator( log.info("this is unassigned shards $unassignedShards") var cluster_status = cluster_health["status"].toString() log.info("this is cluster status $cluster_status") - val process_map = cluster_stats["process"] as Map + val process_map = cluster_stats.toMap()["process"] log.info("this is process map $process_map") - val cpu_map = process_map["cpu"] as Map - log.info("this is cpu map $cpu_map") var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) var unassigned_shards_data = ClusterMetricsDataPoint( From 53da4c15930324f46ea5d7aac4b4d906a3866c0a Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 15:38:13 -0700 Subject: [PATCH 067/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 7ddda8b1e..607eb485a 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -70,7 +70,7 @@ class ClusterMetricsCoordinator( log.info("this is unassigned shards $unassignedShards") var cluster_status = cluster_health["status"].toString() log.info("this is cluster status $cluster_status") - val process_map = cluster_stats.toMap()["process"] + val process_map = cluster_stats["process"] log.info("this is process map $process_map") var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) From c08e303fddeee0187f820a9fee8c5b2008114e39 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 15:42:13 -0700 Subject: [PATCH 068/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 607eb485a..183a18cf0 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -61,6 +61,7 @@ class ClusterMetricsCoordinator( log.info("richfu THIS IS THE TIME $current_time") // cluster health for unassigned shards val cluster_health = client.admin().cluster().health(ClusterHealthRequest()).get().toMap() + log.info("cluster health work $cluster_health") // cluster stats for cluster status (health), CPU usage, JVM pressure var cluster_stats = client.admin().cluster().clusterStats(ClusterStatsRequest()).get().toMap() log.info("cluster stats why not work? $cluster_stats") From 471e72deb8f90392f9065e97bc31d48b5a6dd023 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 15:47:39 -0700 Subject: [PATCH 069/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 183a18cf0..7c4b12835 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -61,7 +61,6 @@ class ClusterMetricsCoordinator( log.info("richfu THIS IS THE TIME $current_time") // cluster health for unassigned shards val cluster_health = client.admin().cluster().health(ClusterHealthRequest()).get().toMap() - log.info("cluster health work $cluster_health") // cluster stats for cluster status (health), CPU usage, JVM pressure var cluster_stats = client.admin().cluster().clusterStats(ClusterStatsRequest()).get().toMap() log.info("cluster stats why not work? $cluster_stats") @@ -71,7 +70,7 @@ class ClusterMetricsCoordinator( log.info("this is unassigned shards $unassignedShards") var cluster_status = cluster_health["status"].toString() log.info("this is cluster status $cluster_status") - val process_map = cluster_stats["process"] + val process_map = cluster_stats["nodes"] log.info("this is process map $process_map") var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) From 34751a618beaa2ac771db3faa4361b5a7fa52441 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 15:51:40 -0700 Subject: [PATCH 070/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 7c4b12835..094713cd1 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -70,7 +70,9 @@ class ClusterMetricsCoordinator( log.info("this is unassigned shards $unassignedShards") var cluster_status = cluster_health["status"].toString() log.info("this is cluster status $cluster_status") - val process_map = cluster_stats["nodes"] + val nodes_map = cluster_stats["nodes"] as Map + log.info("this is nodes map $nodes_map") + val process_map = nodes_map["process"] as Map log.info("this is process map $process_map") var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) From e0cb5bfa1208b566fba829519407b996f16739aa Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 16:00:08 -0700 Subject: [PATCH 071/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 094713cd1..c659d63d4 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -74,6 +74,9 @@ class ClusterMetricsCoordinator( log.info("this is nodes map $nodes_map") val process_map = nodes_map["process"] as Map log.info("this is process map $process_map") + val cpu_map = process_map["cpu"] as Map + val percent = cpu_map["percent"].toString() + log.info("THIS IS CPU USAGE $percent") var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) var unassigned_shards_data = ClusterMetricsDataPoint( From 71f21bc5bd32a373cc8555c3e0bdc111a91de5eb Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 16:19:08 -0700 Subject: [PATCH 072/215] PLS WORK Signed-off-by: Richard --- .../opensearch/alerting/util/ClusterMetricsCoordinator.kt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index c659d63d4..c8e3a0dd0 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -73,11 +73,17 @@ class ClusterMetricsCoordinator( val nodes_map = cluster_stats["nodes"] as Map log.info("this is nodes map $nodes_map") val process_map = nodes_map["process"] as Map - log.info("this is process map $process_map") val cpu_map = process_map["cpu"] as Map val percent = cpu_map["percent"].toString() log.info("THIS IS CPU USAGE $percent") + val jvm_map = nodes_map["jvm"] as Map + var mem_used = jvm_map["heap_used_in_bytes"] + var mem_avail = jvm_map["heap_max_in_bytes"] + log.info("mem_used type is ${mem_used!!::class.qualifiedName}") + log.info("mem_avail type is ${mem_avail!!::class.qualifiedName}") + + var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) var unassigned_shards_data = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, From 13762aeb12ae8dab610681b456b168ae768721c9 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 16:21:37 -0700 Subject: [PATCH 073/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index c8e3a0dd0..57625335f 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -76,14 +76,12 @@ class ClusterMetricsCoordinator( val cpu_map = process_map["cpu"] as Map val percent = cpu_map["percent"].toString() log.info("THIS IS CPU USAGE $percent") - val jvm_map = nodes_map["jvm"] as Map var mem_used = jvm_map["heap_used_in_bytes"] var mem_avail = jvm_map["heap_max_in_bytes"] log.info("mem_used type is ${mem_used!!::class.qualifiedName}") log.info("mem_avail type is ${mem_avail!!::class.qualifiedName}") - var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) var unassigned_shards_data = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, From ac98dc1017df1cee4d080283b40bba2a05a8d032 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 16:28:04 -0700 Subject: [PATCH 074/215] PLS WORK Signed-off-by: Richard --- .../opensearch/alerting/util/ClusterMetricsCoordinator.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 57625335f..80daa57b9 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -77,8 +77,9 @@ class ClusterMetricsCoordinator( val percent = cpu_map["percent"].toString() log.info("THIS IS CPU USAGE $percent") val jvm_map = nodes_map["jvm"] as Map - var mem_used = jvm_map["heap_used_in_bytes"] - var mem_avail = jvm_map["heap_max_in_bytes"] + val mem_map = jvm_map["mem"] as Map + var mem_used = mem_map["heap_used_in_bytes"] + var mem_avail = mem_map["heap_max_in_bytes"] log.info("mem_used type is ${mem_used!!::class.qualifiedName}") log.info("mem_avail type is ${mem_avail!!::class.qualifiedName}") From 83327f5564f472cc5dfe29054c6e057e638395c4 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 16:36:48 -0700 Subject: [PATCH 075/215] PLS WORK Signed-off-by: Richard --- .../opensearch/alerting/util/ClusterMetricsCoordinator.kt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 80daa57b9..8ce6b5a1d 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -80,8 +80,12 @@ class ClusterMetricsCoordinator( val mem_map = jvm_map["mem"] as Map var mem_used = mem_map["heap_used_in_bytes"] var mem_avail = mem_map["heap_max_in_bytes"] - log.info("mem_used type is ${mem_used!!::class.qualifiedName}") - log.info("mem_avail type is ${mem_avail!!::class.qualifiedName}") + val jvm_pressure = 0 + if (mem_used is Int && mem_avail is Int){ + val jvm_pressure = mem_used/mem_avail + } + + log.info("jvm_pressure is $jvm_pressure") var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) var unassigned_shards_data = ClusterMetricsDataPoint( From d434779bef1a5e69a193f5a535aab61f6a0f9e07 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 16:38:01 -0700 Subject: [PATCH 076/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 8ce6b5a1d..2fc12b708 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -81,8 +81,8 @@ class ClusterMetricsCoordinator( var mem_used = mem_map["heap_used_in_bytes"] var mem_avail = mem_map["heap_max_in_bytes"] val jvm_pressure = 0 - if (mem_used is Int && mem_avail is Int){ - val jvm_pressure = mem_used/mem_avail + if (mem_used is Int && mem_avail is Int) { + val jvm_pressure = mem_used / mem_avail } log.info("jvm_pressure is $jvm_pressure") From 7c147f5518aaa5debe7fb121057c9473d9a30751 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 16:43:01 -0700 Subject: [PATCH 077/215] PLS WORK Signed-off-by: Richard --- .../opensearch/alerting/util/ClusterMetricsCoordinator.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 2fc12b708..d4f3d6ba2 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -80,12 +80,12 @@ class ClusterMetricsCoordinator( val mem_map = jvm_map["mem"] as Map var mem_used = mem_map["heap_used_in_bytes"] var mem_avail = mem_map["heap_max_in_bytes"] - val jvm_pressure = 0 + if (mem_used is Int && mem_avail is Int) { - val jvm_pressure = mem_used / mem_avail + val jvm_pressure = mem_used.toDouble() / mem_avail.toDouble() + log.info("jvm pressure is $jvm_pressure") } - log.info("jvm_pressure is $jvm_pressure") var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) var unassigned_shards_data = ClusterMetricsDataPoint( From 726a7413f134af89a5655eb20534b10e9967e749 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 16:43:15 -0700 Subject: [PATCH 078/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index d4f3d6ba2..5c19f6fa0 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -86,7 +86,6 @@ class ClusterMetricsCoordinator( log.info("jvm pressure is $jvm_pressure") } - var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) var unassigned_shards_data = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, From f68bf37d568bf2bf3143a53df5602a60c08ac349 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 16:53:49 -0700 Subject: [PATCH 079/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsCoordinator.kt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 5c19f6fa0..bcc4e15f1 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -82,12 +82,15 @@ class ClusterMetricsCoordinator( var mem_avail = mem_map["heap_max_in_bytes"] if (mem_used is Int && mem_avail is Int) { - val jvm_pressure = mem_used.toDouble() / mem_avail.toDouble() + val jvm_pressure = ((mem_used.toDouble() / mem_avail.toDouble()) * 100).toString() log.info("jvm pressure is $jvm_pressure") + val jvm_pressure_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.JVM_PRESSURE, current_time, jvm_pressure) + val indexRequest_pressure = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(jvm_pressure_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) } - var cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) - var unassigned_shards_data = ClusterMetricsDataPoint( + val cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) + val unassigned_shards_data = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, current_time, unassignedShards From de6d485217c2f8d2de01323fe0f69ab517d0cd21 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 17:02:18 -0700 Subject: [PATCH 080/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsCoordinator.kt | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index bcc4e15f1..7f76e5397 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -81,34 +81,36 @@ class ClusterMetricsCoordinator( var mem_used = mem_map["heap_used_in_bytes"] var mem_avail = mem_map["heap_max_in_bytes"] - if (mem_used is Int && mem_avail is Int) { - val jvm_pressure = ((mem_used.toDouble() / mem_avail.toDouble()) * 100).toString() - log.info("jvm pressure is $jvm_pressure") - val jvm_pressure_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.JVM_PRESSURE, current_time, jvm_pressure) - val indexRequest_pressure = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(jvm_pressure_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - } - val cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) val unassigned_shards_data = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, current_time, unassignedShards ) + val cpu_usage_data = ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.CPU_USAGE, + current_time, + percent + ) val indexRequest_status = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) .source(cluster_status_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) val indexRequest_shards = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) .source(unassigned_shards_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + val indexRequest_cpu = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(cpu_usage_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) try { val indexResponse: IndexResponse = client.suspendUntil { client.index(indexRequest_status, it) } val indexResponse2: IndexResponse = client.suspendUntil { client.index(indexRequest_shards, it) } + val indexResponse3: IndexResponse = client.suspendUntil { client.index(indexRequest_cpu, it) } val failureReasons = checkShardsFailure(indexResponse) val failureReasons2 = checkShardsFailure(indexResponse2) - if (failureReasons != null || failureReasons2 != null) { + val failureReasons3 = checkShardsFailure(indexResponse3) + if (failureReasons != null || failureReasons2 != null || failureReasons3 != null) { log.info("richfu failed because $failureReasons") log.info("richfu failed because $failureReasons2") + log.info("richfu failed because $failureReasons3") return } } catch (t: Exception) { From 66f77fc4f3663c5559ae3cc9af44cd5edf2a5cb1 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 17:49:36 -0700 Subject: [PATCH 081/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 7f76e5397..ce5fb494a 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -63,11 +63,10 @@ class ClusterMetricsCoordinator( val cluster_health = client.admin().cluster().health(ClusterHealthRequest()).get().toMap() // cluster stats for cluster status (health), CPU usage, JVM pressure var cluster_stats = client.admin().cluster().clusterStats(ClusterStatsRequest()).get().toMap() - log.info("cluster stats why not work? $cluster_stats") + ClusterMetricsVisualizationIndex.initFunc(client, clusterService) var unassignedShards = cluster_health["unassigned_shards"].toString() - log.info("this is unassigned shards $unassignedShards") var cluster_status = cluster_health["status"].toString() log.info("this is cluster status $cluster_status") val nodes_map = cluster_stats["nodes"] as Map From b5b2fce04f1c887cff8a01e3c30547964b0f0ebb Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 13 Jul 2022 17:54:00 -0700 Subject: [PATCH 082/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index ce5fb494a..84026ce31 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -67,10 +67,10 @@ class ClusterMetricsCoordinator( ClusterMetricsVisualizationIndex.initFunc(client, clusterService) var unassignedShards = cluster_health["unassigned_shards"].toString() + log.info("this is unassigned shards $unassignedShards") var cluster_status = cluster_health["status"].toString() log.info("this is cluster status $cluster_status") val nodes_map = cluster_stats["nodes"] as Map - log.info("this is nodes map $nodes_map") val process_map = nodes_map["process"] as Map val cpu_map = process_map["cpu"] as Map val percent = cpu_map["percent"].toString() From 1cd3c6fe3c9e08d150bc242c1cbd6999787ef256 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 14 Jul 2022 11:33:17 -0700 Subject: [PATCH 083/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsCoordinator.kt | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 84026ce31..8a9b7c5b4 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -77,8 +77,13 @@ class ClusterMetricsCoordinator( log.info("THIS IS CPU USAGE $percent") val jvm_map = nodes_map["jvm"] as Map val mem_map = jvm_map["mem"] as Map - var mem_used = mem_map["heap_used_in_bytes"] - var mem_avail = mem_map["heap_max_in_bytes"] + val mem_used = mem_map["heap_used_in_bytes"] + val mem_avail = mem_map["heap_max_in_bytes"] + var jvm_pressure = "0.0" + + if (mem_used is Int && mem_avail is Int) { + jvm_pressure = ((mem_used.toDouble() / mem_avail.toDouble()) * 100).toString() + } val cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) val unassigned_shards_data = ClusterMetricsDataPoint( @@ -91,6 +96,11 @@ class ClusterMetricsCoordinator( current_time, percent ) + val jvm_data = ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.JVM_PRESSURE, + current_time, + jvm_pressure + ) val indexRequest_status = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) .source(cluster_status_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) @@ -98,18 +108,23 @@ class ClusterMetricsCoordinator( .source(unassigned_shards_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) val indexRequest_cpu = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) .source(cpu_usage_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + val indexRequest_jvm = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(jvm_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) try { val indexResponse: IndexResponse = client.suspendUntil { client.index(indexRequest_status, it) } val indexResponse2: IndexResponse = client.suspendUntil { client.index(indexRequest_shards, it) } val indexResponse3: IndexResponse = client.suspendUntil { client.index(indexRequest_cpu, it) } + val indexResponse4: IndexResponse = client.suspendUntil { client.index(indexRequest_jvm, it) } val failureReasons = checkShardsFailure(indexResponse) val failureReasons2 = checkShardsFailure(indexResponse2) val failureReasons3 = checkShardsFailure(indexResponse3) - if (failureReasons != null || failureReasons2 != null || failureReasons3 != null) { + val failureReasons4 = checkShardsFailure(indexResponse4) + if (failureReasons != null || failureReasons2 != null || failureReasons3 != null || failureReasons4 != null) { log.info("richfu failed because $failureReasons") log.info("richfu failed because $failureReasons2") log.info("richfu failed because $failureReasons3") + log.info("richfu failed because $failureReasons4") return } } catch (t: Exception) { From 54ea8b0e614a69b287b1602c8ccf3a8cf599f511 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 14 Jul 2022 11:36:40 -0700 Subject: [PATCH 084/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 8a9b7c5b4..edee4bbd9 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -84,6 +84,7 @@ class ClusterMetricsCoordinator( if (mem_used is Int && mem_avail is Int) { jvm_pressure = ((mem_used.toDouble() / mem_avail.toDouble()) * 100).toString() } + log.info("THIS IS JVM PRESSURE $jvm_pressure") val cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) val unassigned_shards_data = ClusterMetricsDataPoint( From a0d96fe54c1385f8d2c4672c1695dd7ac69f59b0 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 14 Jul 2022 11:42:50 -0700 Subject: [PATCH 085/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index edee4bbd9..68d4532f2 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -82,7 +82,8 @@ class ClusterMetricsCoordinator( var jvm_pressure = "0.0" if (mem_used is Int && mem_avail is Int) { - jvm_pressure = ((mem_used.toDouble() / mem_avail.toDouble()) * 100).toString() + val jvm_pressure_num = ((mem_used.toDouble() / mem_avail.toDouble()) * 100) + jvm_pressure = String.format("%.2f", jvm_pressure_num) } log.info("THIS IS JVM PRESSURE $jvm_pressure") From ea8ac5ac151db3a216c7898d4d56fafa28c78a82 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 14 Jul 2022 11:59:28 -0700 Subject: [PATCH 086/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 68d4532f2..dcd18d773 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -45,7 +45,7 @@ class ClusterMetricsCoordinator( destinationHelper(client as NodeClient, clusterService) } } - threadPool.scheduleWithFixedDelay(scheduledJob, TimeValue.timeValueMinutes(1), ThreadPool.Names.SYSTEM_WRITE) + threadPool.scheduleWithFixedDelay(scheduledJob, TimeValue.timeValueMinutes(5), ThreadPool.Names.SYSTEM_WRITE) } suspend fun destinationHelper(client: NodeClient, clusterService: ClusterService) { /* From 80d3ff91556bf84c9d285d5b8b195ba5a008b7ca Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 14 Jul 2022 12:09:43 -0700 Subject: [PATCH 087/215] PLS WORK Signed-off-by: Richard --- .../src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index c48b5c194..54a9ea94b 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -249,7 +249,6 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R // create a cluster metrics visualization index upon initialization of alerting plugin // clusterMetricsVisualization = ClusterMetricsVisualizationIndex(client, clusterService, threadPool) // ClusterMetricsVisualizationIndex(client, clusterService) - log.info("YEP MESSAGE $client") return listOf( sweeper, scheduler, From 16cf578dc50d6195bc9c33fc5f91f8183e61dfa1 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 19 Jul 2022 11:11:46 -0700 Subject: [PATCH 088/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsCoordinator.kt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index dcd18d773..d9327db93 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -31,6 +31,11 @@ class ClusterMetricsCoordinator( private val threadPool: ThreadPool ) : ClusterStateListener, CoroutineScope, LifecycleListener() { + companion object { + var isRunningFlag = false + internal set + } + override val coroutineContext: CoroutineContext get() = Dispatchers.Default + CoroutineName("ClusterMetricsCoordinator") @@ -45,7 +50,10 @@ class ClusterMetricsCoordinator( destinationHelper(client as NodeClient, clusterService) } } - threadPool.scheduleWithFixedDelay(scheduledJob, TimeValue.timeValueMinutes(5), ThreadPool.Names.SYSTEM_WRITE) + if (event!!.localNodeMaster() && !isRunningFlag) { + isRunningFlag = true + threadPool.scheduleWithFixedDelay(scheduledJob, TimeValue.timeValueMinutes(5), ThreadPool.Names.SYSTEM_WRITE) + } } suspend fun destinationHelper(client: NodeClient, clusterService: ClusterService) { /* From fbf74b6a688d626903aef8c285c388a896606329 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 19 Jul 2022 11:19:18 -0700 Subject: [PATCH 089/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index d9327db93..fd2c634cb 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -51,8 +51,8 @@ class ClusterMetricsCoordinator( } } if (event!!.localNodeMaster() && !isRunningFlag) { + threadPool.scheduleWithFixedDelay(scheduledJob, TimeValue.timeValueMinutes(1), ThreadPool.Names.SYSTEM_WRITE) isRunningFlag = true - threadPool.scheduleWithFixedDelay(scheduledJob, TimeValue.timeValueMinutes(5), ThreadPool.Names.SYSTEM_WRITE) } } suspend fun destinationHelper(client: NodeClient, clusterService: ClusterService) { From d204b4799c3c8143a34916e3747a8f3f4b4d6ff1 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 19 Jul 2022 13:02:38 -0700 Subject: [PATCH 090/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsCoordinator.kt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index fd2c634cb..ffba4eac8 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -32,6 +32,7 @@ class ClusterMetricsCoordinator( ) : ClusterStateListener, CoroutineScope, LifecycleListener() { companion object { + @Volatile var isRunningFlag = false internal set } @@ -87,7 +88,7 @@ class ClusterMetricsCoordinator( val mem_map = jvm_map["mem"] as Map val mem_used = mem_map["heap_used_in_bytes"] val mem_avail = mem_map["heap_max_in_bytes"] - var jvm_pressure = "0.0" + var jvm_pressure = "0.00" if (mem_used is Int && mem_avail is Int) { val jvm_pressure_num = ((mem_used.toDouble() / mem_avail.toDouble()) * 100) @@ -95,7 +96,11 @@ class ClusterMetricsCoordinator( } log.info("THIS IS JVM PRESSURE $jvm_pressure") - val cluster_status_data = ClusterMetricsDataPoint(ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, cluster_status) + val cluster_status_data = ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, + current_time, + cluster_status + ) val unassigned_shards_data = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, current_time, From 8df0d141ef190817970e89edafbcf21b104b7454 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 22 Jul 2022 15:19:30 -0700 Subject: [PATCH 091/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsCoordinator.kt | 1 + .../util/ClusterMetricsVisualizationIndex.kt | 3 +- .../mappings/metrics-visualizations.json | 50 ++++++++++++++++++- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index ffba4eac8..612713af8 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -67,6 +67,7 @@ class ClusterMetricsCoordinator( call index API to index the document */ val current_time = Instant.now().toString() +// val curTime = current_time. log.info("richfu THIS IS THE TIME $current_time") // cluster health for unassigned shards val cluster_health = client.admin().cluster().health(ClusterHealthRequest()).get().toMap() diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 1bb57a7e3..4970530bd 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -24,7 +24,7 @@ class ClusterMetricsVisualizationIndex( companion object { /** The index name pattern for all cluster metric visualizations indices */ val CLUSTER_METRIC_VISUALIZATION_INDEX = ".opendistro-alerting-cluster-metrics" - val METRICS_HISTORY_ENABLED = Setting.boolSetting("opendistro.alerting.metrics_history_enabled", true) +// val METRICS_HISTORY_ENABLED = Setting.boolSetting("opendistro.alerting.metrics_history_enabled", true) val METRICS_STORE_TIME = Setting.positiveTimeSetting( "opendistro.alerting.metrics_history_max_age", TimeValue(7, TimeUnit.DAYS), @@ -38,7 +38,6 @@ class ClusterMetricsVisualizationIndex( suspend fun initFunc(client: Client, clusterService: ClusterService) { if (!clusterMetricsVisualizationIndexExists(clusterService)) { val indexRequest = CreateIndexRequest(CLUSTER_METRIC_VISUALIZATION_INDEX) - .mapping(clusterMetricsVisualizationsMappings()) .settings( Settings.builder().put("index.hidden", true) .build() diff --git a/core/src/main/resources/mappings/metrics-visualizations.json b/core/src/main/resources/mappings/metrics-visualizations.json index 3faba2ab7..637d8cdfb 100644 --- a/core/src/main/resources/mappings/metrics-visualizations.json +++ b/core/src/main/resources/mappings/metrics-visualizations.json @@ -3,7 +3,55 @@ "schema_version": 1 }, "properties": { - "metric": { + "cluster_status": { + "properties": { + "timestamp": { + "type": "date" + }, + "value" : { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "cpu_usage": { + "properties": { + "timestamp": { + "type": "date" + }, + "value" : { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "jvm_pressure": { + "properties": { + "timestamp": { + "type": "date" + }, + "value" : { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "unassigned_shards": { "properties": { "timestamp": { "type": "date" From 28e160dfa64fea6f65b096ddb69115ce48448ff4 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 22 Jul 2022 15:24:38 -0700 Subject: [PATCH 092/215] PLS WORK Signed-off-by: Richard --- .../opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 4970530bd..ed2fc9f25 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -38,6 +38,7 @@ class ClusterMetricsVisualizationIndex( suspend fun initFunc(client: Client, clusterService: ClusterService) { if (!clusterMetricsVisualizationIndexExists(clusterService)) { val indexRequest = CreateIndexRequest(CLUSTER_METRIC_VISUALIZATION_INDEX) + .mapping(clusterMetricsVisualizationsMappings()) .settings( Settings.builder().put("index.hidden", true) .build() From 300dfc3eac9cca3ca186a7bf03c0b9f3d5e46c82 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 25 Jul 2022 11:22:40 -0700 Subject: [PATCH 093/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsCoordinator.kt | 65 +++++++++++++++++++ .../util/ClusterMetricsVisualizationIndex.kt | 9 --- 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 612713af8..933c74fe1 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -4,6 +4,7 @@ import kotlinx.coroutines.CoroutineName import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch +import org.opensearch.action.ActionListener import org.opensearch.action.admin.cluster.health.ClusterHealthRequest import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest import org.opensearch.action.index.IndexRequest @@ -19,6 +20,10 @@ import org.opensearch.common.component.LifecycleListener import org.opensearch.common.unit.TimeValue import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.XContentFactory +import org.opensearch.index.query.QueryBuilders +import org.opensearch.index.reindex.BulkByScrollResponse +import org.opensearch.index.reindex.DeleteByQueryAction +import org.opensearch.index.reindex.DeleteByQueryRequestBuilder import org.opensearch.threadpool.ThreadPool import java.time.Instant import kotlin.coroutines.CoroutineContext @@ -49,6 +54,7 @@ class ClusterMetricsCoordinator( val scheduledJob = Runnable { launch { destinationHelper(client as NodeClient, clusterService) + deleteDocs(client) } } if (event!!.localNodeMaster() && !isRunningFlag) { @@ -158,4 +164,63 @@ class ClusterMetricsCoordinator( } return null } + + fun deleteDocs(client: NodeClient) { + + DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) + .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now-10m/m")) + .execute( + object : ActionListener { + override fun onResponse(response: BulkByScrollResponse) { + } + + override fun onFailure(t: Exception) { + } + } + ) + log.info("deleted cluster_status data") + + DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) + .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .filter(QueryBuilders.rangeQuery("cpu_usage.timestamp").lte("now-10m/m")) + .execute( + object : ActionListener { + override fun onResponse(response: BulkByScrollResponse) { + } + + override fun onFailure(t: Exception) { + } + } + ) + log.info("deleted cpu_usage data") + + DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) + .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .filter(QueryBuilders.rangeQuery("jvm_pressure.timestamp").lte("now-10m/m")) + .execute( + object : ActionListener { + override fun onResponse(response: BulkByScrollResponse) { + } + + override fun onFailure(t: Exception) { + } + } + ) + log.info("deleted jvm_pressure data") + + DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) + .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .filter(QueryBuilders.rangeQuery("unassigned_shards.timestamp").lte("now-10m/m")) + .execute( + object : ActionListener { + override fun onResponse(response: BulkByScrollResponse) { + } + + override fun onFailure(t: Exception) { + } + } + ) + log.info("deleted unassigned_shards data") + } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index ed2fc9f25..5a07e3483 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -7,11 +7,8 @@ import org.opensearch.action.admin.indices.create.CreateIndexResponse import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.client.Client import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.settings.Setting import org.opensearch.common.settings.Settings -import org.opensearch.common.unit.TimeValue import org.opensearch.threadpool.ThreadPool -import java.util.concurrent.TimeUnit private val log = LogManager.getLogger(ClusterMetricsVisualizationIndex::class.java) @@ -24,12 +21,6 @@ class ClusterMetricsVisualizationIndex( companion object { /** The index name pattern for all cluster metric visualizations indices */ val CLUSTER_METRIC_VISUALIZATION_INDEX = ".opendistro-alerting-cluster-metrics" -// val METRICS_HISTORY_ENABLED = Setting.boolSetting("opendistro.alerting.metrics_history_enabled", true) - val METRICS_STORE_TIME = Setting.positiveTimeSetting( - "opendistro.alerting.metrics_history_max_age", - TimeValue(7, TimeUnit.DAYS), - Setting.Property.NodeScope, Setting.Property.Dynamic - ) @JvmStatic fun clusterMetricsVisualizationsMappings(): String { From a02f49be04fd0167bd2224c0ef1c4dc395b5c90c Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 25 Jul 2022 11:43:20 -0700 Subject: [PATCH 094/215] PLS WORK Signed-off-by: Richard --- .../opensearch/alerting/util/ClusterMetricsCoordinator.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 933c74fe1..5a9b7f1d4 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -179,7 +179,7 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted cluster_status data") + log.info("deleted cluster_status data from 10 minutes ago") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) @@ -193,7 +193,7 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted cpu_usage data") + log.info("deleted cpu_usage data from 10 minutes ago") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) @@ -207,7 +207,7 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted jvm_pressure data") + log.info("deleted jvm_pressure data from 10 minutes ago") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) @@ -221,6 +221,6 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted unassigned_shards data") + log.info("deleted unassigned_shards data from 10 minutes ago") } } From 8829633fd5815ae65a709c347a7275fab8cc16dd Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 25 Jul 2022 14:20:58 -0700 Subject: [PATCH 095/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 5a9b7f1d4..7243fa3a9 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -197,7 +197,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("jvm_pressure.timestamp").lte("now-10m/m")) + .filter(QueryBuilders.rangeQuery("jvm_pressure.timestamp").lte("now-15m/m")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { From 46daa7d9d45f8c8f0ae2b035e7df284161ea448b Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 25 Jul 2022 14:57:34 -0700 Subject: [PATCH 096/215] PLS WORK Signed-off-by: Richard --- .../kotlin/org/opensearch/alerting/AlertingPlugin.kt | 4 +++- .../opensearch/alerting/settings/AlertingSettings.kt | 12 ++++++++++++ .../alerting/util/ClusterMetricsCoordinator.kt | 6 +++--- .../util/ClusterMetricsVisualizationIndex.kt | 2 ++ 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 54a9ea94b..6803c6dc0 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -319,7 +319,9 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R AlertingSettings.FINDING_HISTORY_MAX_DOCS, AlertingSettings.FINDING_HISTORY_INDEX_MAX_AGE, AlertingSettings.FINDING_HISTORY_ROLLOVER_PERIOD, - AlertingSettings.FINDING_HISTORY_RETENTION_PERIOD + AlertingSettings.FINDING_HISTORY_RETENTION_PERIOD, + AlertingSettings.METRICS_STORE_TIME, + AlertingSettings.METRICS_EXECUTION_FREQUENCY ) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index 1268703c9..5965b18e2 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -156,5 +156,17 @@ class AlertingSettings { -1L, Setting.Property.NodeScope, Setting.Property.Dynamic ) + + val METRICS_STORE_TIME = Setting.positiveTimeSetting( + "plugins.alerting.cluster_metrics.metrics_history_max_age", + TimeValue(7, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val METRICS_EXECUTION_FREQUENCY = Setting.positiveTimeSetting( + "plugins.alerting.cluster_metrics.execution_frequency", + TimeValue(1, TimeUnit.MINUTES), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 7243fa3a9..eac3b4597 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -169,7 +169,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now-10m/m")) + .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now-15m/m")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -183,7 +183,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("cpu_usage.timestamp").lte("now-10m/m")) + .filter(QueryBuilders.rangeQuery("cpu_usage.timestamp").lte("now-15m/m")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -211,7 +211,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("unassigned_shards.timestamp").lte("now-10m/m")) + .filter(QueryBuilders.rangeQuery("unassigned_shards.timestamp").lte("now-15m/m")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 5a07e3483..9a6715b72 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -7,7 +7,9 @@ import org.opensearch.action.admin.indices.create.CreateIndexResponse import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.client.Client import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.settings.Setting import org.opensearch.common.settings.Settings +import org.opensearch.common.unit.TimeValue import org.opensearch.threadpool.ThreadPool private val log = LogManager.getLogger(ClusterMetricsVisualizationIndex::class.java) From 5fa458e25b3c58849f841e39dfe4241a10726ce6 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 25 Jul 2022 15:20:06 -0700 Subject: [PATCH 097/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/AlertingPlugin.kt | 2 +- .../alerting/util/ClusterMetricsCoordinator.kt | 17 ++++++++++++----- .../util/ClusterMetricsVisualizationIndex.kt | 2 -- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 6803c6dc0..e63a8eba0 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -242,7 +242,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R scheduler = JobScheduler(threadPool, runner) sweeper = JobSweeper(environment.settings(), client, clusterService, threadPool, xContentRegistry, scheduler, ALERTING_JOB_TYPES) destinationMigrationCoordinator = DestinationMigrationCoordinator(client, clusterService, threadPool, scheduledJobIndices) - clusterMetricsCoordinator = ClusterMetricsCoordinator(client, clusterService, threadPool) + clusterMetricsCoordinator = ClusterMetricsCoordinator(settings, client, clusterService, threadPool) // Create Monitor using Monitor.kt, and then feed the request to client in some way this.threadPool = threadPool this.clusterService = clusterService diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index eac3b4597..c636302c2 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -11,12 +11,15 @@ import org.opensearch.action.index.IndexRequest import org.opensearch.action.index.IndexResponse import org.opensearch.alerting.model.ClusterMetricsDataPoint import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.alerting.settings.AlertingSettings.Companion.METRICS_EXECUTION_FREQUENCY +import org.opensearch.alerting.settings.AlertingSettings.Companion.METRICS_STORE_TIME import org.opensearch.client.Client import org.opensearch.client.node.NodeClient import org.opensearch.cluster.ClusterChangedEvent import org.opensearch.cluster.ClusterStateListener import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener +import org.opensearch.common.settings.Settings import org.opensearch.common.unit.TimeValue import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.XContentFactory @@ -27,10 +30,12 @@ import org.opensearch.index.reindex.DeleteByQueryRequestBuilder import org.opensearch.threadpool.ThreadPool import java.time.Instant import kotlin.coroutines.CoroutineContext +import kotlin.reflect.typeOf private val log = org.apache.logging.log4j.LogManager.getLogger(ClusterMetricsCoordinator::class.java) class ClusterMetricsCoordinator( + private val settings: Settings, private val client: Client, private val clusterService: ClusterService, private val threadPool: ThreadPool @@ -58,7 +63,7 @@ class ClusterMetricsCoordinator( } } if (event!!.localNodeMaster() && !isRunningFlag) { - threadPool.scheduleWithFixedDelay(scheduledJob, TimeValue.timeValueMinutes(1), ThreadPool.Names.SYSTEM_WRITE) + threadPool.scheduleWithFixedDelay(scheduledJob, METRICS_EXECUTION_FREQUENCY.get(settings), ThreadPool.Names.SYSTEM_WRITE) isRunningFlag = true } } @@ -166,10 +171,12 @@ class ClusterMetricsCoordinator( } fun deleteDocs(client: NodeClient) { + val documentAge = METRICS_STORE_TIME.get(settings) + log.info("documentAge returns $documentAge") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now-15m/m")) + .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now - $documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -183,7 +190,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("cpu_usage.timestamp").lte("now-15m/m")) + .filter(QueryBuilders.rangeQuery("cpu_usage.timestamp").lte("now - $documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -197,7 +204,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("jvm_pressure.timestamp").lte("now-15m/m")) + .filter(QueryBuilders.rangeQuery("jvm_pressure.timestamp").lte("now - $documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -211,7 +218,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("unassigned_shards.timestamp").lte("now-15m/m")) + .filter(QueryBuilders.rangeQuery("unassigned_shards.timestamp").lte("now - $documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 9a6715b72..5a07e3483 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -7,9 +7,7 @@ import org.opensearch.action.admin.indices.create.CreateIndexResponse import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.client.Client import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.settings.Setting import org.opensearch.common.settings.Settings -import org.opensearch.common.unit.TimeValue import org.opensearch.threadpool.ThreadPool private val log = LogManager.getLogger(ClusterMetricsVisualizationIndex::class.java) From 30a7effdeac95e39495fc7100152a8eebdcf1870 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 25 Jul 2022 15:21:00 -0700 Subject: [PATCH 098/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index c636302c2..b270bcc32 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -20,7 +20,6 @@ import org.opensearch.cluster.ClusterStateListener import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener import org.opensearch.common.settings.Settings -import org.opensearch.common.unit.TimeValue import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.XContentFactory import org.opensearch.index.query.QueryBuilders @@ -30,7 +29,6 @@ import org.opensearch.index.reindex.DeleteByQueryRequestBuilder import org.opensearch.threadpool.ThreadPool import java.time.Instant import kotlin.coroutines.CoroutineContext -import kotlin.reflect.typeOf private val log = org.apache.logging.log4j.LogManager.getLogger(ClusterMetricsCoordinator::class.java) From f5a45318116613a2218d1d3eed3474f412c9659f Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 25 Jul 2022 15:51:14 -0700 Subject: [PATCH 099/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsCoordinator.kt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index b270bcc32..375b11b9f 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -171,10 +171,10 @@ class ClusterMetricsCoordinator( fun deleteDocs(client: NodeClient) { val documentAge = METRICS_STORE_TIME.get(settings) log.info("documentAge returns $documentAge") - + val unitTime = documentAge.toString().last() DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now - $documentAge")) + .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now - $documentAge/$unitTime")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -184,11 +184,11 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted cluster_status data from 10 minutes ago") + log.info("deleted cluster_status data from $documentAge ago") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("cpu_usage.timestamp").lte("now - $documentAge")) + .filter(QueryBuilders.rangeQuery("cpu_usage.timestamp").lte("now - $documentAge/$unitTime")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -198,11 +198,11 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted cpu_usage data from 10 minutes ago") + log.info("deleted cpu_usage data from $documentAge ago") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("jvm_pressure.timestamp").lte("now - $documentAge")) + .filter(QueryBuilders.rangeQuery("jvm_pressure.timestamp").lte("now - $documentAge/$unitTime")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -212,11 +212,11 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted jvm_pressure data from 10 minutes ago") + log.info("deleted jvm_pressure data from $documentAge ago") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("unassigned_shards.timestamp").lte("now - $documentAge")) + .filter(QueryBuilders.rangeQuery("unassigned_shards.timestamp").lte("now - $documentAge/$unitTime")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -226,6 +226,6 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted unassigned_shards data from 10 minutes ago") + log.info("deleted unassigned_shards data from $documentAge ago") } } From 538a4b2eecd23d6bfa13a031c9adb1042bcdf72d Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 25 Jul 2022 16:36:04 -0700 Subject: [PATCH 100/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsCoordinator.kt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 375b11b9f..2cf5e7887 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -9,8 +9,10 @@ import org.opensearch.action.admin.cluster.health.ClusterHealthRequest import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest import org.opensearch.action.index.IndexRequest import org.opensearch.action.index.IndexResponse +import org.opensearch.alerting.core.settings.ScheduledJobSettings import org.opensearch.alerting.model.ClusterMetricsDataPoint import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.settings.AlertingSettings.Companion.METRICS_EXECUTION_FREQUENCY import org.opensearch.alerting.settings.AlertingSettings.Companion.METRICS_STORE_TIME import org.opensearch.client.Client @@ -39,6 +41,8 @@ class ClusterMetricsCoordinator( private val threadPool: ThreadPool ) : ClusterStateListener, CoroutineScope, LifecycleListener() { + @Volatile private var metricsExecutionFrequency = METRICS_EXECUTION_FREQUENCY.get(settings) + @Volatile private var metricsStoreTime = METRICS_STORE_TIME.get(settings) companion object { @Volatile var isRunningFlag = false @@ -51,6 +55,8 @@ class ClusterMetricsCoordinator( init { clusterService.addListener(this) clusterService.addLifecycleListener(this) + clusterService.clusterSettings.addSettingsUpdateConsumer(METRICS_EXECUTION_FREQUENCY) { metricsExecutionFrequency = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(METRICS_STORE_TIME) { metricsStoreTime = it } } override fun clusterChanged(event: ClusterChangedEvent?) { @@ -170,8 +176,9 @@ class ClusterMetricsCoordinator( fun deleteDocs(client: NodeClient) { val documentAge = METRICS_STORE_TIME.get(settings) - log.info("documentAge returns $documentAge") val unitTime = documentAge.toString().last() + log.info("documentAge returns $documentAge, unitTime returns $unitTime") + DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now - $documentAge/$unitTime")) From 7dcd7ee48769e5b9d7a19a4a5520690b37c2a64d Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 25 Jul 2022 16:37:07 -0700 Subject: [PATCH 101/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 2cf5e7887..05a6ec7db 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -9,10 +9,8 @@ import org.opensearch.action.admin.cluster.health.ClusterHealthRequest import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest import org.opensearch.action.index.IndexRequest import org.opensearch.action.index.IndexResponse -import org.opensearch.alerting.core.settings.ScheduledJobSettings import org.opensearch.alerting.model.ClusterMetricsDataPoint import org.opensearch.alerting.opensearchapi.suspendUntil -import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.settings.AlertingSettings.Companion.METRICS_EXECUTION_FREQUENCY import org.opensearch.alerting.settings.AlertingSettings.Companion.METRICS_STORE_TIME import org.opensearch.client.Client From 8377c8f4d481372003ea85b63b9ce771fb9e693c Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 25 Jul 2022 17:46:31 -0700 Subject: [PATCH 102/215] PLS WORK Signed-off-by: Richard --- .../alerting/model/ClusterMetricsDataPoint.kt | 9 +++ .../alerting/settings/AlertingSettings.kt | 2 +- .../util/ClusterMetricsCoordinator.kt | 62 ++++++++++++++++--- .../mappings/metrics-visualizations.json | 48 ++++++++++++++ 4 files changed, 111 insertions(+), 10 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt index 6afa3a9fa..337de91b8 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt @@ -37,6 +37,15 @@ data class ClusterMetricsDataPoint( ), UNASSIGNED_SHARDS( "unassigned_shards" + ), + NUM_PENDING_TASKS( + "number_of_pending_tasks" + ), + ACTIVE_SHARDS( + "active_shards" + ), + RELOCATING_SHARDS( + "relocating_shards" ) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index 5965b18e2..b5d52fc44 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -159,7 +159,7 @@ class AlertingSettings { val METRICS_STORE_TIME = Setting.positiveTimeSetting( "plugins.alerting.cluster_metrics.metrics_history_max_age", - TimeValue(7, TimeUnit.DAYS), + TimeValue(10, TimeUnit.MINUTES), Setting.Property.NodeScope, Setting.Property.Dynamic ) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 05a6ec7db..fdc2d1fca 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -89,10 +89,16 @@ class ClusterMetricsCoordinator( ClusterMetricsVisualizationIndex.initFunc(client, clusterService) - var unassignedShards = cluster_health["unassigned_shards"].toString() + val unassignedShards = cluster_health["unassigned_shards"].toString() log.info("this is unassigned shards $unassignedShards") - var cluster_status = cluster_health["status"].toString() - log.info("this is cluster status $cluster_status") + val clusterStatus = cluster_health["status"].toString() + log.info("this is cluster status $clusterStatus") + val numPending = cluster_health["number_of_pending_tasks"].toString() + log.info("this is number of pending tasks $numPending") + val activeShards = cluster_health["active_shards"].toString() + log.info("this is number of active shards $activeShards") + val relocatingShards = cluster_health["relocating_shards"].toString() + log.info("This is number of relocatingShards $relocatingShards") val nodes_map = cluster_stats["nodes"] as Map val process_map = nodes_map["process"] as Map val cpu_map = process_map["cpu"] as Map @@ -110,10 +116,10 @@ class ClusterMetricsCoordinator( } log.info("THIS IS JVM PRESSURE $jvm_pressure") - val cluster_status_data = ClusterMetricsDataPoint( + val clusterStatus_data = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, - cluster_status + clusterStatus ) val unassigned_shards_data = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, @@ -130,30 +136,68 @@ class ClusterMetricsCoordinator( current_time, jvm_pressure ) + val pendingTasksData = ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.NUM_PENDING_TASKS, + current_time, + numPending + ) + val activeShardsData = ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.ACTIVE_SHARDS, + current_time, + activeShards + ) + val relocatingShardsData = ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.RELOCATING_SHARDS, + current_time, + relocatingShards + ) val indexRequest_status = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(cluster_status_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + .source(clusterStatus_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) val indexRequest_shards = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) .source(unassigned_shards_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) val indexRequest_cpu = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) .source(cpu_usage_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) val indexRequest_jvm = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) .source(jvm_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + val indexRequest_pending = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(pendingTasksData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + val indexRequest_active = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(activeShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + val indexRequest_relocating = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(relocatingShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) try { val indexResponse: IndexResponse = client.suspendUntil { client.index(indexRequest_status, it) } val indexResponse2: IndexResponse = client.suspendUntil { client.index(indexRequest_shards, it) } val indexResponse3: IndexResponse = client.suspendUntil { client.index(indexRequest_cpu, it) } val indexResponse4: IndexResponse = client.suspendUntil { client.index(indexRequest_jvm, it) } + val indexResponse5: IndexResponse = client.suspendUntil { client.index(indexRequest_pending, it) } + val indexResponse6: IndexResponse = client.suspendUntil { client.index(indexRequest_active, it) } + val indexResponse7: IndexResponse = client.suspendUntil { client.index(indexRequest_relocating, it) } val failureReasons = checkShardsFailure(indexResponse) val failureReasons2 = checkShardsFailure(indexResponse2) val failureReasons3 = checkShardsFailure(indexResponse3) val failureReasons4 = checkShardsFailure(indexResponse4) - if (failureReasons != null || failureReasons2 != null || failureReasons3 != null || failureReasons4 != null) { + val failureReasons5 = checkShardsFailure(indexResponse5) + val failureReasons6 = checkShardsFailure(indexResponse6) + val failureReasons7 = checkShardsFailure(indexResponse7) + if ( + failureReasons != null || + failureReasons2 != null || + failureReasons3 != null || + failureReasons4 != null || + failureReasons5 != null || + failureReasons6 != null || + failureReasons7 != null + ) { log.info("richfu failed because $failureReasons") log.info("richfu failed because $failureReasons2") log.info("richfu failed because $failureReasons3") log.info("richfu failed because $failureReasons4") + log.info("richfu failed because $failureReasons5") + log.info("richfu failed because $failureReasons6") + log.info("richfu failed because $failureReasons7") return } } catch (t: Exception) { @@ -179,7 +223,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now - $documentAge/$unitTime")) + .filter(QueryBuilders.rangeQuery("clusterStatus.timestamp").lte("now - $documentAge/$unitTime")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -189,7 +233,7 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted cluster_status data from $documentAge ago") + log.info("deleted clusterStatus data from $documentAge ago") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) diff --git a/core/src/main/resources/mappings/metrics-visualizations.json b/core/src/main/resources/mappings/metrics-visualizations.json index 637d8cdfb..b9a5f1552 100644 --- a/core/src/main/resources/mappings/metrics-visualizations.json +++ b/core/src/main/resources/mappings/metrics-visualizations.json @@ -66,6 +66,54 @@ } } } + }, + "number_of_pending_tasks": { + "properties": { + "timestamp": { + "type": "date" + }, + "value" : { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "active_shards": { + "properties": { + "timestamp": { + "type": "date" + }, + "value" : { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "relocating_shards": { + "properties": { + "timestamp": { + "type": "date" + }, + "value" : { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } } } } \ No newline at end of file From 6546a917d6c8b81e62c6396bfbf0c2caa51a295f Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 25 Jul 2022 17:53:19 -0700 Subject: [PATCH 103/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsCoordinator.kt | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index fdc2d1fca..da5751d99 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -276,5 +276,47 @@ class ClusterMetricsCoordinator( } ) log.info("deleted unassigned_shards data from $documentAge ago") + + DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) + .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .filter(QueryBuilders.rangeQuery("number_of_pending_tasks.timestamp").lte("now - $documentAge/$unitTime")) + .execute( + object : ActionListener { + override fun onResponse(response: BulkByScrollResponse) { + } + + override fun onFailure(t: Exception) { + } + } + ) + log.info("deleted number of pending tasks from $documentAge ago") + + DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) + .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .filter(QueryBuilders.rangeQuery("active_shards.timestamp").lte("now - $documentAge/$unitTime")) + .execute( + object : ActionListener { + override fun onResponse(response: BulkByScrollResponse) { + } + + override fun onFailure(t: Exception) { + } + } + ) + log.info("deleted active shards from $documentAge ago") + + DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) + .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .filter(QueryBuilders.rangeQuery("relocating_shards.timestamp").lte("now - $documentAge/$unitTime")) + .execute( + object : ActionListener { + override fun onResponse(response: BulkByScrollResponse) { + } + + override fun onFailure(t: Exception) { + } + } + ) + log.info("deleted number of relocating shards from $documentAge ago") } } From f661029f20d77ef65241dcba3ef134e748590a85 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 25 Jul 2022 18:08:06 -0700 Subject: [PATCH 104/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index da5751d99..4c335fd15 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -223,7 +223,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("clusterStatus.timestamp").lte("now - $documentAge/$unitTime")) + .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now - $documentAge/$unitTime")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { From c7a6a6c6ee03f1fcfa7cf708d33079e34aec311f Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 25 Jul 2022 18:09:11 -0700 Subject: [PATCH 105/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsCoordinator.kt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 4c335fd15..5be1f59aa 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -233,7 +233,7 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted clusterStatus data from $documentAge ago") + log.info("deleted clusterStatus data from $documentAge/$unitTime ago") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) @@ -247,7 +247,7 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted cpu_usage data from $documentAge ago") + log.info("deleted cpu_usage data from $documentAge/$unitTime ago") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) @@ -261,7 +261,7 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted jvm_pressure data from $documentAge ago") + log.info("deleted jvm_pressure data from $documentAge/$unitTime ago") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) @@ -275,7 +275,7 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted unassigned_shards data from $documentAge ago") + log.info("deleted unassigned_shards data from $documentAge/$unitTime ago") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) @@ -289,7 +289,7 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted number of pending tasks from $documentAge ago") + log.info("deleted number of pending tasks from $documentAge/$unitTime ago") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) @@ -303,7 +303,7 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted active shards from $documentAge ago") + log.info("deleted active shards from $documentAge/$unitTime ago") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) @@ -317,6 +317,6 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted number of relocating shards from $documentAge ago") + log.info("deleted number of relocating shards from $documentAge/$unitTime ago") } } From 6ce35db2099137182cf2ca3da2cf08519ce4fe38 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 25 Jul 2022 18:29:23 -0700 Subject: [PATCH 106/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 5be1f59aa..944f61ad6 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -217,8 +217,8 @@ class ClusterMetricsCoordinator( } fun deleteDocs(client: NodeClient) { - val documentAge = METRICS_STORE_TIME.get(settings) - val unitTime = documentAge.toString().last() + val documentAge = METRICS_STORE_TIME.get(settings).toString() + val unitTime = documentAge.last() log.info("documentAge returns $documentAge, unitTime returns $unitTime") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) From 01a79aad80d8c073e0d7b66c56f8379f1fe48f80 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 25 Jul 2022 18:48:49 -0700 Subject: [PATCH 107/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 944f61ad6..a848278e6 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -233,7 +233,7 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted clusterStatus data from $documentAge/$unitTime ago") + log.info("deleted clusterStatus data from $documentAge/$unitTime ago, now - $documentAge/$unitTime") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) From 45ee435a9ad838215fc323e031f545663604bc4d Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 25 Jul 2022 18:55:47 -0700 Subject: [PATCH 108/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsCoordinator.kt | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index a848278e6..6d0d88acb 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -65,7 +65,7 @@ class ClusterMetricsCoordinator( } } if (event!!.localNodeMaster() && !isRunningFlag) { - threadPool.scheduleWithFixedDelay(scheduledJob, METRICS_EXECUTION_FREQUENCY.get(settings), ThreadPool.Names.SYSTEM_WRITE) + threadPool.scheduleWithFixedDelay(scheduledJob, metricsExecutionFrequency, ThreadPool.Names.SYSTEM_WRITE) isRunningFlag = true } } @@ -217,13 +217,13 @@ class ClusterMetricsCoordinator( } fun deleteDocs(client: NodeClient) { - val documentAge = METRICS_STORE_TIME.get(settings).toString() + val documentAge = metricsStoreTime.toString() val unitTime = documentAge.last() log.info("documentAge returns $documentAge, unitTime returns $unitTime") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now - $documentAge/$unitTime")) + .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now - $documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -233,11 +233,11 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted clusterStatus data from $documentAge/$unitTime ago, now - $documentAge/$unitTime") + log.info("deleted clusterStatus data from $documentAge/$unitTime ago, now - $documentAge") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("cpu_usage.timestamp").lte("now - $documentAge/$unitTime")) + .filter(QueryBuilders.rangeQuery("cpu_usage.timestamp").lte("now - $documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -251,7 +251,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("jvm_pressure.timestamp").lte("now - $documentAge/$unitTime")) + .filter(QueryBuilders.rangeQuery("jvm_pressure.timestamp").lte("now - $documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -265,7 +265,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("unassigned_shards.timestamp").lte("now - $documentAge/$unitTime")) + .filter(QueryBuilders.rangeQuery("unassigned_shards.timestamp").lte("now - $documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -279,7 +279,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("number_of_pending_tasks.timestamp").lte("now - $documentAge/$unitTime")) + .filter(QueryBuilders.rangeQuery("number_of_pending_tasks.timestamp").lte("now - $documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -293,7 +293,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("active_shards.timestamp").lte("now - $documentAge/$unitTime")) + .filter(QueryBuilders.rangeQuery("active_shards.timestamp").lte("now - $documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -307,7 +307,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("relocating_shards.timestamp").lte("now - $documentAge/$unitTime")) + .filter(QueryBuilders.rangeQuery("relocating_shards.timestamp").lte("now - $documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { From 7f8bc5b8e817e277725450d1e148d3ed47c32567 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 25 Jul 2022 19:10:13 -0700 Subject: [PATCH 109/215] PLS WORK Signed-off-by: Richard --- .../alerting/util/ClusterMetricsCoordinator.kt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 6d0d88acb..43057d756 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -223,7 +223,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now - $documentAge")) + .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now-$documentAge/$unitTime")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -233,11 +233,11 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted clusterStatus data from $documentAge/$unitTime ago, now - $documentAge") + log.info("deleted clusterStatus data from $documentAge/$unitTime ago, now-$documentAge") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("cpu_usage.timestamp").lte("now - $documentAge")) + .filter(QueryBuilders.rangeQuery("cpu_usage.timestamp").lte("now-$documentAge/$unitTime")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -251,7 +251,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("jvm_pressure.timestamp").lte("now - $documentAge")) + .filter(QueryBuilders.rangeQuery("jvm_pressure.timestamp").lte("now-$documentAge/$unitTime")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -265,7 +265,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("unassigned_shards.timestamp").lte("now - $documentAge")) + .filter(QueryBuilders.rangeQuery("unassigned_shards.timestamp").lte("now-$documentAge/$unitTime")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -279,7 +279,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("number_of_pending_tasks.timestamp").lte("now - $documentAge")) + .filter(QueryBuilders.rangeQuery("number_of_pending_tasks.timestamp").lte("now-$documentAge/$unitTime")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -293,7 +293,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("active_shards.timestamp").lte("now - $documentAge")) + .filter(QueryBuilders.rangeQuery("active_shards.timestamp").lte("now-$documentAge/$unitTime")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -307,7 +307,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("relocating_shards.timestamp").lte("now - $documentAge")) + .filter(QueryBuilders.rangeQuery("relocating_shards.timestamp").lte("now-$documentAge/$unitTime")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { From e00b0c17fd8c32e70e2dc1b2a9e687af4110d673 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 26 Jul 2022 10:33:40 -0700 Subject: [PATCH 110/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/settings/AlertingSettings.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index b5d52fc44..9cab638e8 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -159,13 +159,13 @@ class AlertingSettings { val METRICS_STORE_TIME = Setting.positiveTimeSetting( "plugins.alerting.cluster_metrics.metrics_history_max_age", - TimeValue(10, TimeUnit.MINUTES), + TimeValue(7, TimeUnit.DAYS), Setting.Property.NodeScope, Setting.Property.Dynamic ) val METRICS_EXECUTION_FREQUENCY = Setting.positiveTimeSetting( "plugins.alerting.cluster_metrics.execution_frequency", - TimeValue(1, TimeUnit.MINUTES), + TimeValue(15, TimeUnit.MINUTES), Setting.Property.NodeScope, Setting.Property.Dynamic ) } From 6e461d2e9a6a49c75c6a20e731b62dc1decb78a8 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 26 Jul 2022 11:14:48 -0700 Subject: [PATCH 111/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 43057d756..47cae4dca 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -65,6 +65,7 @@ class ClusterMetricsCoordinator( } } if (event!!.localNodeMaster() && !isRunningFlag) { + log.info("cluster changed metricsExecutionFrequency = $metricsExecutionFrequency") threadPool.scheduleWithFixedDelay(scheduledJob, metricsExecutionFrequency, ThreadPool.Names.SYSTEM_WRITE) isRunningFlag = true } From a339c7294da884ac913181af036550b804d81d36 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 26 Jul 2022 11:41:56 -0700 Subject: [PATCH 112/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/settings/AlertingSettings.kt | 4 ++-- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index 9cab638e8..b5d52fc44 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -159,13 +159,13 @@ class AlertingSettings { val METRICS_STORE_TIME = Setting.positiveTimeSetting( "plugins.alerting.cluster_metrics.metrics_history_max_age", - TimeValue(7, TimeUnit.DAYS), + TimeValue(10, TimeUnit.MINUTES), Setting.Property.NodeScope, Setting.Property.Dynamic ) val METRICS_EXECUTION_FREQUENCY = Setting.positiveTimeSetting( "plugins.alerting.cluster_metrics.execution_frequency", - TimeValue(15, TimeUnit.MINUTES), + TimeValue(1, TimeUnit.MINUTES), Setting.Property.NodeScope, Setting.Property.Dynamic ) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 47cae4dca..83830f6aa 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -26,6 +26,7 @@ import org.opensearch.index.query.QueryBuilders import org.opensearch.index.reindex.BulkByScrollResponse import org.opensearch.index.reindex.DeleteByQueryAction import org.opensearch.index.reindex.DeleteByQueryRequestBuilder +import org.opensearch.threadpool.Scheduler import org.opensearch.threadpool.ThreadPool import java.time.Instant import kotlin.coroutines.CoroutineContext @@ -41,6 +42,8 @@ class ClusterMetricsCoordinator( @Volatile private var metricsExecutionFrequency = METRICS_EXECUTION_FREQUENCY.get(settings) @Volatile private var metricsStoreTime = METRICS_STORE_TIME.get(settings) + private var dataPointCollectionJob: Scheduler.Cancellable? = null + private var dataPointDeletionJob: Scheduler.Cancellable? = null companion object { @Volatile var isRunningFlag = false From 5cc132b8b34c4338a7acfec712c93d0c55c2b159 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 26 Jul 2022 16:23:55 -0700 Subject: [PATCH 113/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsCoordinator.kt | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 83830f6aa..bf3c51c4e 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -48,6 +48,10 @@ class ClusterMetricsCoordinator( @Volatile var isRunningFlag = false internal set + var isDeletionUpdated = false + internal set + var isCollectionUpdated = false + internal set } override val coroutineContext: CoroutineContext @@ -56,24 +60,42 @@ class ClusterMetricsCoordinator( init { clusterService.addListener(this) clusterService.addLifecycleListener(this) - clusterService.clusterSettings.addSettingsUpdateConsumer(METRICS_EXECUTION_FREQUENCY) { metricsExecutionFrequency = it } - clusterService.clusterSettings.addSettingsUpdateConsumer(METRICS_STORE_TIME) { metricsStoreTime = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(METRICS_EXECUTION_FREQUENCY) { metricsExecutionFrequency = it; isCollectionUpdated = true } + clusterService.clusterSettings.addSettingsUpdateConsumer(METRICS_STORE_TIME) { metricsStoreTime = it; isDeletionUpdated = true } } override fun clusterChanged(event: ClusterChangedEvent?) { - val scheduledJob = Runnable { + val scheduledJobCollection = Runnable { launch { - destinationHelper(client as NodeClient, clusterService) - deleteDocs(client) + createDocs(client as NodeClient, clusterService) } } + val scheduledJobDeletion = Runnable { + launch { + deleteDocs(client as NodeClient) + } + } + if (isCollectionUpdated || isDeletionUpdated) { + dataPointCollectionJob?.cancel() + dataPointDeletionJob?.cancel() + log.info("cancelled data collection and deletion jobs ") + isRunningFlag = false + log.info("detected changes to settings, resetting running, deletion and collection flags to false") + isDeletionUpdated = false + isCollectionUpdated = false + } + // if cluster changed event occurs (if settings are changed) + // cancel current scheduled job and change runningFlag to false. + // maybe have individual runningFlags for collection and deletion? if (event!!.localNodeMaster() && !isRunningFlag) { log.info("cluster changed metricsExecutionFrequency = $metricsExecutionFrequency") - threadPool.scheduleWithFixedDelay(scheduledJob, metricsExecutionFrequency, ThreadPool.Names.SYSTEM_WRITE) + dataPointCollectionJob = threadPool.scheduleWithFixedDelay(scheduledJobCollection, metricsExecutionFrequency, ThreadPool.Names.SYSTEM_WRITE) + dataPointDeletionJob = threadPool.scheduleWithFixedDelay(scheduledJobDeletion, metricsExecutionFrequency, ThreadPool.Names.SYSTEM_WRITE) isRunningFlag = true } } - suspend fun destinationHelper(client: NodeClient, clusterService: ClusterService) { + + suspend fun createDocs(client: NodeClient, clusterService: ClusterService) { /* get time from Instant.now(), use this variable for all 4 documents that I'm going to create get clusterHealth API (status + unassigned shards) From 0701fdab735e50a645ef8c5e3d97c4818f593b35 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 26 Jul 2022 16:41:50 -0700 Subject: [PATCH 114/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/settings/AlertingSettings.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index b5d52fc44..9cab638e8 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -159,13 +159,13 @@ class AlertingSettings { val METRICS_STORE_TIME = Setting.positiveTimeSetting( "plugins.alerting.cluster_metrics.metrics_history_max_age", - TimeValue(10, TimeUnit.MINUTES), + TimeValue(7, TimeUnit.DAYS), Setting.Property.NodeScope, Setting.Property.Dynamic ) val METRICS_EXECUTION_FREQUENCY = Setting.positiveTimeSetting( "plugins.alerting.cluster_metrics.execution_frequency", - TimeValue(1, TimeUnit.MINUTES), + TimeValue(15, TimeUnit.MINUTES), Setting.Property.NodeScope, Setting.Property.Dynamic ) } From 5e928f2988ae87a09cc98e16ab9be47d42ecde95 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 27 Jul 2022 11:25:47 -0700 Subject: [PATCH 115/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsCoordinator.kt | 65 ++++++++++--------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index bf3c51c4e..f41377197 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -42,8 +42,7 @@ class ClusterMetricsCoordinator( @Volatile private var metricsExecutionFrequency = METRICS_EXECUTION_FREQUENCY.get(settings) @Volatile private var metricsStoreTime = METRICS_STORE_TIME.get(settings) - private var dataPointCollectionJob: Scheduler.Cancellable? = null - private var dataPointDeletionJob: Scheduler.Cancellable? = null + private var dataPointCollectionDeletionJob: Scheduler.Cancellable? = null companion object { @Volatile var isRunningFlag = false @@ -60,38 +59,46 @@ class ClusterMetricsCoordinator( init { clusterService.addListener(this) clusterService.addLifecycleListener(this) - clusterService.clusterSettings.addSettingsUpdateConsumer(METRICS_EXECUTION_FREQUENCY) { metricsExecutionFrequency = it; isCollectionUpdated = true } - clusterService.clusterSettings.addSettingsUpdateConsumer(METRICS_STORE_TIME) { metricsStoreTime = it; isDeletionUpdated = true } + clusterService.clusterSettings.addSettingsUpdateConsumer(METRICS_EXECUTION_FREQUENCY) { + metricsExecutionFrequency = it + isCollectionUpdated = true + } + clusterService.clusterSettings.addSettingsUpdateConsumer(METRICS_STORE_TIME) { + metricsStoreTime = it + isDeletionUpdated = true + } } override fun clusterChanged(event: ClusterChangedEvent?) { - val scheduledJobCollection = Runnable { - launch { - createDocs(client as NodeClient, clusterService) + if (metricsStoreTime > metricsExecutionFrequency) { + val scheduledJobCollection = Runnable { + launch { + createDocs(client as NodeClient, clusterService) + deleteDocs(client) + } } - } - val scheduledJobDeletion = Runnable { - launch { - deleteDocs(client as NodeClient) + if (isCollectionUpdated || isDeletionUpdated) { + dataPointCollectionDeletionJob?.cancel() + log.info("cancelled data collection and deletion jobs ") + isRunningFlag = false + log.info("detected changes to settings, resetting running, deletion and collection flags to false") + isDeletionUpdated = false + isCollectionUpdated = false } - } - if (isCollectionUpdated || isDeletionUpdated) { - dataPointCollectionJob?.cancel() - dataPointDeletionJob?.cancel() - log.info("cancelled data collection and deletion jobs ") - isRunningFlag = false - log.info("detected changes to settings, resetting running, deletion and collection flags to false") - isDeletionUpdated = false - isCollectionUpdated = false - } - // if cluster changed event occurs (if settings are changed) - // cancel current scheduled job and change runningFlag to false. - // maybe have individual runningFlags for collection and deletion? - if (event!!.localNodeMaster() && !isRunningFlag) { - log.info("cluster changed metricsExecutionFrequency = $metricsExecutionFrequency") - dataPointCollectionJob = threadPool.scheduleWithFixedDelay(scheduledJobCollection, metricsExecutionFrequency, ThreadPool.Names.SYSTEM_WRITE) - dataPointDeletionJob = threadPool.scheduleWithFixedDelay(scheduledJobDeletion, metricsExecutionFrequency, ThreadPool.Names.SYSTEM_WRITE) - isRunningFlag = true + // if cluster changed event occurs (if settings are changed) + // cancel current scheduled job and change runningFlag to false. + // maybe have individual runningFlags for collection and deletion? + if (event!!.localNodeMaster() && !isRunningFlag) { + log.info("cluster changed metricsExecutionFrequency = $metricsExecutionFrequency") + dataPointCollectionDeletionJob = threadPool.scheduleWithFixedDelay( + scheduledJobCollection, + metricsExecutionFrequency, + ThreadPool.Names.SYSTEM_WRITE + ) + isRunningFlag = true + } + } else { + log.info("Execution frequency can not be greater than the storage time. ") } } From a6bc0b9f52b25cb3ac8798324daab434f6238b37 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 27 Jul 2022 11:39:13 -0700 Subject: [PATCH 116/215] PLS WORK Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index f41377197..020634061 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -98,7 +98,9 @@ class ClusterMetricsCoordinator( isRunningFlag = true } } else { - log.info("Execution frequency can not be greater than the storage time. ") + throw java.lang.IllegalArgumentException( + "Execution frequency ($metricsExecutionFrequency) can not be greater than the storage time ($metricsStoreTime). " + ) } } From 79547335105437f986cc272ff58b7be4c6e161f6 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 27 Jul 2022 12:09:28 -0700 Subject: [PATCH 117/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsCoordinator.kt | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 020634061..a3dc17c8d 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -124,10 +124,18 @@ class ClusterMetricsCoordinator( ClusterMetricsVisualizationIndex.initFunc(client, clusterService) - val unassignedShards = cluster_health["unassigned_shards"].toString() - log.info("this is unassigned shards $unassignedShards") - val clusterStatus = cluster_health["status"].toString() - log.info("this is cluster status $clusterStatus") +// val unassignedShards = cluster_health["unassigned_shards"].toString() +// log.info("this is unassigned shards $unassignedShards") +// val clusterStatus = cluster_health["status"].toString() +// log.info("this is cluster status $clusterStatus") + val randomUnassignedShards = (0..4).random().toString() + var clusterStatusRandom = "green" + val colors = arrayOf("yellow", "red") + clusterStatusRandom = if (randomUnassignedShards == "0") { + "green" + } else { + colors.random() + } val numPending = cluster_health["number_of_pending_tasks"].toString() log.info("this is number of pending tasks $numPending") val activeShards = cluster_health["active_shards"].toString() @@ -137,7 +145,8 @@ class ClusterMetricsCoordinator( val nodes_map = cluster_stats["nodes"] as Map val process_map = nodes_map["process"] as Map val cpu_map = process_map["cpu"] as Map - val percent = cpu_map["percent"].toString() +// val percent = cpu_map["percent"].toString() + val percent = (30..80).random().toString() log.info("THIS IS CPU USAGE $percent") val jvm_map = nodes_map["jvm"] as Map val mem_map = jvm_map["mem"] as Map @@ -154,12 +163,12 @@ class ClusterMetricsCoordinator( val clusterStatus_data = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, current_time, - clusterStatus + clusterStatusRandom ) val unassigned_shards_data = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, current_time, - unassignedShards + randomUnassignedShards ) val cpu_usage_data = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.CPU_USAGE, From 462fea2cdd5a5a2df14d1362c5d7c88f1501e371 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 27 Jul 2022 13:48:35 -0700 Subject: [PATCH 118/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt new file mode 100644 index 000000000..7b7d5877e --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -0,0 +1,23 @@ +package org.opensearch.alerting.util.clustermetricscoordinatortest + +import org.opensearch.alerting.AlertingRestTestCase +import org.opensearch.alerting.makeRequest + +class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { + /* + Want to test these five things: + 1. Check that the index has been created, and that the name of the index is ".opendistro-alerting-cluster-metrics". + 2. Check that the total number of documents in the index is divisible by 7. + Additionally want to check that the number of individual metrics documents == total number of docs in index/7 + 3, Ingest a really old document, older than the stated deletion date, and make sure that it is properly deleted. + 4. Adjust the execution frequency setting to 1 minute, and make sure that the timestamps between the datapoints are 1 minute apart. + */ + fun `check name`() { + val indexRequest = client().makeRequest( + "GET", + "_cat/indices/.opendistro-alerting-cluster-metrics?format=json" + ) + val indexName = indexRequest.asMap() + assertNotNull("check name test case, indexRequest = $indexRequest", indexRequest) + } +} From 870d72ee0e4bef3d7621d4714a27ba2ff36aa444 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 27 Jul 2022 14:00:18 -0700 Subject: [PATCH 119/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 7b7d5877e..8bac62490 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -2,6 +2,8 @@ package org.opensearch.alerting.util.clustermetricscoordinatortest import org.opensearch.alerting.AlertingRestTestCase import org.opensearch.alerting.makeRequest +import org.opensearch.alerting.util.ClusterMetricsVisualizationIndex +import org.opensearch.rest.RestStatus class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { /* @@ -13,11 +15,10 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { 4. Adjust the execution frequency setting to 1 minute, and make sure that the timestamps between the datapoints are 1 minute apart. */ fun `check name`() { - val indexRequest = client().makeRequest( - "GET", - "_cat/indices/.opendistro-alerting-cluster-metrics?format=json" - ) - val indexName = indexRequest.asMap() - assertNotNull("check name test case, indexRequest = $indexRequest", indexRequest) + val settingsRequestFrequency = client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1m") + Thread.sleep(90000) + val index = ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX + val response = client().makeRequest("HEAD", index) + assertEquals("Index $index does not exist.", RestStatus.OK, response.restStatus()) } } From b08cdc4388009c67166cee632a0955f4fbc1c3cc Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 27 Jul 2022 14:01:15 -0700 Subject: [PATCH 120/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 8bac62490..267d48e50 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -15,7 +15,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { 4. Adjust the execution frequency setting to 1 minute, and make sure that the timestamps between the datapoints are 1 minute apart. */ fun `check name`() { - val settingsRequestFrequency = client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1m") + client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1m") Thread.sleep(90000) val index = ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX val response = client().makeRequest("HEAD", index) From c58a7a86d9edbfec8f9d8729faaf6bf35357f799 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 27 Jul 2022 14:09:47 -0700 Subject: [PATCH 121/215] PLS WORK Signed-off-by: Richard --- .../opensearch/alerting/model/ClusterMetricsDataPoint.kt | 5 +++++ .../opensearch/alerting/util/ClusterMetricsCoordinator.kt | 8 +++++++- .../alerting/util/ClusterMetricsVisualizationIndex.kt | 5 +++++ .../ClusterMetricsCoordinatorIT.kt | 5 +++++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt index 337de91b8..7e03bf90d 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.alerting.model import org.opensearch.common.xcontent.ToXContent diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index a3dc17c8d..583adeca9 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.alerting.util import kotlinx.coroutines.CoroutineName @@ -29,6 +34,7 @@ import org.opensearch.index.reindex.DeleteByQueryRequestBuilder import org.opensearch.threadpool.Scheduler import org.opensearch.threadpool.ThreadPool import java.time.Instant +import java.util.* import kotlin.coroutines.CoroutineContext private val log = org.apache.logging.log4j.LogManager.getLogger(ClusterMetricsCoordinator::class.java) @@ -156,7 +162,7 @@ class ClusterMetricsCoordinator( if (mem_used is Int && mem_avail is Int) { val jvm_pressure_num = ((mem_used.toDouble() / mem_avail.toDouble()) * 100) - jvm_pressure = String.format("%.2f", jvm_pressure_num) + jvm_pressure = String.format(Locale.ROOT, "%.2f", jvm_pressure_num) } log.info("THIS IS JVM PRESSURE $jvm_pressure") diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt index 5a07e3483..4ad0795a1 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsVisualizationIndex.kt @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.alerting.util import org.apache.logging.log4j.LogManager diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 267d48e50..7ffe21e3f 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.alerting.util.clustermetricscoordinatortest import org.opensearch.alerting.AlertingRestTestCase From 2fd6f4910e68ef75784c87bf5f8a2b9edd2f9c2a Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 27 Jul 2022 14:24:59 -0700 Subject: [PATCH 122/215] PLS WORK Signed-off-by: Richard --- .../clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 7ffe21e3f..d4e92f0e2 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -22,6 +22,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { fun `check name`() { client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1m") Thread.sleep(90000) + logger.info("CHECK NAME METHOD PLEASE WORK") val index = ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX val response = client().makeRequest("HEAD", index) assertEquals("Index $index does not exist.", RestStatus.OK, response.restStatus()) From 6c5de231453ac1b1d60acdde58902e50faa32054 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 27 Jul 2022 14:35:29 -0700 Subject: [PATCH 123/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index d4e92f0e2..e760783fc 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -22,9 +22,11 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { fun `check name`() { client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1m") Thread.sleep(90000) - logger.info("CHECK NAME METHOD PLEASE WORK") val index = ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX val response = client().makeRequest("HEAD", index) + assertNull("CHECK NAME METHOD PLEASE WORK", response) + assertNotNull("CHECK NAME METHOD PLEASE WORK", response) assertEquals("Index $index does not exist.", RestStatus.OK, response.restStatus()) + } } From 9d8ec38150bbf8cfc07d262676739cdd8be639ab Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 27 Jul 2022 14:36:16 -0700 Subject: [PATCH 124/215] PLS WORK Signed-off-by: Richard --- .../clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index e760783fc..0d44c2de7 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -27,6 +27,5 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { assertNull("CHECK NAME METHOD PLEASE WORK", response) assertNotNull("CHECK NAME METHOD PLEASE WORK", response) assertEquals("Index $index does not exist.", RestStatus.OK, response.restStatus()) - } } From bbd6d9a2d5db8db2e9634ad7f612a4a2ac272d59 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 27 Jul 2022 14:46:05 -0700 Subject: [PATCH 125/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 0d44c2de7..28908dbd4 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -19,7 +19,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { 3, Ingest a really old document, older than the stated deletion date, and make sure that it is properly deleted. 4. Adjust the execution frequency setting to 1 minute, and make sure that the timestamps between the datapoints are 1 minute apart. */ - fun `check name`() { + fun `test checkName`() { client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1m") Thread.sleep(90000) val index = ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX From 6286875b3a2e936d71afa698919c04d4c6539525 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 27 Jul 2022 14:56:14 -0700 Subject: [PATCH 126/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 28908dbd4..3575b6ab1 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -24,8 +24,6 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { Thread.sleep(90000) val index = ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX val response = client().makeRequest("HEAD", index) - assertNull("CHECK NAME METHOD PLEASE WORK", response) - assertNotNull("CHECK NAME METHOD PLEASE WORK", response) assertEquals("Index $index does not exist.", RestStatus.OK, response.restStatus()) } } From 1e82e98196d5c46b8b2ff73f5de23c4bee1977de Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 27 Jul 2022 15:37:43 -0700 Subject: [PATCH 127/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 3575b6ab1..5c21f85b6 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -5,6 +5,7 @@ package org.opensearch.alerting.util.clustermetricscoordinatortest +import org.junit.Before import org.opensearch.alerting.AlertingRestTestCase import org.opensearch.alerting.makeRequest import org.opensearch.alerting.util.ClusterMetricsVisualizationIndex @@ -12,18 +13,44 @@ import org.opensearch.rest.RestStatus class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { /* - Want to test these five things: - 1. Check that the index has been created, and that the name of the index is ".opendistro-alerting-cluster-metrics". 2. Check that the total number of documents in the index is divisible by 7. Additionally want to check that the number of individual metrics documents == total number of docs in index/7 3, Ingest a really old document, older than the stated deletion date, and make sure that it is properly deleted. 4. Adjust the execution frequency setting to 1 minute, and make sure that the timestamps between the datapoints are 1 minute apart. + Check 2 minute apart after */ + @Before + fun setup() { + // When setting up the tests, change the execution frequency and history max age settings to 1 minute and 10 minutes from + // 15 minutes and 7 days. + generateData() + val response = client().makeRequest( + "GET", + "_cluster/settings?flat_settings=true" + ).asMap() + val persistentMap = response["persistent"] as Map + val executionFrequency = persistentMap["plugins.alerting.cluster_metrics.execution_frequency"].toString() + val storageTime = persistentMap["plugins.alerting.cluster_metrics.metrics_history_max_age"].toString() + assertEquals(executionFrequency, "1m") + assertEquals(storageTime, "10m") + } + fun `test checkName`() { - client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1m") - Thread.sleep(90000) + // WHEN + THEN, check whether the created index exists and has the name '.opendistro-alerting-cluster-metrics' val index = ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX val response = client().makeRequest("HEAD", index) assertEquals("Index $index does not exist.", RestStatus.OK, response.restStatus()) } +// fun `test numberDocs`() { +// client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1m") +// Thread.sleep(300000) +// +// +// } + private fun generateData() { + client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1s") + client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "10m") + Thread.sleep(60000) + client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1m") + } } From 965fb4ada0abd1f37aea689c93ab88cc80b23269 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 27 Jul 2022 16:28:11 -0700 Subject: [PATCH 128/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 5c21f85b6..690f7f853 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -8,7 +8,9 @@ package org.opensearch.alerting.util.clustermetricscoordinatortest import org.junit.Before import org.opensearch.alerting.AlertingRestTestCase import org.opensearch.alerting.makeRequest +import org.opensearch.alerting.model.ClusterMetricsDataPoint import org.opensearch.alerting.util.ClusterMetricsVisualizationIndex +import org.opensearch.common.xcontent.XContentType import org.opensearch.rest.RestStatus class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { @@ -41,12 +43,18 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { val response = client().makeRequest("HEAD", index) assertEquals("Index $index does not exist.", RestStatus.OK, response.restStatus()) } -// fun `test numberDocs`() { -// client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1m") -// Thread.sleep(300000) -// -// -// } + fun `test numberDocs`() { + val response = client().makeRequest( + "GET", + ".opendistro-alerting-cluster-metrics/_search" + ) + val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) + val hits = xcp.map()["hits"]!! as Map> + val numberOfDocsFound = (hits["total"]?.get("value") as Int) + val size = ClusterMetricsDataPoint.MetricType.values().size + assertEquals((numberOfDocsFound.mod(size)), 0) + } + private fun generateData() { client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1s") client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "10m") From dbaffe8b6edc052cd9fd7520e57e9fd12053a501 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 28 Jul 2022 11:46:42 -0700 Subject: [PATCH 129/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 690f7f853..db53a27c7 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -44,6 +44,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { assertEquals("Index $index does not exist.", RestStatus.OK, response.restStatus()) } fun `test numberDocs`() { + // Check that the total number of documents found is divisible by the total number of metric types. val response = client().makeRequest( "GET", ".opendistro-alerting-cluster-metrics/_search" @@ -53,6 +54,17 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { val numberOfDocsFound = (hits["total"]?.get("value") as Int) val size = ClusterMetricsDataPoint.MetricType.values().size assertEquals((numberOfDocsFound.mod(size)), 0) + logger.info("this is the hits") + + // check that each of the metric types has a unique timestamp, and number of timestamps must be equal to total docs divided by 7 + // expect that there only document created for each metric type. + var mapCheck = hashMapOf>() + ClusterMetricsDataPoint.MetricType.values().forEach { mapCheck[it.metricName] = setOf() } + + val docs = hits["hits"]!! +// for (doc in docs) { +// +// } } private fun generateData() { From cecb40f8849769ce54ae73f60b56a3b3df6c5852 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 28 Jul 2022 11:54:33 -0700 Subject: [PATCH 130/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index db53a27c7..fcf14e20d 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -54,14 +54,14 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { val numberOfDocsFound = (hits["total"]?.get("value") as Int) val size = ClusterMetricsDataPoint.MetricType.values().size assertEquals((numberOfDocsFound.mod(size)), 0) - logger.info("this is the hits") + logger.info("this is the hits $hits") // check that each of the metric types has a unique timestamp, and number of timestamps must be equal to total docs divided by 7 // expect that there only document created for each metric type. var mapCheck = hashMapOf>() ClusterMetricsDataPoint.MetricType.values().forEach { mapCheck[it.metricName] = setOf() } - - val docs = hits["hits"]!! + logger.info("this is mapCheck $mapCheck") +// val docs = hits["hits"]!! // for (doc in docs) { // // } From 5cb9a47b69cf011f933cf62c774c4c1ffef451b0 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 28 Jul 2022 13:34:25 -0700 Subject: [PATCH 131/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index fcf14e20d..ce068137a 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -5,11 +5,15 @@ package org.opensearch.alerting.util.clustermetricscoordinatortest +import org.apache.http.entity.ContentType +import org.apache.http.entity.StringEntity import org.junit.Before import org.opensearch.alerting.AlertingRestTestCase import org.opensearch.alerting.makeRequest import org.opensearch.alerting.model.ClusterMetricsDataPoint +import org.opensearch.alerting.opensearchapi.string import org.opensearch.alerting.util.ClusterMetricsVisualizationIndex +import org.opensearch.common.xcontent.XContentFactory.jsonBuilder import org.opensearch.common.xcontent.XContentType import org.opensearch.rest.RestStatus @@ -45,9 +49,15 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { } fun `test numberDocs`() { // Check that the total number of documents found is divisible by the total number of metric types. + val settings = jsonBuilder() + .startObject() + .field("size", 10000) + .endObject() + .string() val response = client().makeRequest( "GET", - ".opendistro-alerting-cluster-metrics/_search" + ".opendistro-alerting-cluster-metrics/_search", + StringEntity(settings, ContentType.APPLICATION_JSON) ) val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) val hits = xcp.map()["hits"]!! as Map> From 6b0d962ad166d3c535286eeeb5fb1f4de8e509c2 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 28 Jul 2022 13:42:39 -0700 Subject: [PATCH 132/215] PLS WORK Signed-off-by: Richard --- .../clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index ce068137a..31edf145b 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -71,6 +71,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { var mapCheck = hashMapOf>() ClusterMetricsDataPoint.MetricType.values().forEach { mapCheck[it.metricName] = setOf() } logger.info("this is mapCheck $mapCheck") + assertNotNull("Check the number of docs and stuff", 1) // val docs = hits["hits"]!! // for (doc in docs) { // From 14dbf65219fa083f9d05403325431c4ed1efab21 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 28 Jul 2022 13:48:42 -0700 Subject: [PATCH 133/215] PLS WORK Signed-off-by: Richard --- .../clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 31edf145b..3c1cf35b7 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -72,6 +72,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { ClusterMetricsDataPoint.MetricType.values().forEach { mapCheck[it.metricName] = setOf() } logger.info("this is mapCheck $mapCheck") assertNotNull("Check the number of docs and stuff", 1) + assertNull("Check the number of docs and stuff", 1) // val docs = hits["hits"]!! // for (doc in docs) { // From 0a4c096e5d1af7b71287eb13b3b103b0876275a7 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 28 Jul 2022 13:56:20 -0700 Subject: [PATCH 134/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 3c1cf35b7..09e0737dc 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -64,7 +64,8 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { val numberOfDocsFound = (hits["total"]?.get("value") as Int) val size = ClusterMetricsDataPoint.MetricType.values().size assertEquals((numberOfDocsFound.mod(size)), 0) - logger.info("this is the hits $hits") + val hitsData = hits["hits"]!! + logger.info("this is the hits $hitsData") // check that each of the metric types has a unique timestamp, and number of timestamps must be equal to total docs divided by 7 // expect that there only document created for each metric type. From e7f337811ededdf2cb0abe6b1f3a20f4f68bb3f7 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 28 Jul 2022 13:59:04 -0700 Subject: [PATCH 135/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 09e0737dc..ed45bb10c 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -64,7 +64,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { val numberOfDocsFound = (hits["total"]?.get("value") as Int) val size = ClusterMetricsDataPoint.MetricType.values().size assertEquals((numberOfDocsFound.mod(size)), 0) - val hitsData = hits["hits"]!! + val hitsData = hits["hits"] logger.info("this is the hits $hitsData") // check that each of the metric types has a unique timestamp, and number of timestamps must be equal to total docs divided by 7 From dbfe4840ea8746d1d59463a8cdf51581d75d200d Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 28 Jul 2022 14:04:04 -0700 Subject: [PATCH 136/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index ed45bb10c..3c1cf35b7 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -64,8 +64,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { val numberOfDocsFound = (hits["total"]?.get("value") as Int) val size = ClusterMetricsDataPoint.MetricType.values().size assertEquals((numberOfDocsFound.mod(size)), 0) - val hitsData = hits["hits"] - logger.info("this is the hits $hitsData") + logger.info("this is the hits $hits") // check that each of the metric types has a unique timestamp, and number of timestamps must be equal to total docs divided by 7 // expect that there only document created for each metric type. From 3f4d8294d9ba89587f6d77e7652deffd9669189d Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 28 Jul 2022 14:44:26 -0700 Subject: [PATCH 137/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 3c1cf35b7..1445e32ec 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -47,6 +47,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { val response = client().makeRequest("HEAD", index) assertEquals("Index $index does not exist.", RestStatus.OK, response.restStatus()) } + fun `test numberDocs`() { // Check that the total number of documents found is divisible by the total number of metric types. val settings = jsonBuilder() @@ -65,6 +66,8 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { val size = ClusterMetricsDataPoint.MetricType.values().size assertEquals((numberOfDocsFound.mod(size)), 0) logger.info("this is the hits $hits") + val dataHits = hits.size + logger.info("this is dataHits size $dataHits") // check that each of the metric types has a unique timestamp, and number of timestamps must be equal to total docs divided by 7 // expect that there only document created for each metric type. From b55e68bc0f9ae42e7da6915b7b23133fc7405b3a Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 28 Jul 2022 14:48:14 -0700 Subject: [PATCH 138/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 1445e32ec..157152da5 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -66,7 +66,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { val size = ClusterMetricsDataPoint.MetricType.values().size assertEquals((numberOfDocsFound.mod(size)), 0) logger.info("this is the hits $hits") - val dataHits = hits.size + val dataHits = hits["hits"] as ArrayList logger.info("this is dataHits size $dataHits") // check that each of the metric types has a unique timestamp, and number of timestamps must be equal to total docs divided by 7 From 19e9e3b41221ac0010e4a82fdfc4808f3647ffea Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 28 Jul 2022 15:05:21 -0700 Subject: [PATCH 139/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 157152da5..ab4578871 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -65,21 +65,19 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { val numberOfDocsFound = (hits["total"]?.get("value") as Int) val size = ClusterMetricsDataPoint.MetricType.values().size assertEquals((numberOfDocsFound.mod(size)), 0) - logger.info("this is the hits $hits") - val dataHits = hits["hits"] as ArrayList - logger.info("this is dataHits size $dataHits") + val docs = hits["hits"] as ArrayList> + // check that each of the metric types has a unique timestamp, and number of timestamps must be equal to total docs divided by 7 // expect that there only document created for each metric type. var mapCheck = hashMapOf>() ClusterMetricsDataPoint.MetricType.values().forEach { mapCheck[it.metricName] = setOf() } logger.info("this is mapCheck $mapCheck") - assertNotNull("Check the number of docs and stuff", 1) - assertNull("Check the number of docs and stuff", 1) -// val docs = hits["hits"]!! -// for (doc in docs) { -// -// } + for (doc in docs) { + var source = doc["_source"] as Map + logger.info("this is source data $source") + } + assertNull("fail the test", 1) } private fun generateData() { From 226a8facafea3fc16b7c867b9365d2186f105b7c Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 28 Jul 2022 15:05:42 -0700 Subject: [PATCH 140/215] PLS WORK Signed-off-by: Richard --- .../clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index ab4578871..fc3dc81a1 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -67,7 +67,6 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { assertEquals((numberOfDocsFound.mod(size)), 0) val docs = hits["hits"] as ArrayList> - // check that each of the metric types has a unique timestamp, and number of timestamps must be equal to total docs divided by 7 // expect that there only document created for each metric type. var mapCheck = hashMapOf>() From 9ebc14004fbd441d63f795e127687bc08b763beb Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 28 Jul 2022 15:43:51 -0700 Subject: [PATCH 141/215] PLS WORK Signed-off-by: Richard --- .../alerting/model/ClusterMetricsDataPoint.kt | 2 +- .../util/ClusterMetricsCoordinator.kt | 2 +- .../ClusterMetricsCoordinatorIT.kt | 19 ++++++++++++++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt index 7e03bf90d..8b50ccaa1 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt @@ -43,7 +43,7 @@ data class ClusterMetricsDataPoint( UNASSIGNED_SHARDS( "unassigned_shards" ), - NUM_PENDING_TASKS( + NUMBER_OF_PENDING_TASKS( "number_of_pending_tasks" ), ACTIVE_SHARDS( diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 583adeca9..275267121 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -187,7 +187,7 @@ class ClusterMetricsCoordinator( jvm_pressure ) val pendingTasksData = ClusterMetricsDataPoint( - ClusterMetricsDataPoint.MetricType.NUM_PENDING_TASKS, + ClusterMetricsDataPoint.MetricType.NUMBER_OF_PENDING_TASKS, current_time, numPending ) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index fc3dc81a1..a23ac5509 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -16,6 +16,8 @@ import org.opensearch.alerting.util.ClusterMetricsVisualizationIndex import org.opensearch.common.xcontent.XContentFactory.jsonBuilder import org.opensearch.common.xcontent.XContentType import org.opensearch.rest.RestStatus +import java.util.* +import kotlin.collections.ArrayList class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { /* @@ -70,12 +72,23 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { // check that each of the metric types has a unique timestamp, and number of timestamps must be equal to total docs divided by 7 // expect that there only document created for each metric type. var mapCheck = hashMapOf>() - ClusterMetricsDataPoint.MetricType.values().forEach { mapCheck[it.metricName] = setOf() } - logger.info("this is mapCheck $mapCheck") + ClusterMetricsDataPoint.MetricType.values().forEach { mapCheck[it.metricName] = mutableSetOf() } for (doc in docs) { - var source = doc["_source"] as Map + var source = doc["_source"] as Map> logger.info("this is source data $source") + + ClusterMetricsDataPoint.MetricType.values().forEach { + val metricType = source.keys.first() + try { + ClusterMetricsDataPoint.MetricType.valueOf(metricType.uppercase(Locale.getDefault())) + mapCheck[metricType]?.plus(source[metricType]?.get("timestamp")) + } catch (e: java.lang.IllegalArgumentException) { + logger.info("Key does not exist in the enum class.") + } + } } + logger.info("mapcheck is this $mapCheck") + assertEquals(mapCheck.values.toSet().size, 1) assertNull("fail the test", 1) } From d466d945129621cd57802cf6ef05637cb321950b Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 28 Jul 2022 15:49:46 -0700 Subject: [PATCH 142/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index a23ac5509..d6ee1379e 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -76,15 +76,12 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { for (doc in docs) { var source = doc["_source"] as Map> logger.info("this is source data $source") - - ClusterMetricsDataPoint.MetricType.values().forEach { - val metricType = source.keys.first() - try { - ClusterMetricsDataPoint.MetricType.valueOf(metricType.uppercase(Locale.getDefault())) - mapCheck[metricType]?.plus(source[metricType]?.get("timestamp")) - } catch (e: java.lang.IllegalArgumentException) { - logger.info("Key does not exist in the enum class.") - } + val metricType = source.keys.first() + try { + ClusterMetricsDataPoint.MetricType.valueOf(metricType.uppercase(Locale.getDefault())) + mapCheck[metricType] = mapCheck[metricType]?.plus(source[metricType]?.get("timestamp")) as Set + } catch (e: java.lang.IllegalArgumentException) { + logger.info("Key does not exist in the enum class.") } } logger.info("mapcheck is this $mapCheck") From 58db93bdb8bfd514bcd5b2bffaa72c5afd8dbe5d Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 28 Jul 2022 15:52:40 -0700 Subject: [PATCH 143/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index d6ee1379e..301a8e87c 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -84,7 +84,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { logger.info("Key does not exist in the enum class.") } } - logger.info("mapcheck is this $mapCheck") + logger.info("mapcheck is this $mapCheck, and mapcheck set size ${mapCheck.values.toSet().size}") assertEquals(mapCheck.values.toSet().size, 1) assertNull("fail the test", 1) } From 1e215f3c819292aa79f59118b908497bf8f840b6 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 28 Jul 2022 16:37:33 -0700 Subject: [PATCH 144/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 56 +++++++++++++------ 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 301a8e87c..362c7838f 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -13,12 +13,12 @@ import org.opensearch.alerting.makeRequest import org.opensearch.alerting.model.ClusterMetricsDataPoint import org.opensearch.alerting.opensearchapi.string import org.opensearch.alerting.util.ClusterMetricsVisualizationIndex +import org.opensearch.client.Response import org.opensearch.common.xcontent.XContentFactory.jsonBuilder import org.opensearch.common.xcontent.XContentType import org.opensearch.rest.RestStatus import java.util.* import kotlin.collections.ArrayList - class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { /* 2. Check that the total number of documents in the index is divisible by 7. @@ -52,16 +52,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { fun `test numberDocs`() { // Check that the total number of documents found is divisible by the total number of metric types. - val settings = jsonBuilder() - .startObject() - .field("size", 10000) - .endObject() - .string() - val response = client().makeRequest( - "GET", - ".opendistro-alerting-cluster-metrics/_search", - StringEntity(settings, ContentType.APPLICATION_JSON) - ) + val response = getResponse() val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) val hits = xcp.map()["hits"]!! as Map> val numberOfDocsFound = (hits["total"]?.get("value") as Int) @@ -70,11 +61,11 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { val docs = hits["hits"] as ArrayList> // check that each of the metric types has a unique timestamp, and number of timestamps must be equal to total docs divided by 7 - // expect that there only document created for each metric type. - var mapCheck = hashMapOf>() + // expect that there should only be one document created for each metric type at a time. + val mapCheck = hashMapOf>() ClusterMetricsDataPoint.MetricType.values().forEach { mapCheck[it.metricName] = mutableSetOf() } for (doc in docs) { - var source = doc["_source"] as Map> + val source = doc["_source"] as Map> logger.info("this is source data $source") val metricType = source.keys.first() try { @@ -84,9 +75,12 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { logger.info("Key does not exist in the enum class.") } } - logger.info("mapcheck is this $mapCheck, and mapcheck set size ${mapCheck.values.toSet().size}") assertEquals(mapCheck.values.toSet().size, 1) - assertNull("fail the test", 1) + } + + fun `test deleteDocs`() { + createDoc() + Thread.sleep(60000) } private fun generateData() { @@ -95,4 +89,34 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { Thread.sleep(60000) client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1m") } + + private fun getResponse(): Response { + val settings = jsonBuilder() + .startObject() + .field("size", 10000) + .endObject() + .string() + return client().makeRequest( + "GET", + ".opendistro-alerting-cluster-metrics/_search", + StringEntity(settings, ContentType.APPLICATION_JSON) + ) + } + + private fun createDoc() { + var year = (0..2000).random().toString() + val doc = jsonBuilder() + .startObject() + .startObject("cluster_status") + .field("timestamp", "$year-01-01T12:00:00") + .field("value", "yellow") + .endObject() + .endObject() + .string() + client().makeRequest( + "POST", + ".opendistro-alerting-cluster-metrics/_doc", + StringEntity(doc, ContentType.APPLICATION_JSON) + ) + } } From 37f40e4cbc228af71d71a8d9017f44d9ccb6405b Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 28 Jul 2022 17:40:14 -0700 Subject: [PATCH 145/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 362c7838f..ee6684608 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -7,19 +7,27 @@ package org.opensearch.alerting.util.clustermetricscoordinatortest import org.apache.http.entity.ContentType import org.apache.http.entity.StringEntity +import org.joda.time.Minutes import org.junit.Before import org.opensearch.alerting.AlertingRestTestCase import org.opensearch.alerting.makeRequest import org.opensearch.alerting.model.ClusterMetricsDataPoint import org.opensearch.alerting.opensearchapi.string +import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.util.ClusterMetricsVisualizationIndex import org.opensearch.client.Response +import org.opensearch.common.settings.Settings import org.opensearch.common.xcontent.XContentFactory.jsonBuilder import org.opensearch.common.xcontent.XContentType import org.opensearch.rest.RestStatus +import java.time.Instant +import java.time.temporal.ChronoUnit +import java.time.temporal.TemporalAmount import java.util.* import kotlin.collections.ArrayList -class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { +class ClusterMetricsCoordinatorIT( + private val settings: Settings +) : AlertingRestTestCase() { /* 2. Check that the total number of documents in the index is divisible by 7. Additionally want to check that the number of individual metrics documents == total number of docs in index/7 @@ -32,10 +40,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { // When setting up the tests, change the execution frequency and history max age settings to 1 minute and 10 minutes from // 15 minutes and 7 days. generateData() - val response = client().makeRequest( - "GET", - "_cluster/settings?flat_settings=true" - ).asMap() + val response = getSettings() val persistentMap = response["persistent"] as Map val executionFrequency = persistentMap["plugins.alerting.cluster_metrics.execution_frequency"].toString() val storageTime = persistentMap["plugins.alerting.cluster_metrics.metrics_history_max_age"].toString() @@ -81,6 +86,12 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { fun `test deleteDocs`() { createDoc() Thread.sleep(60000) + val response = getResponse() + val storageTime = AlertingSettings.METRICS_STORE_TIME.get(settings) + val time = Instant.now().minus(storageTime as TemporalAmount).minus(1, ChronoUnit.MINUTES).toString() + logger.info("this is time now - 10 minutes - 1 minute $time") + assertNull("YEP ERROR", 1) + } private fun generateData() { @@ -104,11 +115,11 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { } private fun createDoc() { - var year = (0..2000).random().toString() + val storageTime = AlertingSettings.METRICS_STORE_TIME.get(settings) val doc = jsonBuilder() .startObject() .startObject("cluster_status") - .field("timestamp", "$year-01-01T12:00:00") + .field("timestamp", storageTime.toString()) .field("value", "yellow") .endObject() .endObject() @@ -119,4 +130,11 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { StringEntity(doc, ContentType.APPLICATION_JSON) ) } + + private fun getSettings(): Map { + return client().makeRequest( + "GET", + ".opendistro-alerting-cluster-metrics/_settings" + ).asMap() + } } From 4996ded931437299bf402febaee7d5404ec1663c Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 28 Jul 2022 17:41:30 -0700 Subject: [PATCH 146/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index ee6684608..6afa16b3f 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -7,7 +7,6 @@ package org.opensearch.alerting.util.clustermetricscoordinatortest import org.apache.http.entity.ContentType import org.apache.http.entity.StringEntity -import org.joda.time.Minutes import org.junit.Before import org.opensearch.alerting.AlertingRestTestCase import org.opensearch.alerting.makeRequest @@ -91,7 +90,6 @@ class ClusterMetricsCoordinatorIT( val time = Instant.now().minus(storageTime as TemporalAmount).minus(1, ChronoUnit.MINUTES).toString() logger.info("this is time now - 10 minutes - 1 minute $time") assertNull("YEP ERROR", 1) - } private fun generateData() { From 700d735c8daf4357685559cb9b129368360109a3 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 14:03:32 -0700 Subject: [PATCH 147/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 6afa16b3f..41bf46402 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -12,21 +12,16 @@ import org.opensearch.alerting.AlertingRestTestCase import org.opensearch.alerting.makeRequest import org.opensearch.alerting.model.ClusterMetricsDataPoint import org.opensearch.alerting.opensearchapi.string -import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.util.ClusterMetricsVisualizationIndex import org.opensearch.client.Response -import org.opensearch.common.settings.Settings import org.opensearch.common.xcontent.XContentFactory.jsonBuilder import org.opensearch.common.xcontent.XContentType import org.opensearch.rest.RestStatus import java.time.Instant import java.time.temporal.ChronoUnit -import java.time.temporal.TemporalAmount import java.util.* import kotlin.collections.ArrayList -class ClusterMetricsCoordinatorIT( - private val settings: Settings -) : AlertingRestTestCase() { +class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { /* 2. Check that the total number of documents in the index is divisible by 7. Additionally want to check that the number of individual metrics documents == total number of docs in index/7 @@ -56,9 +51,7 @@ class ClusterMetricsCoordinatorIT( fun `test numberDocs`() { // Check that the total number of documents found is divisible by the total number of metric types. - val response = getResponse() - val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) - val hits = xcp.map()["hits"]!! as Map> + val hits = parseResponse() val numberOfDocsFound = (hits["total"]?.get("value") as Int) val size = ClusterMetricsDataPoint.MetricType.values().size assertEquals((numberOfDocsFound.mod(size)), 0) @@ -83,13 +76,25 @@ class ClusterMetricsCoordinatorIT( } fun `test deleteDocs`() { - createDoc() + val time = getTime() + createDoc(time) Thread.sleep(60000) - val response = getResponse() - val storageTime = AlertingSettings.METRICS_STORE_TIME.get(settings) - val time = Instant.now().minus(storageTime as TemporalAmount).minus(1, ChronoUnit.MINUTES).toString() - logger.info("this is time now - 10 minutes - 1 minute $time") - assertNull("YEP ERROR", 1) + val hits = parseResponse() + var flag = false + val docs = hits["hits"] as ArrayList> + + for (doc in docs) { + val source = doc["_source"] as Map> + val metricType = source.keys.first() + if (metricType == "cluster_status") { + if (source[metricType]?.get("timestamp") == time) { + assertTrue(flag) + return + } + } + } + flag = true + assertTrue(flag) } private fun generateData() { @@ -111,13 +116,17 @@ class ClusterMetricsCoordinatorIT( StringEntity(settings, ContentType.APPLICATION_JSON) ) } + private fun parseResponse(): Map> { + val response = getResponse() + val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) + return xcp.map()["hits"]!! as Map> + } - private fun createDoc() { - val storageTime = AlertingSettings.METRICS_STORE_TIME.get(settings) + private fun createDoc(time: Instant?) { val doc = jsonBuilder() .startObject() .startObject("cluster_status") - .field("timestamp", storageTime.toString()) + .field("timestamp", time.toString()) .field("value", "yellow") .endObject() .endObject() @@ -128,6 +137,9 @@ class ClusterMetricsCoordinatorIT( StringEntity(doc, ContentType.APPLICATION_JSON) ) } + private fun getTime(): Instant? { + return Instant.now().minus(10, ChronoUnit.MINUTES).minus(1, ChronoUnit.MINUTES) + } private fun getSettings(): Map { return client().makeRequest( From 42f18c7aba43afbf9514809771cb5417955d0e68 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 14:11:14 -0700 Subject: [PATCH 148/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 41bf46402..a12ac6e58 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -51,7 +51,8 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { fun `test numberDocs`() { // Check that the total number of documents found is divisible by the total number of metric types. - val hits = parseResponse() + val response = getResponse() + val hits = parseResponse(response) val numberOfDocsFound = (hits["total"]?.get("value") as Int) val size = ClusterMetricsDataPoint.MetricType.values().size assertEquals((numberOfDocsFound.mod(size)), 0) @@ -79,7 +80,8 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { val time = getTime() createDoc(time) Thread.sleep(60000) - val hits = parseResponse() + val response = getResponse() + val hits = parseResponse(response) var flag = false val docs = hits["hits"] as ArrayList> @@ -116,8 +118,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { StringEntity(settings, ContentType.APPLICATION_JSON) ) } - private fun parseResponse(): Map> { - val response = getResponse() + private fun parseResponse(response: Response): Map> { val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) return xcp.map()["hits"]!! as Map> } From 623ab099c93270759593dcf7b18ca8715e6b17ce Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 14:15:00 -0700 Subject: [PATCH 149/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index a12ac6e58..2b5130316 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -52,7 +52,8 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { fun `test numberDocs`() { // Check that the total number of documents found is divisible by the total number of metric types. val response = getResponse() - val hits = parseResponse(response) + val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) + val hits = xcp.map()["hits"]!! as Map> val numberOfDocsFound = (hits["total"]?.get("value") as Int) val size = ClusterMetricsDataPoint.MetricType.values().size assertEquals((numberOfDocsFound.mod(size)), 0) @@ -81,7 +82,8 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { createDoc(time) Thread.sleep(60000) val response = getResponse() - val hits = parseResponse(response) + val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) + val hits = xcp.map()["hits"]!! as Map> var flag = false val docs = hits["hits"] as ArrayList> @@ -118,10 +120,6 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { StringEntity(settings, ContentType.APPLICATION_JSON) ) } - private fun parseResponse(response: Response): Map> { - val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) - return xcp.map()["hits"]!! as Map> - } private fun createDoc(time: Instant?) { val doc = jsonBuilder() From ba47a16941df3c90f267fc178aa5c2139381f5aa Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 14:15:21 -0700 Subject: [PATCH 150/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 2b5130316..9edb6e630 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -83,7 +83,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { Thread.sleep(60000) val response = getResponse() val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) - val hits = xcp.map()["hits"]!! as Map> + val hits = xcp.map()["hits"]!! as Map> var flag = false val docs = hits["hits"] as ArrayList> From 6346c7f6e85b2fd490d1e2ee638b03f9c69cafa9 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 14:23:05 -0700 Subject: [PATCH 151/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 9edb6e630..02c1967c0 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -143,7 +143,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { private fun getSettings(): Map { return client().makeRequest( "GET", - ".opendistro-alerting-cluster-metrics/_settings" + "_cluster/settings?flat_settings=true" ).asMap() } } From ba50b4958fdc7acd7e061e66a836d0bd575b1403 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 14:49:26 -0700 Subject: [PATCH 152/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 02c1967c0..0d0afd156 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -23,9 +23,6 @@ import java.util.* import kotlin.collections.ArrayList class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { /* - 2. Check that the total number of documents in the index is divisible by 7. - Additionally want to check that the number of individual metrics documents == total number of docs in index/7 - 3, Ingest a really old document, older than the stated deletion date, and make sure that it is properly deleted. 4. Adjust the execution frequency setting to 1 minute, and make sure that the timestamps between the datapoints are 1 minute apart. Check 2 minute apart after */ @@ -52,8 +49,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { fun `test numberDocs`() { // Check that the total number of documents found is divisible by the total number of metric types. val response = getResponse() - val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) - val hits = xcp.map()["hits"]!! as Map> + val hits = getHits(response) val numberOfDocsFound = (hits["total"]?.get("value") as Int) val size = ClusterMetricsDataPoint.MetricType.values().size assertEquals((numberOfDocsFound.mod(size)), 0) @@ -82,8 +78,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { createDoc(time) Thread.sleep(60000) val response = getResponse() - val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) - val hits = xcp.map()["hits"]!! as Map> + val hits = getHits(response) var flag = false val docs = hits["hits"] as ArrayList> @@ -101,6 +96,24 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { assertTrue(flag) } + fun `test frequency`() { + client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "2m") + Thread.sleep(300000) + + val response = getResponse() + val hits = getHits(response) + val docs = hits["hits"] as ArrayList> + val times = mutableSetOf() + + for (doc in docs) { + val source = doc["_source"] as Map> + val metricType = source.keys.first() + times.plus(source[metricType]?.get("timestamp")) + } + logger.info("this is the times Set $times") + assertFalse(true) + } + private fun generateData() { client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1s") client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "10m") @@ -140,6 +153,11 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { return Instant.now().minus(10, ChronoUnit.MINUTES).minus(1, ChronoUnit.MINUTES) } + private fun getHits(response: Response): Map> { + val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) + return xcp.map()["hits"]!! as Map> + } + private fun getSettings(): Map { return client().makeRequest( "GET", From be78c1e1044d96119697300011727b2a4359e786 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 14:57:42 -0700 Subject: [PATCH 153/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 0d0afd156..7cc9768be 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -108,7 +108,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { for (doc in docs) { val source = doc["_source"] as Map> val metricType = source.keys.first() - times.plus(source[metricType]?.get("timestamp")) + times.plus(source[metricType]?.get("timestamp")) as Set } logger.info("this is the times Set $times") assertFalse(true) From fe54dcd0cbea824c2a32f36005af2c57876dcaab Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 15:07:52 -0700 Subject: [PATCH 154/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 7cc9768be..157f7b140 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -108,7 +108,8 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { for (doc in docs) { val source = doc["_source"] as Map> val metricType = source.keys.first() - times.plus(source[metricType]?.get("timestamp")) as Set + logger.info("this is data ${source[metricType]?.get("timestamp")}") + times.plus(source[metricType]?.get("timestamp")) } logger.info("this is the times Set $times") assertFalse(true) From 1875ce7acfe42de25918ac2e5ced16c1b7f08e3f Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 15:16:06 -0700 Subject: [PATCH 155/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 157f7b140..73872d99d 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -108,8 +108,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { for (doc in docs) { val source = doc["_source"] as Map> val metricType = source.keys.first() - logger.info("this is data ${source[metricType]?.get("timestamp")}") - times.plus(source[metricType]?.get("timestamp")) + times.plus(source[metricType]?.get("timestamp").toString()) } logger.info("this is the times Set $times") assertFalse(true) From dd3c1a6c2e057192a8270f4644bec08166809a18 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 15:25:41 -0700 Subject: [PATCH 156/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 73872d99d..ab65ec7dc 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -108,7 +108,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { for (doc in docs) { val source = doc["_source"] as Map> val metricType = source.keys.first() - times.plus(source[metricType]?.get("timestamp").toString()) + times.add(source[metricType]?.get("timestamp").toString()) } logger.info("this is the times Set $times") assertFalse(true) From 6b2eccccf2811f5a8bf57a3e148e32457a0ff631 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 15:34:41 -0700 Subject: [PATCH 157/215] PLS WORK Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index ab65ec7dc..7bc1972ae 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -110,7 +110,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { val metricType = source.keys.first() times.add(source[metricType]?.get("timestamp").toString()) } - logger.info("this is the times Set $times") + logger.info("this is the times Set length ${times.size}") assertFalse(true) } From d2d287c5b1bb9b81b087bd96a9836375215e1fd9 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 16:29:46 -0700 Subject: [PATCH 158/215] PLS WORK Signed-off-by: Richard --- .../util/ClusterMetricsCoordinator.kt | 215 ++++++++---------- 1 file changed, 98 insertions(+), 117 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 275267121..ba3241c99 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -11,7 +11,7 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import org.opensearch.action.ActionListener import org.opensearch.action.admin.cluster.health.ClusterHealthRequest -import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest +import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest import org.opensearch.action.index.IndexRequest import org.opensearch.action.index.IndexResponse import org.opensearch.alerting.model.ClusterMetricsDataPoint @@ -126,133 +126,114 @@ class ClusterMetricsCoordinator( // cluster health for unassigned shards val cluster_health = client.admin().cluster().health(ClusterHealthRequest()).get().toMap() // cluster stats for cluster status (health), CPU usage, JVM pressure - var cluster_stats = client.admin().cluster().clusterStats(ClusterStatsRequest()).get().toMap() + var nodeStats = client.admin().cluster().nodesStats(NodesStatsRequest()).get().toMap() ClusterMetricsVisualizationIndex.initFunc(client, clusterService) -// val unassignedShards = cluster_health["unassigned_shards"].toString() -// log.info("this is unassigned shards $unassignedShards") -// val clusterStatus = cluster_health["status"].toString() -// log.info("this is cluster status $clusterStatus") - val randomUnassignedShards = (0..4).random().toString() - var clusterStatusRandom = "green" - val colors = arrayOf("yellow", "red") - clusterStatusRandom = if (randomUnassignedShards == "0") { - "green" - } else { - colors.random() - } + val unassignedShards = cluster_health["unassigned_shards"].toString() + log.info("this is unassigned shards $unassignedShards") + val clusterStatus = cluster_health["status"].toString() + log.info("this is cluster status $clusterStatus") val numPending = cluster_health["number_of_pending_tasks"].toString() log.info("this is number of pending tasks $numPending") val activeShards = cluster_health["active_shards"].toString() log.info("this is number of active shards $activeShards") val relocatingShards = cluster_health["relocating_shards"].toString() log.info("This is number of relocatingShards $relocatingShards") - val nodes_map = cluster_stats["nodes"] as Map - val process_map = nodes_map["process"] as Map - val cpu_map = process_map["cpu"] as Map -// val percent = cpu_map["percent"].toString() - val percent = (30..80).random().toString() - log.info("THIS IS CPU USAGE $percent") - val jvm_map = nodes_map["jvm"] as Map - val mem_map = jvm_map["mem"] as Map - val mem_used = mem_map["heap_used_in_bytes"] - val mem_avail = mem_map["heap_max_in_bytes"] - var jvm_pressure = "0.00" - - if (mem_used is Int && mem_avail is Int) { - val jvm_pressure_num = ((mem_used.toDouble() / mem_avail.toDouble()) * 100) - jvm_pressure = String.format(Locale.ROOT, "%.2f", jvm_pressure_num) - } - log.info("THIS IS JVM PRESSURE $jvm_pressure") - val clusterStatus_data = ClusterMetricsDataPoint( - ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, - current_time, - clusterStatusRandom - ) - val unassigned_shards_data = ClusterMetricsDataPoint( - ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, - current_time, - randomUnassignedShards - ) - val cpu_usage_data = ClusterMetricsDataPoint( - ClusterMetricsDataPoint.MetricType.CPU_USAGE, - current_time, - percent - ) - val jvm_data = ClusterMetricsDataPoint( - ClusterMetricsDataPoint.MetricType.JVM_PRESSURE, - current_time, - jvm_pressure - ) - val pendingTasksData = ClusterMetricsDataPoint( - ClusterMetricsDataPoint.MetricType.NUMBER_OF_PENDING_TASKS, - current_time, - numPending - ) - val activeShardsData = ClusterMetricsDataPoint( - ClusterMetricsDataPoint.MetricType.ACTIVE_SHARDS, - current_time, - activeShards - ) - val relocatingShardsData = ClusterMetricsDataPoint( - ClusterMetricsDataPoint.MetricType.RELOCATING_SHARDS, - current_time, - relocatingShards - ) + val nodesMap = nodeStats["nodes"] as Map + val keys = nodesMap.keys + log.info("this is nodesMap keys $keys") - val indexRequest_status = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(clusterStatus_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexRequest_shards = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(unassigned_shards_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexRequest_cpu = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(cpu_usage_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexRequest_jvm = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(jvm_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexRequest_pending = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(pendingTasksData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexRequest_active = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(activeShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexRequest_relocating = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(relocatingShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - - try { - val indexResponse: IndexResponse = client.suspendUntil { client.index(indexRequest_status, it) } - val indexResponse2: IndexResponse = client.suspendUntil { client.index(indexRequest_shards, it) } - val indexResponse3: IndexResponse = client.suspendUntil { client.index(indexRequest_cpu, it) } - val indexResponse4: IndexResponse = client.suspendUntil { client.index(indexRequest_jvm, it) } - val indexResponse5: IndexResponse = client.suspendUntil { client.index(indexRequest_pending, it) } - val indexResponse6: IndexResponse = client.suspendUntil { client.index(indexRequest_active, it) } - val indexResponse7: IndexResponse = client.suspendUntil { client.index(indexRequest_relocating, it) } - val failureReasons = checkShardsFailure(indexResponse) - val failureReasons2 = checkShardsFailure(indexResponse2) - val failureReasons3 = checkShardsFailure(indexResponse3) - val failureReasons4 = checkShardsFailure(indexResponse4) - val failureReasons5 = checkShardsFailure(indexResponse5) - val failureReasons6 = checkShardsFailure(indexResponse6) - val failureReasons7 = checkShardsFailure(indexResponse7) - if ( - failureReasons != null || - failureReasons2 != null || - failureReasons3 != null || - failureReasons4 != null || - failureReasons5 != null || - failureReasons6 != null || - failureReasons7 != null - ) { - log.info("richfu failed because $failureReasons") - log.info("richfu failed because $failureReasons2") - log.info("richfu failed because $failureReasons3") - log.info("richfu failed because $failureReasons4") - log.info("richfu failed because $failureReasons5") - log.info("richfu failed because $failureReasons6") - log.info("richfu failed because $failureReasons7") - return - } - } catch (t: Exception) { - log.info("richfu CLUSTER METRICS NOT WORK $t") - } +// log.info("THIS IS JVM PRESSURE $jvm_pressure") +// +// val clusterStatus_data = ClusterMetricsDataPoint( +// ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, +// current_time, +// clusterStatusRandom +// ) +// val unassigned_shards_data = ClusterMetricsDataPoint( +// ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, +// current_time, +// randomUnassignedShards +// ) +// val cpu_usage_data = ClusterMetricsDataPoint( +// ClusterMetricsDataPoint.MetricType.CPU_USAGE, +// current_time, +// percent +// ) +// val jvm_data = ClusterMetricsDataPoint( +// ClusterMetricsDataPoint.MetricType.JVM_PRESSURE, +// current_time, +// jvm_pressure +// ) +// val pendingTasksData = ClusterMetricsDataPoint( +// ClusterMetricsDataPoint.MetricType.NUMBER_OF_PENDING_TASKS, +// current_time, +// numPending +// ) +// val activeShardsData = ClusterMetricsDataPoint( +// ClusterMetricsDataPoint.MetricType.ACTIVE_SHARDS, +// current_time, +// activeShards +// ) +// val relocatingShardsData = ClusterMetricsDataPoint( +// ClusterMetricsDataPoint.MetricType.RELOCATING_SHARDS, +// current_time, +// relocatingShards +// ) +// +// val indexRequest_status = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .source(clusterStatus_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) +// val indexRequest_shards = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .source(unassigned_shards_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) +// val indexRequest_cpu = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .source(cpu_usage_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) +// val indexRequest_jvm = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .source(jvm_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) +// val indexRequest_pending = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .source(pendingTasksData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) +// val indexRequest_active = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .source(activeShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) +// val indexRequest_relocating = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .source(relocatingShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) +// +// try { +// val indexResponse: IndexResponse = client.suspendUntil { client.index(indexRequest_status, it) } +// val indexResponse2: IndexResponse = client.suspendUntil { client.index(indexRequest_shards, it) } +// val indexResponse3: IndexResponse = client.suspendUntil { client.index(indexRequest_cpu, it) } +// val indexResponse4: IndexResponse = client.suspendUntil { client.index(indexRequest_jvm, it) } +// val indexResponse5: IndexResponse = client.suspendUntil { client.index(indexRequest_pending, it) } +// val indexResponse6: IndexResponse = client.suspendUntil { client.index(indexRequest_active, it) } +// val indexResponse7: IndexResponse = client.suspendUntil { client.index(indexRequest_relocating, it) } +// val failureReasons = checkShardsFailure(indexResponse) +// val failureReasons2 = checkShardsFailure(indexResponse2) +// val failureReasons3 = checkShardsFailure(indexResponse3) +// val failureReasons4 = checkShardsFailure(indexResponse4) +// val failureReasons5 = checkShardsFailure(indexResponse5) +// val failureReasons6 = checkShardsFailure(indexResponse6) +// val failureReasons7 = checkShardsFailure(indexResponse7) +// if ( +// failureReasons != null || +// failureReasons2 != null || +// failureReasons3 != null || +// failureReasons4 != null || +// failureReasons5 != null || +// failureReasons6 != null || +// failureReasons7 != null +// ) { +// log.info("richfu failed because $failureReasons") +// log.info("richfu failed because $failureReasons2") +// log.info("richfu failed because $failureReasons3") +// log.info("richfu failed because $failureReasons4") +// log.info("richfu failed because $failureReasons5") +// log.info("richfu failed because $failureReasons6") +// log.info("richfu failed because $failureReasons7") +// return +// } +// } catch (t: Exception) { +// log.info("richfu CLUSTER METRICS NOT WORK $t") +// } } fun checkShardsFailure(response: IndexResponse): String? { val failureReasons = StringBuilder() From f783d91ebb067068a2c52666674eae2c03a87085 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 16:37:59 -0700 Subject: [PATCH 159/215] ./gr Signed-off-by: Richard --- .../opensearch/alerting/util/ClusterMetricsCoordinator.kt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index ba3241c99..276b2eeeb 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -12,10 +12,7 @@ import kotlinx.coroutines.launch import org.opensearch.action.ActionListener import org.opensearch.action.admin.cluster.health.ClusterHealthRequest import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest -import org.opensearch.action.index.IndexRequest import org.opensearch.action.index.IndexResponse -import org.opensearch.alerting.model.ClusterMetricsDataPoint -import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.settings.AlertingSettings.Companion.METRICS_EXECUTION_FREQUENCY import org.opensearch.alerting.settings.AlertingSettings.Companion.METRICS_STORE_TIME import org.opensearch.client.Client @@ -25,8 +22,6 @@ import org.opensearch.cluster.ClusterStateListener import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.XContentFactory import org.opensearch.index.query.QueryBuilders import org.opensearch.index.reindex.BulkByScrollResponse import org.opensearch.index.reindex.DeleteByQueryAction @@ -34,7 +29,6 @@ import org.opensearch.index.reindex.DeleteByQueryRequestBuilder import org.opensearch.threadpool.Scheduler import org.opensearch.threadpool.ThreadPool import java.time.Instant -import java.util.* import kotlin.coroutines.CoroutineContext private val log = org.apache.logging.log4j.LogManager.getLogger(ClusterMetricsCoordinator::class.java) From 89479f62ad3f6ecd01aa0b4082a71f427995367c Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 16:59:33 -0700 Subject: [PATCH 160/215] test Signed-off-by: Richard --- .../alerting/model/ClusterMetricsDataPoint.kt | 14 ++++++++++++-- .../alerting/util/ClusterMetricsCoordinator.kt | 17 +++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt index 8b50ccaa1..ddd15a094 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt @@ -11,21 +11,31 @@ import org.opensearch.common.xcontent.XContentBuilder data class ClusterMetricsDataPoint( var metric: MetricType, var timestamp: String, - var value: String + var value: String, + var minimum: String? = null, + var maximum: String? = null ) : ToXContent { companion object { val TIMESTAMP_FIELD = "timestamp" val VALUE_FIELD = "value" + val MINIMUM_FIELD = "minimum" + val MAXIMUM_FIELD = "maximum" } override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - return builder.startObject() + val output = builder.startObject() .startObject(metric.metricName) .field(TIMESTAMP_FIELD, timestamp) .field(VALUE_FIELD, value) + if (metric === MetricType.JVM_PRESSURE || metric === MetricType.CPU_USAGE) { + builder.field(MINIMUM_FIELD, minimum) + builder.field(MAXIMUM_FIELD, maximum) + } + output .endObject() .endObject() + return output } enum class MetricType( diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 276b2eeeb..11688a24d 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -138,6 +138,23 @@ class ClusterMetricsCoordinator( val nodesMap = nodeStats["nodes"] as Map val keys = nodesMap.keys log.info("this is nodesMap keys $keys") + val JVM_data = arrayListOf() + val CPU_data = arrayListOf() + + for (key in keys) { + val keyData = nodesMap[key] as Map + val osMap = keyData["os"] as Map + val cpuMap = osMap["cpu"] as Map + val percent = cpuMap["percent"].toString() + CPU_data.add(percent) + + val jvmMap = keyData["jvm"] as Map + val memMap = jvmMap["mem"] as Map + val pressure = memMap["heap_used_percent"].toString() + JVM_data.add(pressure) + } + log.info("this is CPU data $CPU_data") + log.info("this is JVM data $JVM_data") // log.info("THIS IS JVM PRESSURE $jvm_pressure") // From e932e0cfbd0a8f5ef22e94800f93a5dae00e6122 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 17:06:29 -0700 Subject: [PATCH 161/215] test Signed-off-by: Richard --- .../opensearch/alerting/util/ClusterMetricsCoordinator.kt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 11688a24d..023d5411a 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -143,15 +143,18 @@ class ClusterMetricsCoordinator( for (key in keys) { val keyData = nodesMap[key] as Map + log.info("this is keyData $keyData") val osMap = keyData["os"] as Map + log.info("This is osMap $osMap") val cpuMap = osMap["cpu"] as Map + log.info("This is cpuMap $cpuMap") val percent = cpuMap["percent"].toString() - CPU_data.add(percent) val jvmMap = keyData["jvm"] as Map + log.info("this is jvmMap $jvmMap") val memMap = jvmMap["mem"] as Map + log.info("this is memMap $memMap") val pressure = memMap["heap_used_percent"].toString() - JVM_data.add(pressure) } log.info("this is CPU data $CPU_data") log.info("this is JVM data $JVM_data") From 5b16b77a8b708c55bc85d853b4c58c7505a0b5d3 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 17:18:34 -0700 Subject: [PATCH 162/215] test Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 023d5411a..2b15d3b5c 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -120,7 +120,7 @@ class ClusterMetricsCoordinator( // cluster health for unassigned shards val cluster_health = client.admin().cluster().health(ClusterHealthRequest()).get().toMap() // cluster stats for cluster status (health), CPU usage, JVM pressure - var nodeStats = client.admin().cluster().nodesStats(NodesStatsRequest()).get().toMap() + var nodeStats = client.admin().cluster().nodesStats(NodesStatsRequest().addMetrics("process", "jvm")).get().toMap() ClusterMetricsVisualizationIndex.initFunc(client, clusterService) From d82d1665d53c2001e33251302e4fa6d83790ebb4 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 17:19:41 -0700 Subject: [PATCH 163/215] test Signed-off-by: Richard --- .../opensearch/alerting/util/ClusterMetricsCoordinator.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 2b15d3b5c..381dd2c36 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -144,9 +144,9 @@ class ClusterMetricsCoordinator( for (key in keys) { val keyData = nodesMap[key] as Map log.info("this is keyData $keyData") - val osMap = keyData["os"] as Map - log.info("This is osMap $osMap") - val cpuMap = osMap["cpu"] as Map + val processMap = keyData["process"] as Map + log.info("This is osMap $processMap") + val cpuMap = processMap["cpu"] as Map log.info("This is cpuMap $cpuMap") val percent = cpuMap["percent"].toString() From 214b690ae4606049fc2641ee10734a96a5819abb Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 17:24:02 -0700 Subject: [PATCH 164/215] test Signed-off-by: Richard --- .../alerting/util/ClusterMetricsCoordinator.kt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 381dd2c36..bf5062d48 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -138,8 +138,8 @@ class ClusterMetricsCoordinator( val nodesMap = nodeStats["nodes"] as Map val keys = nodesMap.keys log.info("this is nodesMap keys $keys") - val JVM_data = arrayListOf() - val CPU_data = arrayListOf() + val jvmData = arrayListOf() + val cpuData = arrayListOf() for (key in keys) { val keyData = nodesMap[key] as Map @@ -149,15 +149,17 @@ class ClusterMetricsCoordinator( val cpuMap = processMap["cpu"] as Map log.info("This is cpuMap $cpuMap") val percent = cpuMap["percent"].toString() + cpuData.add(percent) val jvmMap = keyData["jvm"] as Map log.info("this is jvmMap $jvmMap") val memMap = jvmMap["mem"] as Map log.info("this is memMap $memMap") val pressure = memMap["heap_used_percent"].toString() + jvmData.add(pressure) } - log.info("this is CPU data $CPU_data") - log.info("this is JVM data $JVM_data") + log.info("this is CPU data $cpuData") + log.info("this is JVM data $jvmData") // log.info("THIS IS JVM PRESSURE $jvm_pressure") // From f5b0837cb89de97b5fc367377233d9e8bb3716fb Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 17:41:36 -0700 Subject: [PATCH 165/215] test Signed-off-by: Richard --- .../util/ClusterMetricsCoordinator.kt | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index bf5062d48..c28828b7a 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -29,7 +29,9 @@ import org.opensearch.index.reindex.DeleteByQueryRequestBuilder import org.opensearch.threadpool.Scheduler import org.opensearch.threadpool.ThreadPool import java.time.Instant +import java.util.* import kotlin.coroutines.CoroutineContext +import kotlin.math.min private val log = org.apache.logging.log4j.LogManager.getLogger(ClusterMetricsCoordinator::class.java) @@ -138,8 +140,8 @@ class ClusterMetricsCoordinator( val nodesMap = nodeStats["nodes"] as Map val keys = nodesMap.keys log.info("this is nodesMap keys $keys") - val jvmData = arrayListOf() - val cpuData = arrayListOf() + val jvmData = arrayListOf() + val cpuData = arrayListOf() for (key in keys) { val keyData = nodesMap[key] as Map @@ -148,20 +150,28 @@ class ClusterMetricsCoordinator( log.info("This is osMap $processMap") val cpuMap = processMap["cpu"] as Map log.info("This is cpuMap $cpuMap") - val percent = cpuMap["percent"].toString() - cpuData.add(percent) + val percent = cpuMap["percent"] + cpuData.add(percent as Int) val jvmMap = keyData["jvm"] as Map log.info("this is jvmMap $jvmMap") val memMap = jvmMap["mem"] as Map log.info("this is memMap $memMap") - val pressure = memMap["heap_used_percent"].toString() - jvmData.add(pressure) + val pressure = memMap["heap_used_percent"] + jvmData.add(pressure as Int) } log.info("this is CPU data $cpuData") log.info("this is JVM data $jvmData") -// log.info("THIS IS JVM PRESSURE $jvm_pressure") + val minimumCPU = Collections.min(cpuData) + val maximumCPU = Collections.max(cpuData) + log.info("this is min CPU $minimumCPU") + log.info("this is max CPU $maximumCPU") + + val minimumJVM = Collections.min(jvmData) + val maximumJVM = Collections.max(jvmData) + log.info("this is max JVM $maximumJVM") + log.info("this is min JVM $minimumJVM") // // val clusterStatus_data = ClusterMetricsDataPoint( // ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, From 964da3d9091c3bb10787d58e5d7d646d5bae87bf Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 17:54:16 -0700 Subject: [PATCH 166/215] test Signed-off-by: Richard --- .../alerting/util/ClusterMetricsCoordinator.kt | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index c28828b7a..eb3fd16d7 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -31,7 +31,6 @@ import org.opensearch.threadpool.ThreadPool import java.time.Instant import java.util.* import kotlin.coroutines.CoroutineContext -import kotlin.math.min private val log = org.apache.logging.log4j.LogManager.getLogger(ClusterMetricsCoordinator::class.java) @@ -172,6 +171,19 @@ class ClusterMetricsCoordinator( val maximumJVM = Collections.max(jvmData) log.info("this is max JVM $maximumJVM") log.info("this is min JVM $minimumJVM") + + var avgCPU = 0 + var avgJVM = 0 + + for (i in cpuData.indices) { + avgCPU += cpuData[i] + avgJVM += jvmData[i] + } + + avgCPU /= cpuData.size + avgJVM /= jvmData.size + log.info("this is avgCPU $avgCPU") + log.info("this is avgJVM $avgJVM") // // val clusterStatus_data = ClusterMetricsDataPoint( // ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, From b47fb645d37552a45223debaa37de45144f656d2 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 17:59:25 -0700 Subject: [PATCH 167/215] test Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index eb3fd16d7..20cc8ca57 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -172,8 +172,8 @@ class ClusterMetricsCoordinator( log.info("this is max JVM $maximumJVM") log.info("this is min JVM $minimumJVM") - var avgCPU = 0 - var avgJVM = 0 + var avgCPU = 0.0 + var avgJVM = 0.0 for (i in cpuData.indices) { avgCPU += cpuData[i] From fd727a7faee29b592eed8f27763d949da30d030e Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 29 Jul 2022 18:07:52 -0700 Subject: [PATCH 168/215] test Signed-off-by: Richard --- .../util/ClusterMetricsCoordinator.kt | 210 +++++++++--------- 1 file changed, 110 insertions(+), 100 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 20cc8ca57..d236eead5 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -12,7 +12,10 @@ import kotlinx.coroutines.launch import org.opensearch.action.ActionListener import org.opensearch.action.admin.cluster.health.ClusterHealthRequest import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest +import org.opensearch.action.index.IndexRequest import org.opensearch.action.index.IndexResponse +import org.opensearch.alerting.model.ClusterMetricsDataPoint +import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.settings.AlertingSettings.Companion.METRICS_EXECUTION_FREQUENCY import org.opensearch.alerting.settings.AlertingSettings.Companion.METRICS_STORE_TIME import org.opensearch.client.Client @@ -22,6 +25,8 @@ import org.opensearch.cluster.ClusterStateListener import org.opensearch.cluster.service.ClusterService import org.opensearch.common.component.LifecycleListener import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentFactory import org.opensearch.index.query.QueryBuilders import org.opensearch.index.reindex.BulkByScrollResponse import org.opensearch.index.reindex.DeleteByQueryAction @@ -162,116 +167,121 @@ class ClusterMetricsCoordinator( log.info("this is CPU data $cpuData") log.info("this is JVM data $jvmData") - val minimumCPU = Collections.min(cpuData) - val maximumCPU = Collections.max(cpuData) + val minimumCPU = Collections.min(cpuData).toString() + val maximumCPU = Collections.max(cpuData).toString() log.info("this is min CPU $minimumCPU") log.info("this is max CPU $maximumCPU") - val minimumJVM = Collections.min(jvmData) - val maximumJVM = Collections.max(jvmData) + val minimumJVM = Collections.min(jvmData).toString() + val maximumJVM = Collections.max(jvmData).toString() log.info("this is max JVM $maximumJVM") log.info("this is min JVM $minimumJVM") - var avgCPU = 0.0 - var avgJVM = 0.0 + var avgCPUcalc = 0.0 + var avgJVMcalc = 0.0 for (i in cpuData.indices) { - avgCPU += cpuData[i] - avgJVM += jvmData[i] + avgCPUcalc += cpuData[i] + avgJVMcalc += jvmData[i] } - avgCPU /= cpuData.size - avgJVM /= jvmData.size - log.info("this is avgCPU $avgCPU") - log.info("this is avgJVM $avgJVM") -// -// val clusterStatus_data = ClusterMetricsDataPoint( -// ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, -// current_time, -// clusterStatusRandom -// ) -// val unassigned_shards_data = ClusterMetricsDataPoint( -// ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, -// current_time, -// randomUnassignedShards -// ) -// val cpu_usage_data = ClusterMetricsDataPoint( -// ClusterMetricsDataPoint.MetricType.CPU_USAGE, -// current_time, -// percent -// ) -// val jvm_data = ClusterMetricsDataPoint( -// ClusterMetricsDataPoint.MetricType.JVM_PRESSURE, -// current_time, -// jvm_pressure -// ) -// val pendingTasksData = ClusterMetricsDataPoint( -// ClusterMetricsDataPoint.MetricType.NUMBER_OF_PENDING_TASKS, -// current_time, -// numPending -// ) -// val activeShardsData = ClusterMetricsDataPoint( -// ClusterMetricsDataPoint.MetricType.ACTIVE_SHARDS, -// current_time, -// activeShards -// ) -// val relocatingShardsData = ClusterMetricsDataPoint( -// ClusterMetricsDataPoint.MetricType.RELOCATING_SHARDS, -// current_time, -// relocatingShards -// ) -// -// val indexRequest_status = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .source(clusterStatus_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) -// val indexRequest_shards = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .source(unassigned_shards_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) -// val indexRequest_cpu = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .source(cpu_usage_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) -// val indexRequest_jvm = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .source(jvm_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) -// val indexRequest_pending = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .source(pendingTasksData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) -// val indexRequest_active = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .source(activeShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) -// val indexRequest_relocating = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .source(relocatingShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) -// -// try { -// val indexResponse: IndexResponse = client.suspendUntil { client.index(indexRequest_status, it) } -// val indexResponse2: IndexResponse = client.suspendUntil { client.index(indexRequest_shards, it) } -// val indexResponse3: IndexResponse = client.suspendUntil { client.index(indexRequest_cpu, it) } -// val indexResponse4: IndexResponse = client.suspendUntil { client.index(indexRequest_jvm, it) } -// val indexResponse5: IndexResponse = client.suspendUntil { client.index(indexRequest_pending, it) } -// val indexResponse6: IndexResponse = client.suspendUntil { client.index(indexRequest_active, it) } -// val indexResponse7: IndexResponse = client.suspendUntil { client.index(indexRequest_relocating, it) } -// val failureReasons = checkShardsFailure(indexResponse) -// val failureReasons2 = checkShardsFailure(indexResponse2) -// val failureReasons3 = checkShardsFailure(indexResponse3) -// val failureReasons4 = checkShardsFailure(indexResponse4) -// val failureReasons5 = checkShardsFailure(indexResponse5) -// val failureReasons6 = checkShardsFailure(indexResponse6) -// val failureReasons7 = checkShardsFailure(indexResponse7) -// if ( -// failureReasons != null || -// failureReasons2 != null || -// failureReasons3 != null || -// failureReasons4 != null || -// failureReasons5 != null || -// failureReasons6 != null || -// failureReasons7 != null -// ) { -// log.info("richfu failed because $failureReasons") -// log.info("richfu failed because $failureReasons2") -// log.info("richfu failed because $failureReasons3") -// log.info("richfu failed because $failureReasons4") -// log.info("richfu failed because $failureReasons5") -// log.info("richfu failed because $failureReasons6") -// log.info("richfu failed because $failureReasons7") -// return -// } -// } catch (t: Exception) { -// log.info("richfu CLUSTER METRICS NOT WORK $t") -// } + avgCPUcalc /= cpuData.size + avgJVMcalc /= jvmData.size + + val avgCPU = String.format("%.2f", avgCPUcalc) + val avgJVM = String.format("%.2f", avgJVMcalc) + + val clusterStatus_data = ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, + current_time, + clusterStatus + ) + val unassigned_shards_data = ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, + current_time, + unassignedShards + ) + val cpu_usage_data = ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.CPU_USAGE, + current_time, + avgCPU, + minimumCPU, + maximumCPU + ) + val jvm_data = ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.JVM_PRESSURE, + current_time, + avgJVM, + minimumJVM, + maximumJVM + ) + val pendingTasksData = ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.NUMBER_OF_PENDING_TASKS, + current_time, + numPending + ) + val activeShardsData = ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.ACTIVE_SHARDS, + current_time, + activeShards + ) + val relocatingShardsData = ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.RELOCATING_SHARDS, + current_time, + relocatingShards + ) + + val indexRequest_status = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(clusterStatus_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + val indexRequest_shards = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(unassigned_shards_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + val indexRequest_cpu = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(cpu_usage_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + val indexRequest_jvm = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(jvm_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + val indexRequest_pending = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(pendingTasksData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + val indexRequest_active = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(activeShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + val indexRequest_relocating = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(relocatingShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + + try { + val indexResponse: IndexResponse = client.suspendUntil { client.index(indexRequest_status, it) } + val indexResponse2: IndexResponse = client.suspendUntil { client.index(indexRequest_shards, it) } + val indexResponse3: IndexResponse = client.suspendUntil { client.index(indexRequest_cpu, it) } + val indexResponse4: IndexResponse = client.suspendUntil { client.index(indexRequest_jvm, it) } + val indexResponse5: IndexResponse = client.suspendUntil { client.index(indexRequest_pending, it) } + val indexResponse6: IndexResponse = client.suspendUntil { client.index(indexRequest_active, it) } + val indexResponse7: IndexResponse = client.suspendUntil { client.index(indexRequest_relocating, it) } + val failureReasons = checkShardsFailure(indexResponse) + val failureReasons2 = checkShardsFailure(indexResponse2) + val failureReasons3 = checkShardsFailure(indexResponse3) + val failureReasons4 = checkShardsFailure(indexResponse4) + val failureReasons5 = checkShardsFailure(indexResponse5) + val failureReasons6 = checkShardsFailure(indexResponse6) + val failureReasons7 = checkShardsFailure(indexResponse7) + if ( + failureReasons != null || + failureReasons2 != null || + failureReasons3 != null || + failureReasons4 != null || + failureReasons5 != null || + failureReasons6 != null || + failureReasons7 != null + ) { + log.info("richfu failed because $failureReasons") + log.info("richfu failed because $failureReasons2") + log.info("richfu failed because $failureReasons3") + log.info("richfu failed because $failureReasons4") + log.info("richfu failed because $failureReasons5") + log.info("richfu failed because $failureReasons6") + log.info("richfu failed because $failureReasons7") + return + } + } catch (t: Exception) { + log.info("richfu CLUSTER METRICS NOT WORK $t") + } } fun checkShardsFailure(response: IndexResponse): String? { val failureReasons = StringBuilder() From 7678861123f8ebaed5c513aaa3b867f8226d3ebe Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 1 Aug 2022 13:58:57 -0700 Subject: [PATCH 169/215] test Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 7bc1972ae..cd1e2fa4e 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -96,23 +96,23 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { assertTrue(flag) } - fun `test frequency`() { - client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "2m") - Thread.sleep(300000) - - val response = getResponse() - val hits = getHits(response) - val docs = hits["hits"] as ArrayList> - val times = mutableSetOf() - - for (doc in docs) { - val source = doc["_source"] as Map> - val metricType = source.keys.first() - times.add(source[metricType]?.get("timestamp").toString()) - } - logger.info("this is the times Set length ${times.size}") - assertFalse(true) - } +// fun `test frequency`() { +// client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "2m") +// Thread.sleep(300000) +// +// val response = getResponse() +// val hits = getHits(response) +// val docs = hits["hits"] as ArrayList> +// val times = mutableSetOf() +// +// for (doc in docs) { +// val source = doc["_source"] as Map> +// val metricType = source.keys.first() +// times.add(source[metricType]?.get("timestamp").toString()) +// } +// logger.info("this is the times Set length ${times.size}") +// assertFalse(true) +// } private fun generateData() { client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1s") From 053c43a24cfc3b2a35847b41f4d7f09e21665812 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 1 Aug 2022 14:17:21 -0700 Subject: [PATCH 170/215] test Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index cd1e2fa4e..7bc1972ae 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -96,23 +96,23 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { assertTrue(flag) } -// fun `test frequency`() { -// client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "2m") -// Thread.sleep(300000) -// -// val response = getResponse() -// val hits = getHits(response) -// val docs = hits["hits"] as ArrayList> -// val times = mutableSetOf() -// -// for (doc in docs) { -// val source = doc["_source"] as Map> -// val metricType = source.keys.first() -// times.add(source[metricType]?.get("timestamp").toString()) -// } -// logger.info("this is the times Set length ${times.size}") -// assertFalse(true) -// } + fun `test frequency`() { + client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "2m") + Thread.sleep(300000) + + val response = getResponse() + val hits = getHits(response) + val docs = hits["hits"] as ArrayList> + val times = mutableSetOf() + + for (doc in docs) { + val source = doc["_source"] as Map> + val metricType = source.keys.first() + times.add(source[metricType]?.get("timestamp").toString()) + } + logger.info("this is the times Set length ${times.size}") + assertFalse(true) + } private fun generateData() { client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1s") From 83fd357b2292861b889791f3932a06f7d60b730b Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 1 Aug 2022 14:17:42 -0700 Subject: [PATCH 171/215] test Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 7bc1972ae..484486c9a 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -110,7 +110,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { val metricType = source.keys.first() times.add(source[metricType]?.get("timestamp").toString()) } - logger.info("this is the times Set length ${times.size}") + logger.info("this is the times Set length ${times.size}, this is times set data $times") assertFalse(true) } From 09685331d5073cafc5b2c71975116fd0a12756b3 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 1 Aug 2022 14:28:00 -0700 Subject: [PATCH 172/215] test Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 484486c9a..b563e7fbd 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -111,6 +111,9 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { times.add(source[metricType]?.get("timestamp").toString()) } logger.info("this is the times Set length ${times.size}, this is times set data $times") + val time1 = times.elementAt(times.size-1) + val time2 = times.elementAt(times.size-2) + logger.info("this is time1 $time1, and this is time2 $time2") assertFalse(true) } From f356d442447307fc47ec792ffae083f779525867 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 1 Aug 2022 14:29:19 -0700 Subject: [PATCH 173/215] test Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index b563e7fbd..89d0e74e4 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -111,8 +111,8 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { times.add(source[metricType]?.get("timestamp").toString()) } logger.info("this is the times Set length ${times.size}, this is times set data $times") - val time1 = times.elementAt(times.size-1) - val time2 = times.elementAt(times.size-2) + val time1 = times.elementAt(times.size - 1) + val time2 = times.elementAt(times.size - 2) logger.info("this is time1 $time1, and this is time2 $time2") assertFalse(true) } From a61bc1b3b2b0c19d49f75a28c9bbc6e5a1ca94ec Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 1 Aug 2022 14:58:07 -0700 Subject: [PATCH 174/215] test Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 89d0e74e4..1c36057b8 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -111,9 +111,11 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { times.add(source[metricType]?.get("timestamp").toString()) } logger.info("this is the times Set length ${times.size}, this is times set data $times") - val time1 = times.elementAt(times.size - 1) - val time2 = times.elementAt(times.size - 2) + var time1 = Instant.parse(times.elementAt(times.size-1)) + var time2 = Instant.parse(times.elementAt(times.size-2)) logger.info("this is time1 $time1, and this is time2 $time2") + val diff = time1.until(time2, ChronoUnit.SECONDS) + logger.info("this is diff between time1 and time2 $diff") assertFalse(true) } From c08ee67d942732ed8a3b81af61f6214ef118cea5 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 1 Aug 2022 14:59:02 -0700 Subject: [PATCH 175/215] test Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 1c36057b8..c5c0b817a 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -111,8 +111,8 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { times.add(source[metricType]?.get("timestamp").toString()) } logger.info("this is the times Set length ${times.size}, this is times set data $times") - var time1 = Instant.parse(times.elementAt(times.size-1)) - var time2 = Instant.parse(times.elementAt(times.size-2)) + var time1 = Instant.parse(times.elementAt(times.size - 1)) + var time2 = Instant.parse(times.elementAt(times.size - 2)) logger.info("this is time1 $time1, and this is time2 $time2") val diff = time1.until(time2, ChronoUnit.SECONDS) logger.info("this is diff between time1 and time2 $diff") From e9890361e4e00de57998db6609257c7693d50283 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 1 Aug 2022 15:06:18 -0700 Subject: [PATCH 176/215] test Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index c5c0b817a..9a0d6c1f0 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -114,7 +114,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { var time1 = Instant.parse(times.elementAt(times.size - 1)) var time2 = Instant.parse(times.elementAt(times.size - 2)) logger.info("this is time1 $time1, and this is time2 $time2") - val diff = time1.until(time2, ChronoUnit.SECONDS) + val diff = time2.until(time1, ChronoUnit.MINUTES) logger.info("this is diff between time1 and time2 $diff") assertFalse(true) } From 009408816a8a8eb65236da6cc693b014539311f5 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 1 Aug 2022 15:14:50 -0700 Subject: [PATCH 177/215] test Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 9a0d6c1f0..67ce9b9fb 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -116,7 +116,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { logger.info("this is time1 $time1, and this is time2 $time2") val diff = time2.until(time1, ChronoUnit.MINUTES) logger.info("this is diff between time1 and time2 $diff") - assertFalse(true) + assertEquals(diff, 2) } private fun generateData() { From 3a3f01e7bf595a705dc520992c3720ef33abdce8 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 1 Aug 2022 16:24:37 -0700 Subject: [PATCH 178/215] test Signed-off-by: Richard --- .../alerting/model/ClusterMetricsDataPoint.kt | 6 ++++ .../util/ClusterMetricsCoordinator.kt | 28 ++++++++++++++++++- .../ClusterMetricsCoordinatorIT.kt | 7 ++--- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt index ddd15a094..1404c2d08 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsDataPoint.kt @@ -61,6 +61,12 @@ data class ClusterMetricsDataPoint( ), RELOCATING_SHARDS( "relocating_shards" + ), + NUMBER_OF_NODES( + "number_of_nodes" + ), + NUMBER_OF_DATA_NODES( + "number_of_data_nodes" ) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index d236eead5..8bb50aec6 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -140,6 +140,10 @@ class ClusterMetricsCoordinator( log.info("this is number of active shards $activeShards") val relocatingShards = cluster_health["relocating_shards"].toString() log.info("This is number of relocatingShards $relocatingShards") + val numNodes = cluster_health["number_of_nodes"].toString() + log.info("This is number of nodes (total) $numNodes") + val numDataNodes = cluster_health["number_of_data_nodes"].toString() + log.info("This is number of data nodes $numDataNodes") val nodesMap = nodeStats["nodes"] as Map val keys = nodesMap.keys @@ -230,6 +234,16 @@ class ClusterMetricsCoordinator( current_time, relocatingShards ) + val nodesData = ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.NUMBER_OF_NODES, + current_time, + numNodes + ) + val dataNodesData = ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.NUMBER_OF_DATA_NODES, + current_time, + numDataNodes + ) val indexRequest_status = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) .source(clusterStatus_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) @@ -245,6 +259,10 @@ class ClusterMetricsCoordinator( .source(activeShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) val indexRequest_relocating = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) .source(relocatingShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + val indexRequest_nodes = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(nodesData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + val indexRequest_dataNodes = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(dataNodesData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) try { val indexResponse: IndexResponse = client.suspendUntil { client.index(indexRequest_status, it) } @@ -254,6 +272,8 @@ class ClusterMetricsCoordinator( val indexResponse5: IndexResponse = client.suspendUntil { client.index(indexRequest_pending, it) } val indexResponse6: IndexResponse = client.suspendUntil { client.index(indexRequest_active, it) } val indexResponse7: IndexResponse = client.suspendUntil { client.index(indexRequest_relocating, it) } + val indexResponse8: IndexResponse = client.suspendUntil { client.index(indexRequest_nodes, it) } + val indexResponse9: IndexResponse = client.suspendUntil { client.index(indexRequest_dataNodes, it) } val failureReasons = checkShardsFailure(indexResponse) val failureReasons2 = checkShardsFailure(indexResponse2) val failureReasons3 = checkShardsFailure(indexResponse3) @@ -261,6 +281,8 @@ class ClusterMetricsCoordinator( val failureReasons5 = checkShardsFailure(indexResponse5) val failureReasons6 = checkShardsFailure(indexResponse6) val failureReasons7 = checkShardsFailure(indexResponse7) + val failureReasons8 = checkShardsFailure(indexResponse8) + val failureReasons9 = checkShardsFailure(indexResponse9) if ( failureReasons != null || failureReasons2 != null || @@ -268,7 +290,9 @@ class ClusterMetricsCoordinator( failureReasons4 != null || failureReasons5 != null || failureReasons6 != null || - failureReasons7 != null + failureReasons7 != null || + failureReasons8 != null || + failureReasons9 != null ) { log.info("richfu failed because $failureReasons") log.info("richfu failed because $failureReasons2") @@ -277,6 +301,8 @@ class ClusterMetricsCoordinator( log.info("richfu failed because $failureReasons5") log.info("richfu failed because $failureReasons6") log.info("richfu failed because $failureReasons7") + log.info("failed because $failureReasons8") + log.info("failed because $failureReasons9") return } } catch (t: Exception) { diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 67ce9b9fb..b95fc6253 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -110,12 +110,9 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { val metricType = source.keys.first() times.add(source[metricType]?.get("timestamp").toString()) } - logger.info("this is the times Set length ${times.size}, this is times set data $times") - var time1 = Instant.parse(times.elementAt(times.size - 1)) - var time2 = Instant.parse(times.elementAt(times.size - 2)) - logger.info("this is time1 $time1, and this is time2 $time2") + val time1 = Instant.parse(times.elementAt(times.size - 1)) + val time2 = Instant.parse(times.elementAt(times.size - 2)) val diff = time2.until(time1, ChronoUnit.MINUTES) - logger.info("this is diff between time1 and time2 $diff") assertEquals(diff, 2) } From 64dd014ba5f315a8357b7879e608c8ff8e3f9d3d Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 1 Aug 2022 17:05:07 -0700 Subject: [PATCH 179/215] test Signed-off-by: Richard --- .../util/ClusterMetricsCoordinator.kt | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 8bb50aec6..00e4c5006 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -423,5 +423,33 @@ class ClusterMetricsCoordinator( } ) log.info("deleted number of relocating shards from $documentAge/$unitTime ago") + + DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) + .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .filter(QueryBuilders.rangeQuery("number_of_nodes.timestamp").lte("now-$documentAge/$unitTime")) + .execute( + object : ActionListener { + override fun onResponse(response: BulkByScrollResponse) { + } + + override fun onFailure(t: Exception) { + } + } + ) + log.info("deleted number of nodes from $documentAge/$unitTime ago") + + DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) + .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .filter(QueryBuilders.rangeQuery("number_of_data_nodes.timestamp").lte("now-$documentAge/$unitTime")) + .execute( + object : ActionListener { + override fun onResponse(response: BulkByScrollResponse) { + } + + override fun onFailure(t: Exception) { + } + } + ) + log.info("deleted number of data nodes from $documentAge/$unitTime ago") } } From 2c64d0d6987aa87198c95a332c6e246dc6f5f228 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 2 Aug 2022 15:44:48 -0700 Subject: [PATCH 180/215] test Signed-off-by: Richard --- .../alerting/settings/AlertingSettings.kt | 83 ++++++++++++++++--- 1 file changed, 72 insertions(+), 11 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index 9cab638e8..a97894afb 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -15,11 +15,26 @@ import java.util.concurrent.TimeUnit */ class AlertingSettings { + init { + + } + companion object { const val MONITOR_MAX_INPUTS = 1 const val MONITOR_MAX_TRIGGERS = 10 const val DEFAULT_MAX_ACTIONABLE_ALERT_COUNT = 50L + val METRICS_EXECUTION_FREQUENCY_DEFAULT = Setting.positiveTimeSetting( + "plugins.alerting.cluster_metrics.execution_frequency", + TimeValue(15, TimeUnit.MINUTES), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + val METRICS_STORE_TIME_DEFAULT = Setting.positiveTimeSetting( + "plugins.alerting.cluster_metrics.metrics_history_max_age", + TimeValue(7, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + val MINIMUM_TIME_VALUE = TimeValue(1, TimeUnit.SECONDS) val ALERTING_MAX_MONITORS = Setting.intSetting( "plugins.alerting.monitor.max_monitors", @@ -157,16 +172,62 @@ class AlertingSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ) - val METRICS_STORE_TIME = Setting.positiveTimeSetting( - "plugins.alerting.cluster_metrics.metrics_history_max_age", - TimeValue(7, TimeUnit.DAYS), - Setting.Property.NodeScope, Setting.Property.Dynamic - ) - - val METRICS_EXECUTION_FREQUENCY = Setting.positiveTimeSetting( - "plugins.alerting.cluster_metrics.execution_frequency", - TimeValue(15, TimeUnit.MINUTES), - Setting.Property.NodeScope, Setting.Property.Dynamic - ) + val METRICS_STORE_TIME = Setting.timeSetting( + METRICS_STORE_TIME_DEFAULT.key, + METRICS_STORE_TIME_DEFAULT, + StorageTimeValidator(), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val METRICS_EXECUTION_FREQUENCY = Setting.timeSetting( + METRICS_EXECUTION_FREQUENCY_DEFAULT.key, + METRICS_EXECUTION_FREQUENCY_DEFAULT, + ExecutionFrequencyValidator(), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + internal class ExecutionFrequencyValidator : Setting.Validator { + override fun validate(value: TimeValue) {} + + override fun validate(value: TimeValue, settings: Map, Any>) { + val executionFrequency = settings[METRICS_EXECUTION_FREQUENCY] as TimeValue + validateExecutionFrequency(executionFrequency, value) + } + + override fun settings(): MutableIterator> { + val settings = mutableListOf>( + METRICS_EXECUTION_FREQUENCY + ) + return settings.iterator() + } + } + + internal class StorageTimeValidator : Setting.Validator { + override fun validate(value: TimeValue) {} + + override fun validate(value: TimeValue, settings: Map, Any>) { + val storageTime = settings[METRICS_STORE_TIME] as TimeValue + validateExecutionFrequency(value, storageTime) + } + + override fun settings(): MutableIterator> { + val settings = mutableListOf>( + METRICS_STORE_TIME + ) + return settings.iterator() + } + } + fun validateExecutionFrequency(executionFrequency: TimeValue, storageTime: TimeValue) { + if (executionFrequency < MINIMUM_TIME_VALUE || storageTime < MINIMUM_TIME_VALUE) { + throw java.lang.IllegalArgumentException( + "Execution frequency or storage time cannot be less than $MINIMUM_TIME_VALUE." + ) + } + if (executionFrequency < storageTime) return + if (executionFrequency > storageTime) { + throw java.lang.IllegalArgumentException( + "Execution frequency cannot be greater than the storage time." + ) + } + } } } From c00a8fca960cd89d07e9f33016ca6199d5059085 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 2 Aug 2022 15:45:36 -0700 Subject: [PATCH 181/215] test Signed-off-by: Richard --- .../org/opensearch/alerting/settings/AlertingSettings.kt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index a97894afb..9ba417b82 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -15,10 +15,6 @@ import java.util.concurrent.TimeUnit */ class AlertingSettings { - init { - - } - companion object { const val MONITOR_MAX_INPUTS = 1 From 17f903a472212dc7d10ac5b583feaff1c0ee99cc Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 2 Aug 2022 15:58:19 -0700 Subject: [PATCH 182/215] test Signed-off-by: Richard --- .../util/ClusterMetricsCoordinator.kt | 84 ++++++++++++------- 1 file changed, 55 insertions(+), 29 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 00e4c5006..9316079a3 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -76,37 +76,63 @@ class ClusterMetricsCoordinator( } override fun clusterChanged(event: ClusterChangedEvent?) { - if (metricsStoreTime > metricsExecutionFrequency) { - val scheduledJobCollection = Runnable { - launch { - createDocs(client as NodeClient, clusterService) - deleteDocs(client) - } - } - if (isCollectionUpdated || isDeletionUpdated) { - dataPointCollectionDeletionJob?.cancel() - log.info("cancelled data collection and deletion jobs ") - isRunningFlag = false - log.info("detected changes to settings, resetting running, deletion and collection flags to false") - isDeletionUpdated = false - isCollectionUpdated = false - } - // if cluster changed event occurs (if settings are changed) - // cancel current scheduled job and change runningFlag to false. - // maybe have individual runningFlags for collection and deletion? - if (event!!.localNodeMaster() && !isRunningFlag) { - log.info("cluster changed metricsExecutionFrequency = $metricsExecutionFrequency") - dataPointCollectionDeletionJob = threadPool.scheduleWithFixedDelay( - scheduledJobCollection, - metricsExecutionFrequency, - ThreadPool.Names.SYSTEM_WRITE - ) - isRunningFlag = true +// if (metricsStoreTime > metricsExecutionFrequency) { +// val scheduledJobCollection = Runnable { +// launch { +// createDocs(client as NodeClient, clusterService) +// deleteDocs(client) +// } +// } +// if (isCollectionUpdated || isDeletionUpdated) { +// dataPointCollectionDeletionJob?.cancel() +// log.info("cancelled data collection and deletion jobs ") +// isRunningFlag = false +// log.info("detected changes to settings, resetting running, deletion and collection flags to false") +// isDeletionUpdated = false +// isCollectionUpdated = false +// } +// // if cluster changed event occurs (if settings are changed) +// // cancel current scheduled job and change runningFlag to false. +// // maybe have individual runningFlags for collection and deletion? +// if (event!!.localNodeMaster() && !isRunningFlag) { +// log.info("cluster changed metricsExecutionFrequency = $metricsExecutionFrequency") +// dataPointCollectionDeletionJob = threadPool.scheduleWithFixedDelay( +// scheduledJobCollection, +// metricsExecutionFrequency, +// ThreadPool.Names.SYSTEM_WRITE +// ) +// isRunningFlag = true +// } +// } else { +// throw java.lang.IllegalArgumentException( +// "Execution frequency ($metricsExecutionFrequency) can not be greater than the storage time ($metricsStoreTime). " +// ) +// } + val scheduledJobCollection = Runnable { + launch { + createDocs(client as NodeClient, clusterService) + deleteDocs(client) } - } else { - throw java.lang.IllegalArgumentException( - "Execution frequency ($metricsExecutionFrequency) can not be greater than the storage time ($metricsStoreTime). " + } + if (isCollectionUpdated || isDeletionUpdated) { + dataPointCollectionDeletionJob?.cancel() + log.info("cancelled data collection and deletion jobs ") + isRunningFlag = false + log.info("detected changes to settings, resetting running, deletion and collection flags to false") + isDeletionUpdated = false + isCollectionUpdated = false + } + // if cluster changed event occurs (if settings are changed) + // cancel current scheduled job and change runningFlag to false. + // maybe have individual runningFlags for collection and deletion? + if (event!!.localNodeMaster() && !isRunningFlag) { + log.info("cluster changed metricsExecutionFrequency = $metricsExecutionFrequency") + dataPointCollectionDeletionJob = threadPool.scheduleWithFixedDelay( + scheduledJobCollection, + metricsExecutionFrequency, + ThreadPool.Names.SYSTEM_WRITE ) + isRunningFlag = true } } From df6ce4ce893cf1b58759e055399ec169910ca7cd Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 2 Aug 2022 16:16:47 -0700 Subject: [PATCH 183/215] test Signed-off-by: Richard --- .../org/opensearch/alerting/settings/AlertingSettings.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index 9ba417b82..72e4c86c3 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -13,6 +13,8 @@ import java.util.concurrent.TimeUnit /** * settings specific to [AlertingPlugin]. These settings include things like history index max age, request timeout, etc... */ +private val log = org.apache.logging.log4j.LogManager.getLogger(AlertingSettings::class.java) + class AlertingSettings { companion object { @@ -213,6 +215,9 @@ class AlertingSettings { } } fun validateExecutionFrequency(executionFrequency: TimeValue, storageTime: TimeValue) { + log.info("THIS IS VALIDATEEXECUTIONFREQUENCY PARAMS $executionFrequency, $storageTime") + log.info("THIS IS MINIMUM_TIME_VALUE $MINIMUM_TIME_VALUE") + if (executionFrequency < MINIMUM_TIME_VALUE || storageTime < MINIMUM_TIME_VALUE) { throw java.lang.IllegalArgumentException( "Execution frequency or storage time cannot be less than $MINIMUM_TIME_VALUE." From 873571c3dbc40df6e66bdd8dbc68157b3229a755 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 2 Aug 2022 16:21:56 -0700 Subject: [PATCH 184/215] test Signed-off-by: Richard --- .../opensearch/alerting/settings/AlertingSettings.kt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index 72e4c86c3..1dbea9703 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -187,8 +187,8 @@ class AlertingSettings { override fun validate(value: TimeValue) {} override fun validate(value: TimeValue, settings: Map, Any>) { - val executionFrequency = settings[METRICS_EXECUTION_FREQUENCY] as TimeValue - validateExecutionFrequency(executionFrequency, value) + val storageTime = settings[METRICS_STORE_TIME] as TimeValue + validateExecutionFrequency(value, storageTime) } override fun settings(): MutableIterator> { @@ -203,8 +203,8 @@ class AlertingSettings { override fun validate(value: TimeValue) {} override fun validate(value: TimeValue, settings: Map, Any>) { - val storageTime = settings[METRICS_STORE_TIME] as TimeValue - validateExecutionFrequency(value, storageTime) + val executionFrequency = settings[METRICS_EXECUTION_FREQUENCY] as TimeValue + validateExecutionFrequency(executionFrequency, value) } override fun settings(): MutableIterator> { @@ -219,13 +219,13 @@ class AlertingSettings { log.info("THIS IS MINIMUM_TIME_VALUE $MINIMUM_TIME_VALUE") if (executionFrequency < MINIMUM_TIME_VALUE || storageTime < MINIMUM_TIME_VALUE) { - throw java.lang.IllegalArgumentException( + throw IllegalArgumentException( "Execution frequency or storage time cannot be less than $MINIMUM_TIME_VALUE." ) } if (executionFrequency < storageTime) return if (executionFrequency > storageTime) { - throw java.lang.IllegalArgumentException( + throw IllegalArgumentException( "Execution frequency cannot be greater than the storage time." ) } From 5ab8857bab7414078f97f2ddc9e83863ff9cc0aa Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 2 Aug 2022 16:29:36 -0700 Subject: [PATCH 185/215] test Signed-off-by: Richard --- .../org/opensearch/alerting/settings/AlertingSettings.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index 1dbea9703..39a1060d5 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -187,8 +187,10 @@ class AlertingSettings { override fun validate(value: TimeValue) {} override fun validate(value: TimeValue, settings: Map, Any>) { - val storageTime = settings[METRICS_STORE_TIME] as TimeValue - validateExecutionFrequency(value, storageTime) + log.info("THIS IS SETTING $settings") + log.info("THIS IS VALUE $value") + val storageTime = settings[METRICS_STORE_TIME] + validateExecutionFrequency(value, storageTime as TimeValue) } override fun settings(): MutableIterator> { From c5eea3fab45d7ea5aaaf568e8f0c7dfb0808eb3f Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 2 Aug 2022 16:56:33 -0700 Subject: [PATCH 186/215] test Signed-off-by: Richard --- .../org/opensearch/alerting/AlertingPlugin.kt | 3 +- .../alerting/settings/AlertingSettings.kt | 65 ++++++++----------- 2 files changed, 30 insertions(+), 38 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index e63a8eba0..f109d9488 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -145,6 +145,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R lateinit var clusterService: ClusterService lateinit var destinationMigrationCoordinator: DestinationMigrationCoordinator lateinit var clusterMetricsCoordinator: ClusterMetricsCoordinator + lateinit var alertingSettings: AlertingSettings override fun getRestHandlers( settings: Settings, @@ -241,9 +242,9 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R docLevelMonitorQueries = DocLevelMonitorQueries(client, clusterService) scheduler = JobScheduler(threadPool, runner) sweeper = JobSweeper(environment.settings(), client, clusterService, threadPool, xContentRegistry, scheduler, ALERTING_JOB_TYPES) + alertingSettings = AlertingSettings(client, settings) destinationMigrationCoordinator = DestinationMigrationCoordinator(client, clusterService, threadPool, scheduledJobIndices) clusterMetricsCoordinator = ClusterMetricsCoordinator(settings, client, clusterService, threadPool) - // Create Monitor using Monitor.kt, and then feed the request to client in some way this.threadPool = threadPool this.clusterService = clusterService // create a cluster metrics visualization index upon initialization of alerting plugin diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index 39a1060d5..eed222d95 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -6,7 +6,9 @@ package org.opensearch.alerting.settings import org.opensearch.alerting.AlertingPlugin +import org.opensearch.client.Client import org.opensearch.common.settings.Setting +import org.opensearch.common.settings.Settings import org.opensearch.common.unit.TimeValue import java.util.concurrent.TimeUnit @@ -15,9 +17,15 @@ import java.util.concurrent.TimeUnit */ private val log = org.apache.logging.log4j.LogManager.getLogger(AlertingSettings::class.java) -class AlertingSettings { +class AlertingSettings(val client: Client, val settings: Settings) { + init { + internalClient = client + internalSettings = settings + } companion object { + internal var internalClient: Client? = null + internal var internalSettings: Settings? = null const val MONITOR_MAX_INPUTS = 1 const val MONITOR_MAX_TRIGGERS = 10 @@ -173,24 +181,23 @@ class AlertingSettings { val METRICS_STORE_TIME = Setting.timeSetting( METRICS_STORE_TIME_DEFAULT.key, METRICS_STORE_TIME_DEFAULT, - StorageTimeValidator(), + TimeValueValidator(internalClient, internalSettings), Setting.Property.NodeScope, Setting.Property.Dynamic ) val METRICS_EXECUTION_FREQUENCY = Setting.timeSetting( METRICS_EXECUTION_FREQUENCY_DEFAULT.key, METRICS_EXECUTION_FREQUENCY_DEFAULT, - ExecutionFrequencyValidator(), + TimeValueValidator(internalClient, internalSettings), Setting.Property.NodeScope, Setting.Property.Dynamic ) - internal class ExecutionFrequencyValidator : Setting.Validator { + internal class TimeValueValidator(val client: Client?, val clusterSettings: Settings?) : Setting.Validator { override fun validate(value: TimeValue) {} override fun validate(value: TimeValue, settings: Map, Any>) { log.info("THIS IS SETTING $settings") log.info("THIS IS VALUE $value") - val storageTime = settings[METRICS_STORE_TIME] - validateExecutionFrequency(value, storageTime as TimeValue) + log.info("THIS IS CLUSTERSETTINGS $clusterSettings") } override fun settings(): MutableIterator> { @@ -199,37 +206,21 @@ class AlertingSettings { ) return settings.iterator() } - } - - internal class StorageTimeValidator : Setting.Validator { - override fun validate(value: TimeValue) {} - - override fun validate(value: TimeValue, settings: Map, Any>) { - val executionFrequency = settings[METRICS_EXECUTION_FREQUENCY] as TimeValue - validateExecutionFrequency(executionFrequency, value) - } - - override fun settings(): MutableIterator> { - val settings = mutableListOf>( - METRICS_STORE_TIME - ) - return settings.iterator() - } - } - fun validateExecutionFrequency(executionFrequency: TimeValue, storageTime: TimeValue) { - log.info("THIS IS VALIDATEEXECUTIONFREQUENCY PARAMS $executionFrequency, $storageTime") - log.info("THIS IS MINIMUM_TIME_VALUE $MINIMUM_TIME_VALUE") - - if (executionFrequency < MINIMUM_TIME_VALUE || storageTime < MINIMUM_TIME_VALUE) { - throw IllegalArgumentException( - "Execution frequency or storage time cannot be less than $MINIMUM_TIME_VALUE." - ) - } - if (executionFrequency < storageTime) return - if (executionFrequency > storageTime) { - throw IllegalArgumentException( - "Execution frequency cannot be greater than the storage time." - ) + private fun validateExecutionFrequency(executionFrequency: TimeValue, storageTime: TimeValue) { + log.info("THIS IS VALIDATEEXECUTIONFREQUENCY PARAMS $executionFrequency, $storageTime") + log.info("THIS IS MINIMUM_TIME_VALUE $MINIMUM_TIME_VALUE") + + if (executionFrequency < MINIMUM_TIME_VALUE || storageTime < MINIMUM_TIME_VALUE) { + throw IllegalArgumentException( + "Execution frequency or storage time cannot be less than $MINIMUM_TIME_VALUE." + ) + } + if (executionFrequency < storageTime) return + if (executionFrequency > storageTime) { + throw IllegalArgumentException( + "Execution frequency cannot be greater than the storage time." + ) + } } } } From 756ed15f48abe5b50670419737d985a80ca5148a Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 2 Aug 2022 17:09:16 -0700 Subject: [PATCH 187/215] test Signed-off-by: Richard --- .../org/opensearch/alerting/AlertingPlugin.kt | 2 - .../alerting/settings/AlertingSettings.kt | 79 +++-------------- .../util/ClusterMetricsCoordinator.kt | 84 +++++++------------ 3 files changed, 41 insertions(+), 124 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index f109d9488..c003f058a 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -145,7 +145,6 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R lateinit var clusterService: ClusterService lateinit var destinationMigrationCoordinator: DestinationMigrationCoordinator lateinit var clusterMetricsCoordinator: ClusterMetricsCoordinator - lateinit var alertingSettings: AlertingSettings override fun getRestHandlers( settings: Settings, @@ -242,7 +241,6 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R docLevelMonitorQueries = DocLevelMonitorQueries(client, clusterService) scheduler = JobScheduler(threadPool, runner) sweeper = JobSweeper(environment.settings(), client, clusterService, threadPool, xContentRegistry, scheduler, ALERTING_JOB_TYPES) - alertingSettings = AlertingSettings(client, settings) destinationMigrationCoordinator = DestinationMigrationCoordinator(client, clusterService, threadPool, scheduledJobIndices) clusterMetricsCoordinator = ClusterMetricsCoordinator(settings, client, clusterService, threadPool) this.threadPool = threadPool diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index eed222d95..9cab638e8 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -6,41 +6,20 @@ package org.opensearch.alerting.settings import org.opensearch.alerting.AlertingPlugin -import org.opensearch.client.Client import org.opensearch.common.settings.Setting -import org.opensearch.common.settings.Settings import org.opensearch.common.unit.TimeValue import java.util.concurrent.TimeUnit /** * settings specific to [AlertingPlugin]. These settings include things like history index max age, request timeout, etc... */ -private val log = org.apache.logging.log4j.LogManager.getLogger(AlertingSettings::class.java) +class AlertingSettings { -class AlertingSettings(val client: Client, val settings: Settings) { - - init { - internalClient = client - internalSettings = settings - } companion object { - internal var internalClient: Client? = null - internal var internalSettings: Settings? = null const val MONITOR_MAX_INPUTS = 1 const val MONITOR_MAX_TRIGGERS = 10 const val DEFAULT_MAX_ACTIONABLE_ALERT_COUNT = 50L - val METRICS_EXECUTION_FREQUENCY_DEFAULT = Setting.positiveTimeSetting( - "plugins.alerting.cluster_metrics.execution_frequency", - TimeValue(15, TimeUnit.MINUTES), - Setting.Property.NodeScope, Setting.Property.Dynamic - ) - val METRICS_STORE_TIME_DEFAULT = Setting.positiveTimeSetting( - "plugins.alerting.cluster_metrics.metrics_history_max_age", - TimeValue(7, TimeUnit.DAYS), - Setting.Property.NodeScope, Setting.Property.Dynamic - ) - val MINIMUM_TIME_VALUE = TimeValue(1, TimeUnit.SECONDS) val ALERTING_MAX_MONITORS = Setting.intSetting( "plugins.alerting.monitor.max_monitors", @@ -178,50 +157,16 @@ class AlertingSettings(val client: Client, val settings: Settings) { Setting.Property.NodeScope, Setting.Property.Dynamic ) - val METRICS_STORE_TIME = Setting.timeSetting( - METRICS_STORE_TIME_DEFAULT.key, - METRICS_STORE_TIME_DEFAULT, - TimeValueValidator(internalClient, internalSettings), - Setting.Property.NodeScope, Setting.Property.Dynamic - ) - - val METRICS_EXECUTION_FREQUENCY = Setting.timeSetting( - METRICS_EXECUTION_FREQUENCY_DEFAULT.key, - METRICS_EXECUTION_FREQUENCY_DEFAULT, - TimeValueValidator(internalClient, internalSettings), - Setting.Property.NodeScope, Setting.Property.Dynamic - ) - internal class TimeValueValidator(val client: Client?, val clusterSettings: Settings?) : Setting.Validator { - override fun validate(value: TimeValue) {} - - override fun validate(value: TimeValue, settings: Map, Any>) { - log.info("THIS IS SETTING $settings") - log.info("THIS IS VALUE $value") - log.info("THIS IS CLUSTERSETTINGS $clusterSettings") - } - - override fun settings(): MutableIterator> { - val settings = mutableListOf>( - METRICS_EXECUTION_FREQUENCY - ) - return settings.iterator() - } - private fun validateExecutionFrequency(executionFrequency: TimeValue, storageTime: TimeValue) { - log.info("THIS IS VALIDATEEXECUTIONFREQUENCY PARAMS $executionFrequency, $storageTime") - log.info("THIS IS MINIMUM_TIME_VALUE $MINIMUM_TIME_VALUE") - - if (executionFrequency < MINIMUM_TIME_VALUE || storageTime < MINIMUM_TIME_VALUE) { - throw IllegalArgumentException( - "Execution frequency or storage time cannot be less than $MINIMUM_TIME_VALUE." - ) - } - if (executionFrequency < storageTime) return - if (executionFrequency > storageTime) { - throw IllegalArgumentException( - "Execution frequency cannot be greater than the storage time." - ) - } - } - } + val METRICS_STORE_TIME = Setting.positiveTimeSetting( + "plugins.alerting.cluster_metrics.metrics_history_max_age", + TimeValue(7, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val METRICS_EXECUTION_FREQUENCY = Setting.positiveTimeSetting( + "plugins.alerting.cluster_metrics.execution_frequency", + TimeValue(15, TimeUnit.MINUTES), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 9316079a3..00e4c5006 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -76,63 +76,37 @@ class ClusterMetricsCoordinator( } override fun clusterChanged(event: ClusterChangedEvent?) { -// if (metricsStoreTime > metricsExecutionFrequency) { -// val scheduledJobCollection = Runnable { -// launch { -// createDocs(client as NodeClient, clusterService) -// deleteDocs(client) -// } -// } -// if (isCollectionUpdated || isDeletionUpdated) { -// dataPointCollectionDeletionJob?.cancel() -// log.info("cancelled data collection and deletion jobs ") -// isRunningFlag = false -// log.info("detected changes to settings, resetting running, deletion and collection flags to false") -// isDeletionUpdated = false -// isCollectionUpdated = false -// } -// // if cluster changed event occurs (if settings are changed) -// // cancel current scheduled job and change runningFlag to false. -// // maybe have individual runningFlags for collection and deletion? -// if (event!!.localNodeMaster() && !isRunningFlag) { -// log.info("cluster changed metricsExecutionFrequency = $metricsExecutionFrequency") -// dataPointCollectionDeletionJob = threadPool.scheduleWithFixedDelay( -// scheduledJobCollection, -// metricsExecutionFrequency, -// ThreadPool.Names.SYSTEM_WRITE -// ) -// isRunningFlag = true -// } -// } else { -// throw java.lang.IllegalArgumentException( -// "Execution frequency ($metricsExecutionFrequency) can not be greater than the storage time ($metricsStoreTime). " -// ) -// } - val scheduledJobCollection = Runnable { - launch { - createDocs(client as NodeClient, clusterService) - deleteDocs(client) + if (metricsStoreTime > metricsExecutionFrequency) { + val scheduledJobCollection = Runnable { + launch { + createDocs(client as NodeClient, clusterService) + deleteDocs(client) + } } - } - if (isCollectionUpdated || isDeletionUpdated) { - dataPointCollectionDeletionJob?.cancel() - log.info("cancelled data collection and deletion jobs ") - isRunningFlag = false - log.info("detected changes to settings, resetting running, deletion and collection flags to false") - isDeletionUpdated = false - isCollectionUpdated = false - } - // if cluster changed event occurs (if settings are changed) - // cancel current scheduled job and change runningFlag to false. - // maybe have individual runningFlags for collection and deletion? - if (event!!.localNodeMaster() && !isRunningFlag) { - log.info("cluster changed metricsExecutionFrequency = $metricsExecutionFrequency") - dataPointCollectionDeletionJob = threadPool.scheduleWithFixedDelay( - scheduledJobCollection, - metricsExecutionFrequency, - ThreadPool.Names.SYSTEM_WRITE + if (isCollectionUpdated || isDeletionUpdated) { + dataPointCollectionDeletionJob?.cancel() + log.info("cancelled data collection and deletion jobs ") + isRunningFlag = false + log.info("detected changes to settings, resetting running, deletion and collection flags to false") + isDeletionUpdated = false + isCollectionUpdated = false + } + // if cluster changed event occurs (if settings are changed) + // cancel current scheduled job and change runningFlag to false. + // maybe have individual runningFlags for collection and deletion? + if (event!!.localNodeMaster() && !isRunningFlag) { + log.info("cluster changed metricsExecutionFrequency = $metricsExecutionFrequency") + dataPointCollectionDeletionJob = threadPool.scheduleWithFixedDelay( + scheduledJobCollection, + metricsExecutionFrequency, + ThreadPool.Names.SYSTEM_WRITE + ) + isRunningFlag = true + } + } else { + throw java.lang.IllegalArgumentException( + "Execution frequency ($metricsExecutionFrequency) can not be greater than the storage time ($metricsStoreTime). " ) - isRunningFlag = true } } From 599769a333f5af67d9e2b4ff7b0b49ddba43d88a Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 3 Aug 2022 15:02:10 -0700 Subject: [PATCH 188/215] Project PR Signed-off-by: Richard --- .../util/ClusterMetricsCoordinator.kt | 123 ++++++++---------- .../ClusterMetricsCoordinatorIT.kt | 4 - 2 files changed, 53 insertions(+), 74 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 00e4c5006..8cafca64b 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -91,9 +91,6 @@ class ClusterMetricsCoordinator( isDeletionUpdated = false isCollectionUpdated = false } - // if cluster changed event occurs (if settings are changed) - // cancel current scheduled job and change runningFlag to false. - // maybe have individual runningFlags for collection and deletion? if (event!!.localNodeMaster() && !isRunningFlag) { log.info("cluster changed metricsExecutionFrequency = $metricsExecutionFrequency") dataPointCollectionDeletionJob = threadPool.scheduleWithFixedDelay( @@ -111,38 +108,25 @@ class ClusterMetricsCoordinator( } suspend fun createDocs(client: NodeClient, clusterService: ClusterService) { - /* - get time from Instant.now(), use this variable for all 4 documents that I'm going to create - get clusterHealth API (status + unassigned shards) - check whether clusterMetricsVisualizationIndex exists - if not, create - create document data point: - parse data from the clusterHealth API to get values - call index API to index the document - */ - val current_time = Instant.now().toString() -// val curTime = current_time. - log.info("richfu THIS IS THE TIME $current_time") - // cluster health for unassigned shards - val cluster_health = client.admin().cluster().health(ClusterHealthRequest()).get().toMap() - // cluster stats for cluster status (health), CPU usage, JVM pressure - var nodeStats = client.admin().cluster().nodesStats(NodesStatsRequest().addMetrics("process", "jvm")).get().toMap() + val currentTime = Instant.now().toString() + val clusterHealth = client.admin().cluster().health(ClusterHealthRequest()).get().toMap() + val nodeStats = client.admin().cluster().nodesStats(NodesStatsRequest().addMetrics("process", "jvm")).get().toMap() ClusterMetricsVisualizationIndex.initFunc(client, clusterService) - val unassignedShards = cluster_health["unassigned_shards"].toString() + val unassignedShards = clusterHealth["unassigned_shards"].toString() log.info("this is unassigned shards $unassignedShards") - val clusterStatus = cluster_health["status"].toString() + val clusterStatus = clusterHealth["status"].toString() log.info("this is cluster status $clusterStatus") - val numPending = cluster_health["number_of_pending_tasks"].toString() + val numPending = clusterHealth["number_of_pending_tasks"].toString() log.info("this is number of pending tasks $numPending") - val activeShards = cluster_health["active_shards"].toString() + val activeShards = clusterHealth["active_shards"].toString() log.info("this is number of active shards $activeShards") - val relocatingShards = cluster_health["relocating_shards"].toString() + val relocatingShards = clusterHealth["relocating_shards"].toString() log.info("This is number of relocatingShards $relocatingShards") - val numNodes = cluster_health["number_of_nodes"].toString() + val numNodes = clusterHealth["number_of_nodes"].toString() log.info("This is number of nodes (total) $numNodes") - val numDataNodes = cluster_health["number_of_data_nodes"].toString() + val numDataNodes = clusterHealth["number_of_data_nodes"].toString() log.info("This is number of data nodes $numDataNodes") val nodesMap = nodeStats["nodes"] as Map @@ -195,85 +179,85 @@ class ClusterMetricsCoordinator( val avgCPU = String.format("%.2f", avgCPUcalc) val avgJVM = String.format("%.2f", avgJVMcalc) - val clusterStatus_data = ClusterMetricsDataPoint( + val clusterstatusData = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, - current_time, + currentTime, clusterStatus ) - val unassigned_shards_data = ClusterMetricsDataPoint( + val unassignedShardsData = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, - current_time, + currentTime, unassignedShards ) - val cpu_usage_data = ClusterMetricsDataPoint( + val cpuUsageData = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.CPU_USAGE, - current_time, + currentTime, avgCPU, minimumCPU, maximumCPU ) - val jvm_data = ClusterMetricsDataPoint( + val jvmDataPoint = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.JVM_PRESSURE, - current_time, + currentTime, avgJVM, minimumJVM, maximumJVM ) val pendingTasksData = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.NUMBER_OF_PENDING_TASKS, - current_time, + currentTime, numPending ) val activeShardsData = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.ACTIVE_SHARDS, - current_time, + currentTime, activeShards ) val relocatingShardsData = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.RELOCATING_SHARDS, - current_time, + currentTime, relocatingShards ) val nodesData = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.NUMBER_OF_NODES, - current_time, + currentTime, numNodes ) val dataNodesData = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.NUMBER_OF_DATA_NODES, - current_time, + currentTime, numDataNodes ) - val indexRequest_status = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(clusterStatus_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexRequest_shards = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(unassigned_shards_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexRequest_cpu = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(cpu_usage_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexRequest_jvm = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(jvm_data.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexRequest_pending = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + val indexrequestStatus = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(clusterstatusData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + val indexrequestShards = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(unassignedShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + val indexrequestCpu = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(cpuUsageData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + val indexrequestJvm = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source(jvmDataPoint.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) + val indexrequestPending = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) .source(pendingTasksData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexRequest_active = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + val indexrequestActive = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) .source(activeShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexRequest_relocating = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + val indexrequestRelocating = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) .source(relocatingShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexRequest_nodes = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + val indexrequestNodes = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) .source(nodesData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexRequest_dataNodes = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + val indexrequestDatanodes = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) .source(dataNodesData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) try { - val indexResponse: IndexResponse = client.suspendUntil { client.index(indexRequest_status, it) } - val indexResponse2: IndexResponse = client.suspendUntil { client.index(indexRequest_shards, it) } - val indexResponse3: IndexResponse = client.suspendUntil { client.index(indexRequest_cpu, it) } - val indexResponse4: IndexResponse = client.suspendUntil { client.index(indexRequest_jvm, it) } - val indexResponse5: IndexResponse = client.suspendUntil { client.index(indexRequest_pending, it) } - val indexResponse6: IndexResponse = client.suspendUntil { client.index(indexRequest_active, it) } - val indexResponse7: IndexResponse = client.suspendUntil { client.index(indexRequest_relocating, it) } - val indexResponse8: IndexResponse = client.suspendUntil { client.index(indexRequest_nodes, it) } - val indexResponse9: IndexResponse = client.suspendUntil { client.index(indexRequest_dataNodes, it) } + val indexResponse: IndexResponse = client.suspendUntil { client.index(indexrequestStatus, it) } + val indexResponse2: IndexResponse = client.suspendUntil { client.index(indexrequestShards, it) } + val indexResponse3: IndexResponse = client.suspendUntil { client.index(indexrequestCpu, it) } + val indexResponse4: IndexResponse = client.suspendUntil { client.index(indexrequestJvm, it) } + val indexResponse5: IndexResponse = client.suspendUntil { client.index(indexrequestPending, it) } + val indexResponse6: IndexResponse = client.suspendUntil { client.index(indexrequestActive, it) } + val indexResponse7: IndexResponse = client.suspendUntil { client.index(indexrequestRelocating, it) } + val indexResponse8: IndexResponse = client.suspendUntil { client.index(indexrequestNodes, it) } + val indexResponse9: IndexResponse = client.suspendUntil { client.index(indexrequestDatanodes, it) } val failureReasons = checkShardsFailure(indexResponse) val failureReasons2 = checkShardsFailure(indexResponse2) val failureReasons3 = checkShardsFailure(indexResponse3) @@ -294,19 +278,19 @@ class ClusterMetricsCoordinator( failureReasons8 != null || failureReasons9 != null ) { - log.info("richfu failed because $failureReasons") - log.info("richfu failed because $failureReasons2") - log.info("richfu failed because $failureReasons3") - log.info("richfu failed because $failureReasons4") - log.info("richfu failed because $failureReasons5") - log.info("richfu failed because $failureReasons6") - log.info("richfu failed because $failureReasons7") + log.info("failed because $failureReasons") + log.info("failed because $failureReasons2") + log.info("failed because $failureReasons3") + log.info("failed because $failureReasons4") + log.info("failed because $failureReasons5") + log.info("failed because $failureReasons6") + log.info("failed because $failureReasons7") log.info("failed because $failureReasons8") log.info("failed because $failureReasons9") return } } catch (t: Exception) { - log.info("richfu CLUSTER METRICS NOT WORK $t") + log.info("Unable to get index response, $t") } } fun checkShardsFailure(response: IndexResponse): String? { @@ -324,7 +308,6 @@ class ClusterMetricsCoordinator( fun deleteDocs(client: NodeClient) { val documentAge = metricsStoreTime.toString() val unitTime = documentAge.last() - log.info("documentAge returns $documentAge, unitTime returns $unitTime") DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index b95fc6253..001b35be1 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -22,10 +22,6 @@ import java.time.temporal.ChronoUnit import java.util.* import kotlin.collections.ArrayList class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { - /* - 4. Adjust the execution frequency setting to 1 minute, and make sure that the timestamps between the datapoints are 1 minute apart. - Check 2 minute apart after - */ @Before fun setup() { // When setting up the tests, change the execution frequency and history max age settings to 1 minute and 10 minutes from From 1a6db368fe24c6b5d1b54a8a2831c9840cc8c1f6 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 3 Aug 2022 15:10:28 -0700 Subject: [PATCH 189/215] Project PR Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 8cafca64b..f27cc5745 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -85,7 +85,7 @@ class ClusterMetricsCoordinator( } if (isCollectionUpdated || isDeletionUpdated) { dataPointCollectionDeletionJob?.cancel() - log.info("cancelled data collection and deletion jobs ") + log.info("Cancelled data collection and deletion jobs") isRunningFlag = false log.info("detected changes to settings, resetting running, deletion and collection flags to false") isDeletionUpdated = false From f7f5112bfe20097d9d22350086b7cacdecfd0a8c Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 3 Aug 2022 15:20:59 -0700 Subject: [PATCH 190/215] Project PR Signed-off-by: Richard --- .../util/ClusterMetricsCoordinator.kt | 19 ------------------- .../DestinationMigrationCoordinator.kt | 13 ------------- 2 files changed, 32 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index f27cc5745..7262902ef 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -115,55 +115,36 @@ class ClusterMetricsCoordinator( ClusterMetricsVisualizationIndex.initFunc(client, clusterService) val unassignedShards = clusterHealth["unassigned_shards"].toString() - log.info("this is unassigned shards $unassignedShards") val clusterStatus = clusterHealth["status"].toString() - log.info("this is cluster status $clusterStatus") val numPending = clusterHealth["number_of_pending_tasks"].toString() - log.info("this is number of pending tasks $numPending") val activeShards = clusterHealth["active_shards"].toString() - log.info("this is number of active shards $activeShards") val relocatingShards = clusterHealth["relocating_shards"].toString() - log.info("This is number of relocatingShards $relocatingShards") val numNodes = clusterHealth["number_of_nodes"].toString() - log.info("This is number of nodes (total) $numNodes") val numDataNodes = clusterHealth["number_of_data_nodes"].toString() - log.info("This is number of data nodes $numDataNodes") val nodesMap = nodeStats["nodes"] as Map val keys = nodesMap.keys - log.info("this is nodesMap keys $keys") val jvmData = arrayListOf() val cpuData = arrayListOf() for (key in keys) { val keyData = nodesMap[key] as Map - log.info("this is keyData $keyData") val processMap = keyData["process"] as Map - log.info("This is osMap $processMap") val cpuMap = processMap["cpu"] as Map - log.info("This is cpuMap $cpuMap") val percent = cpuMap["percent"] cpuData.add(percent as Int) val jvmMap = keyData["jvm"] as Map - log.info("this is jvmMap $jvmMap") val memMap = jvmMap["mem"] as Map - log.info("this is memMap $memMap") val pressure = memMap["heap_used_percent"] jvmData.add(pressure as Int) } - log.info("this is CPU data $cpuData") - log.info("this is JVM data $jvmData") val minimumCPU = Collections.min(cpuData).toString() val maximumCPU = Collections.max(cpuData).toString() - log.info("this is min CPU $minimumCPU") - log.info("this is max CPU $maximumCPU") val minimumJVM = Collections.min(jvmData).toString() val maximumJVM = Collections.max(jvmData).toString() - log.info("this is max JVM $maximumJVM") - log.info("this is min JVM $minimumJVM") var avgCPUcalc = 0.0 var avgJVMcalc = 0.0 diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt index 52c6b74a3..eb22e6275 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/destinationmigration/DestinationMigrationCoordinator.kt @@ -56,26 +56,18 @@ class DestinationMigrationCoordinator( !runningLock && (scheduledMigration == null || scheduledMigration!!.isCancelled) ) { - logger.info("richfu destination migration") try { - logger.info("richfu try before") runningLock = true initMigrateDestinations() - logger.info("richfu try after") } finally { - logger.info("richfu finally before") runningLock = false - logger.info("richfu finally after") } } else if (!event.localNodeMaster()) { - logger.info("richfu no destination migration") scheduledMigration?.cancel() } } private fun initMigrateDestinations() { - logger.info("start of initMigrateDestination") - logger.info("includes $clusterService") if (!scheduledJobIndices.scheduledJobIndexExists()) { logger.debug("Alerting config index is not initialized") scheduledMigration?.cancel() @@ -100,8 +92,6 @@ class DestinationMigrationCoordinator( logger.info("Cancel background destination migration process.") scheduledMigration?.cancel() } - logger.info("richfu calling class") - logger.info("richfu after called class") logger.info("Performing migration of destination data.") DestinationMigrationUtilService.migrateDestinations(client as NodeClient) } catch (e: Exception) { @@ -109,9 +99,6 @@ class DestinationMigrationCoordinator( } } } - - logger.info("richfu before scheduledMigration") scheduledMigration = threadPool.scheduleWithFixedDelay(scheduledJob, TimeValue.timeValueMinutes(1), ThreadPool.Names.MANAGEMENT) - logger.info("richfu after scheduledMigration call") } } From cbdd767a40db8e86f8085fe1cde4fdf27c2e35cb Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 3 Aug 2022 15:50:00 -0700 Subject: [PATCH 191/215] Project PR, changed deleting logic tiny bit Signed-off-by: Richard --- .../alerting/util/ClusterMetricsCoordinator.kt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 7262902ef..64e76af6c 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -292,7 +292,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now-$documentAge/$unitTime")) + .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now-$documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -306,7 +306,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("cpu_usage.timestamp").lte("now-$documentAge/$unitTime")) + .filter(QueryBuilders.rangeQuery("cpu_usage.timestamp").lte("now-$documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -320,7 +320,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("jvm_pressure.timestamp").lte("now-$documentAge/$unitTime")) + .filter(QueryBuilders.rangeQuery("jvm_pressure.timestamp").lte("now-$documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -334,7 +334,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("unassigned_shards.timestamp").lte("now-$documentAge/$unitTime")) + .filter(QueryBuilders.rangeQuery("unassigned_shards.timestamp").lte("now-$documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -348,7 +348,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("number_of_pending_tasks.timestamp").lte("now-$documentAge/$unitTime")) + .filter(QueryBuilders.rangeQuery("number_of_pending_tasks.timestamp").lte("now-$documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -362,7 +362,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("active_shards.timestamp").lte("now-$documentAge/$unitTime")) + .filter(QueryBuilders.rangeQuery("active_shards.timestamp").lte("now-$documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -376,7 +376,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("relocating_shards.timestamp").lte("now-$documentAge/$unitTime")) + .filter(QueryBuilders.rangeQuery("relocating_shards.timestamp").lte("now-$documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -390,7 +390,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("number_of_nodes.timestamp").lte("now-$documentAge/$unitTime")) + .filter(QueryBuilders.rangeQuery("number_of_nodes.timestamp").lte("now-$documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { @@ -404,7 +404,7 @@ class ClusterMetricsCoordinator( DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("number_of_data_nodes.timestamp").lte("now-$documentAge/$unitTime")) + .filter(QueryBuilders.rangeQuery("number_of_data_nodes.timestamp").lte("now-$documentAge")) .execute( object : ActionListener { override fun onResponse(response: BulkByScrollResponse) { From a9bfbb9e0653befb85ea6dfc4a4d15c420a45722 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 3 Aug 2022 16:48:13 -0700 Subject: [PATCH 192/215] Project PR, added logs for presentation Signed-off-by: Richard --- .../alerting/util/ClusterMetricsCoordinator.kt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 64e76af6c..aa40b3955 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -109,18 +109,26 @@ class ClusterMetricsCoordinator( suspend fun createDocs(client: NodeClient, clusterService: ClusterService) { val currentTime = Instant.now().toString() + log.info("This is the current time: $currentTime") val clusterHealth = client.admin().cluster().health(ClusterHealthRequest()).get().toMap() val nodeStats = client.admin().cluster().nodesStats(NodesStatsRequest().addMetrics("process", "jvm")).get().toMap() ClusterMetricsVisualizationIndex.initFunc(client, clusterService) val unassignedShards = clusterHealth["unassigned_shards"].toString() + log.info("This is unassigned shards value: $unassignedShards") val clusterStatus = clusterHealth["status"].toString() + log.info("This is cluster status value: $clusterStatus") val numPending = clusterHealth["number_of_pending_tasks"].toString() + log.info("This is the number of pending tasks: $numPending") val activeShards = clusterHealth["active_shards"].toString() + log.info("This is active shards $activeShards") val relocatingShards = clusterHealth["relocating_shards"].toString() + log.info("This is relocating shards $relocatingShards") val numNodes = clusterHealth["number_of_nodes"].toString() + log.info("This is number of nodes $numNodes") val numDataNodes = clusterHealth["number_of_data_nodes"].toString() + log.info("this is number of data nodes $numDataNodes") val nodesMap = nodeStats["nodes"] as Map val keys = nodesMap.keys @@ -142,9 +150,13 @@ class ClusterMetricsCoordinator( val minimumCPU = Collections.min(cpuData).toString() val maximumCPU = Collections.max(cpuData).toString() + log.info("This is minimum CPU Usage, $minimumCPU") + log.info("This is maximum CPU usage, $maximumCPU") val minimumJVM = Collections.min(jvmData).toString() val maximumJVM = Collections.max(jvmData).toString() + log.info("This is minimum JVM, $minimumJVM") + log.info("This is maximum JVM, $maximumJVM") var avgCPUcalc = 0.0 var avgJVMcalc = 0.0 @@ -159,6 +171,8 @@ class ClusterMetricsCoordinator( val avgCPU = String.format("%.2f", avgCPUcalc) val avgJVM = String.format("%.2f", avgJVMcalc) + log.info("This is average CPU, $avgCPU") + log.info("This is average JVM, $avgJVM") val clusterstatusData = ClusterMetricsDataPoint( ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, From 40eadc83aaa3fb859de78cb980f76eaf0d1073dc Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 5 Aug 2022 16:31:39 -0700 Subject: [PATCH 193/215] Project PR, added cleanup for IT Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 001b35be1..3cac2e4cd 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -7,6 +7,7 @@ package org.opensearch.alerting.util.clustermetricscoordinatortest import org.apache.http.entity.ContentType import org.apache.http.entity.StringEntity +import org.junit.After import org.junit.Before import org.opensearch.alerting.AlertingRestTestCase import org.opensearch.alerting.makeRequest @@ -112,6 +113,16 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { assertEquals(diff, 2) } + @After + // Reset the settings back to default, delete the created index. + fun cleanup() { + // delete the documents from index first, and then reset execution frequency + client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "now") + client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "15m") + // reset storage time and delete the index + client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "7d") + client().makeRequest("DELETE", ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + } private fun generateData() { client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1s") client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "10m") From 6eebc777179d9156c6d587d1223633fa259af25b Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 5 Aug 2022 16:31:39 -0700 Subject: [PATCH 194/215] Project PR, added cleanup for IT Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 3cac2e4cd..8d2abb367 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -116,10 +116,8 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { @After // Reset the settings back to default, delete the created index. fun cleanup() { - // delete the documents from index first, and then reset execution frequency - client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "now") + // reset settings client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "15m") - // reset storage time and delete the index client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "7d") client().makeRequest("DELETE", ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) } From d1b4e5a70dd7fb15d619009ce3f17b0e403c03db Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 9 Aug 2022 14:26:07 -0700 Subject: [PATCH 195/215] Project PR, added cleanup failing test Signed-off-by: Richard --- .../org/opensearch/alerting/util/ClusterMetricsCoordinator.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index aa40b3955..d661686fb 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -169,8 +169,8 @@ class ClusterMetricsCoordinator( avgCPUcalc /= cpuData.size avgJVMcalc /= jvmData.size - val avgCPU = String.format("%.2f", avgCPUcalc) - val avgJVM = String.format("%.2f", avgJVMcalc) + val avgCPU = avgCPUcalc.toString() + val avgJVM = avgJVMcalc.toString() log.info("This is average CPU, $avgCPU") log.info("This is average JVM, $avgJVM") From bdc8633011eb5fcb47ed7992eeb2dbfd95721cfa Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 9 Aug 2022 14:55:57 -0700 Subject: [PATCH 196/215] Project PR, added cleanup failing test Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 8d2abb367..40c18ace7 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -58,7 +58,6 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { ClusterMetricsDataPoint.MetricType.values().forEach { mapCheck[it.metricName] = mutableSetOf() } for (doc in docs) { val source = doc["_source"] as Map> - logger.info("this is source data $source") val metricType = source.keys.first() try { ClusterMetricsDataPoint.MetricType.valueOf(metricType.uppercase(Locale.getDefault())) @@ -107,6 +106,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { val metricType = source.keys.first() times.add(source[metricType]?.get("timestamp").toString()) } + logger.info("this is times $times") val time1 = Instant.parse(times.elementAt(times.size - 1)) val time2 = Instant.parse(times.elementAt(times.size - 2)) val diff = time2.until(time1, ChronoUnit.MINUTES) From e5f2603fbe9a2511d200f6384ef5578a1dc3b61a Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 10 Aug 2022 10:07:50 -0700 Subject: [PATCH 197/215] Project PR, cleaned up deleteDocs() method. Signed-off-by: Richard --- .../util/ClusterMetricsCoordinator.kt | 270 +++++++++--------- 1 file changed, 142 insertions(+), 128 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index d661686fb..32ef93d96 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -107,7 +107,7 @@ class ClusterMetricsCoordinator( } } - suspend fun createDocs(client: NodeClient, clusterService: ClusterService) { + private suspend fun createDocs(client: NodeClient, clusterService: ClusterService) { val currentTime = Instant.now().toString() log.info("This is the current time: $currentTime") val clusterHealth = client.admin().cluster().health(ClusterHealthRequest()).get().toMap() @@ -288,7 +288,7 @@ class ClusterMetricsCoordinator( log.info("Unable to get index response, $t") } } - fun checkShardsFailure(response: IndexResponse): String? { + private fun checkShardsFailure(response: IndexResponse): String? { val failureReasons = StringBuilder() if (response.shardInfo.failed > 0) { response.shardInfo.failures.forEach { @@ -300,134 +300,148 @@ class ClusterMetricsCoordinator( return null } - fun deleteDocs(client: NodeClient) { + private fun deleteDocs(client: NodeClient) { val documentAge = metricsStoreTime.toString() - val unitTime = documentAge.last() - DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) - .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now-$documentAge")) - .execute( - object : ActionListener { - override fun onResponse(response: BulkByScrollResponse) { + ClusterMetricsDataPoint.MetricType.values().forEach { + DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) + .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .filter(QueryBuilders.rangeQuery(it.metricName + ".timestamp").lte("now-$documentAge")) + .execute( + object : ActionListener { + override fun onResponse(response: BulkByScrollResponse) { + } + + override fun onFailure(t: Exception) { + } } - - override fun onFailure(t: Exception) { - } - } - ) - log.info("deleted clusterStatus data from $documentAge/$unitTime ago, now-$documentAge") - - DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) - .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("cpu_usage.timestamp").lte("now-$documentAge")) - .execute( - object : ActionListener { - override fun onResponse(response: BulkByScrollResponse) { - } - - override fun onFailure(t: Exception) { - } - } - ) - log.info("deleted cpu_usage data from $documentAge/$unitTime ago") - - DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) - .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("jvm_pressure.timestamp").lte("now-$documentAge")) - .execute( - object : ActionListener { - override fun onResponse(response: BulkByScrollResponse) { - } - - override fun onFailure(t: Exception) { - } - } - ) - log.info("deleted jvm_pressure data from $documentAge/$unitTime ago") - - DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) - .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("unassigned_shards.timestamp").lte("now-$documentAge")) - .execute( - object : ActionListener { - override fun onResponse(response: BulkByScrollResponse) { - } - - override fun onFailure(t: Exception) { - } - } - ) - log.info("deleted unassigned_shards data from $documentAge/$unitTime ago") - - DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) - .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("number_of_pending_tasks.timestamp").lte("now-$documentAge")) - .execute( - object : ActionListener { - override fun onResponse(response: BulkByScrollResponse) { - } - - override fun onFailure(t: Exception) { - } - } - ) - log.info("deleted number of pending tasks from $documentAge/$unitTime ago") - - DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) - .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("active_shards.timestamp").lte("now-$documentAge")) - .execute( - object : ActionListener { - override fun onResponse(response: BulkByScrollResponse) { - } - - override fun onFailure(t: Exception) { - } - } - ) - log.info("deleted active shards from $documentAge/$unitTime ago") - - DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) - .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("relocating_shards.timestamp").lte("now-$documentAge")) - .execute( - object : ActionListener { - override fun onResponse(response: BulkByScrollResponse) { - } - - override fun onFailure(t: Exception) { - } - } - ) - log.info("deleted number of relocating shards from $documentAge/$unitTime ago") - - DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) - .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("number_of_nodes.timestamp").lte("now-$documentAge")) - .execute( - object : ActionListener { - override fun onResponse(response: BulkByScrollResponse) { - } - - override fun onFailure(t: Exception) { - } - } - ) - log.info("deleted number of nodes from $documentAge/$unitTime ago") - - DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) - .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .filter(QueryBuilders.rangeQuery("number_of_data_nodes.timestamp").lte("now-$documentAge")) - .execute( - object : ActionListener { - override fun onResponse(response: BulkByScrollResponse) { - } - - override fun onFailure(t: Exception) { - } - } - ) - log.info("deleted number of data nodes from $documentAge/$unitTime ago") + ) + log.info("deleted ${it.metricName} data from $documentAge ago") + } +// DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) +// .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now-$documentAge")) +// .execute( +// object : ActionListener { +// override fun onResponse(response: BulkByScrollResponse) { +// } +// +// override fun onFailure(t: Exception) { +// } +// } +// ) +// log.info("deleted clusterStatus data from $documentAge/$unitTime ago, now-$documentAge") +// +// DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) +// .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .filter(QueryBuilders.rangeQuery("cpu_usage.timestamp").lte("now-$documentAge")) +// .execute( +// object : ActionListener { +// override fun onResponse(response: BulkByScrollResponse) { +// } +// +// override fun onFailure(t: Exception) { +// } +// } +// ) +// log.info("deleted cpu_usage data from $documentAge/$unitTime ago") +// +// DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) +// .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .filter(QueryBuilders.rangeQuery("jvm_pressure.timestamp").lte("now-$documentAge")) +// .execute( +// object : ActionListener { +// override fun onResponse(response: BulkByScrollResponse) { +// } +// +// override fun onFailure(t: Exception) { +// } +// } +// ) +// log.info("deleted jvm_pressure data from $documentAge/$unitTime ago") +// +// DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) +// .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .filter(QueryBuilders.rangeQuery("unassigned_shards.timestamp").lte("now-$documentAge")) +// .execute( +// object : ActionListener { +// override fun onResponse(response: BulkByScrollResponse) { +// } +// +// override fun onFailure(t: Exception) { +// } +// } +// ) +// log.info("deleted unassigned_shards data from $documentAge/$unitTime ago") +// +// DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) +// .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .filter(QueryBuilders.rangeQuery("number_of_pending_tasks.timestamp").lte("now-$documentAge")) +// .execute( +// object : ActionListener { +// override fun onResponse(response: BulkByScrollResponse) { +// } +// +// override fun onFailure(t: Exception) { +// } +// } +// ) +// log.info("deleted number of pending tasks from $documentAge/$unitTime ago") +// +// DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) +// .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .filter(QueryBuilders.rangeQuery("active_shards.timestamp").lte("now-$documentAge")) +// .execute( +// object : ActionListener { +// override fun onResponse(response: BulkByScrollResponse) { +// } +// +// override fun onFailure(t: Exception) { +// } +// } +// ) +// log.info("deleted active shards from $documentAge/$unitTime ago") +// +// DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) +// .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .filter(QueryBuilders.rangeQuery("relocating_shards.timestamp").lte("now-$documentAge")) +// .execute( +// object : ActionListener { +// override fun onResponse(response: BulkByScrollResponse) { +// } +// +// override fun onFailure(t: Exception) { +// } +// } +// ) +// log.info("deleted number of relocating shards from $documentAge/$unitTime ago") +// +// DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) +// .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .filter(QueryBuilders.rangeQuery("number_of_nodes.timestamp").lte("now-$documentAge")) +// .execute( +// object : ActionListener { +// override fun onResponse(response: BulkByScrollResponse) { +// } +// +// override fun onFailure(t: Exception) { +// } +// } +// ) +// log.info("deleted number of nodes from $documentAge/$unitTime ago") +// +// DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) +// .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .filter(QueryBuilders.rangeQuery("number_of_data_nodes.timestamp").lte("now-$documentAge")) +// .execute( +// object : ActionListener { +// override fun onResponse(response: BulkByScrollResponse) { +// } +// +// override fun onFailure(t: Exception) { +// } +// } +// ) +// log.info("deleted number of data nodes from $documentAge/$unitTime ago") } } From b1451cef2bd734fc597b6880d3cba169aa2a21ce Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 10 Aug 2022 11:18:20 -0700 Subject: [PATCH 198/215] Project PR, cleaned up createDocs() method Signed-off-by: Richard --- .../util/ClusterMetricsCoordinator.kt | 411 ++++++++---------- 1 file changed, 178 insertions(+), 233 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 32ef93d96..f002bc8a3 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -174,119 +174,189 @@ class ClusterMetricsCoordinator( log.info("This is average CPU, $avgCPU") log.info("This is average JVM, $avgJVM") - val clusterstatusData = ClusterMetricsDataPoint( - ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, - currentTime, - clusterStatus - ) - val unassignedShardsData = ClusterMetricsDataPoint( - ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, - currentTime, - unassignedShards - ) - val cpuUsageData = ClusterMetricsDataPoint( - ClusterMetricsDataPoint.MetricType.CPU_USAGE, - currentTime, - avgCPU, - minimumCPU, - maximumCPU - ) - val jvmDataPoint = ClusterMetricsDataPoint( - ClusterMetricsDataPoint.MetricType.JVM_PRESSURE, - currentTime, - avgJVM, - minimumJVM, - maximumJVM - ) - val pendingTasksData = ClusterMetricsDataPoint( - ClusterMetricsDataPoint.MetricType.NUMBER_OF_PENDING_TASKS, - currentTime, - numPending - ) - val activeShardsData = ClusterMetricsDataPoint( - ClusterMetricsDataPoint.MetricType.ACTIVE_SHARDS, - currentTime, - activeShards - ) - val relocatingShardsData = ClusterMetricsDataPoint( - ClusterMetricsDataPoint.MetricType.RELOCATING_SHARDS, - currentTime, - relocatingShards - ) - val nodesData = ClusterMetricsDataPoint( - ClusterMetricsDataPoint.MetricType.NUMBER_OF_NODES, - currentTime, - numNodes - ) - val dataNodesData = ClusterMetricsDataPoint( - ClusterMetricsDataPoint.MetricType.NUMBER_OF_DATA_NODES, - currentTime, - numDataNodes + val dataPoints = arrayListOf( + ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, + currentTime, + clusterStatus + ), + ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, + currentTime, + unassignedShards + ), + ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.CPU_USAGE, + currentTime, + avgCPU, + minimumCPU, + maximumCPU + ), + ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.JVM_PRESSURE, + currentTime, + avgJVM, + minimumJVM, + maximumJVM + ), + ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.NUMBER_OF_PENDING_TASKS, + currentTime, + numPending + ), + ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.ACTIVE_SHARDS, + currentTime, + activeShards + ), + ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.RELOCATING_SHARDS, + currentTime, + relocatingShards + ), + ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.NUMBER_OF_NODES, + currentTime, + numNodes + ), + ClusterMetricsDataPoint( + ClusterMetricsDataPoint.MetricType.NUMBER_OF_DATA_NODES, + currentTime, + numDataNodes + ) ) - val indexrequestStatus = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(clusterstatusData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexrequestShards = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(unassignedShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexrequestCpu = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(cpuUsageData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexrequestJvm = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(jvmDataPoint.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexrequestPending = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(pendingTasksData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexrequestActive = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(activeShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexrequestRelocating = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(relocatingShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexrequestNodes = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(nodesData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) - val indexrequestDatanodes = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) - .source(dataNodesData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) +// val clusterstatusData = ClusterMetricsDataPoint( +// ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, +// currentTime, +// clusterStatus +// ) +// val unassignedShardsData = ClusterMetricsDataPoint( +// ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, +// currentTime, +// unassignedShards +// ) +// val cpuUsageData = ClusterMetricsDataPoint( +// ClusterMetricsDataPoint.MetricType.CPU_USAGE, +// currentTime, +// avgCPU, +// minimumCPU, +// maximumCPU +// ) +// val jvmDataPoint = ClusterMetricsDataPoint( +// ClusterMetricsDataPoint.MetricType.JVM_PRESSURE, +// currentTime, +// avgJVM, +// minimumJVM, +// maximumJVM +// ) +// val pendingTasksData = ClusterMetricsDataPoint( +// ClusterMetricsDataPoint.MetricType.NUMBER_OF_PENDING_TASKS, +// currentTime, +// numPending +// ) +// val activeShardsData = ClusterMetricsDataPoint( +// ClusterMetricsDataPoint.MetricType.ACTIVE_SHARDS, +// currentTime, +// activeShards +// ) +// val relocatingShardsData = ClusterMetricsDataPoint( +// ClusterMetricsDataPoint.MetricType.RELOCATING_SHARDS, +// currentTime, +// relocatingShards +// ) +// val nodesData = ClusterMetricsDataPoint( +// ClusterMetricsDataPoint.MetricType.NUMBER_OF_NODES, +// currentTime, +// numNodes +// ) +// val dataNodesData = ClusterMetricsDataPoint( +// ClusterMetricsDataPoint.MetricType.NUMBER_OF_DATA_NODES, +// currentTime, +// numDataNodes +// ) - try { - val indexResponse: IndexResponse = client.suspendUntil { client.index(indexrequestStatus, it) } - val indexResponse2: IndexResponse = client.suspendUntil { client.index(indexrequestShards, it) } - val indexResponse3: IndexResponse = client.suspendUntil { client.index(indexrequestCpu, it) } - val indexResponse4: IndexResponse = client.suspendUntil { client.index(indexrequestJvm, it) } - val indexResponse5: IndexResponse = client.suspendUntil { client.index(indexrequestPending, it) } - val indexResponse6: IndexResponse = client.suspendUntil { client.index(indexrequestActive, it) } - val indexResponse7: IndexResponse = client.suspendUntil { client.index(indexrequestRelocating, it) } - val indexResponse8: IndexResponse = client.suspendUntil { client.index(indexrequestNodes, it) } - val indexResponse9: IndexResponse = client.suspendUntil { client.index(indexrequestDatanodes, it) } - val failureReasons = checkShardsFailure(indexResponse) - val failureReasons2 = checkShardsFailure(indexResponse2) - val failureReasons3 = checkShardsFailure(indexResponse3) - val failureReasons4 = checkShardsFailure(indexResponse4) - val failureReasons5 = checkShardsFailure(indexResponse5) - val failureReasons6 = checkShardsFailure(indexResponse6) - val failureReasons7 = checkShardsFailure(indexResponse7) - val failureReasons8 = checkShardsFailure(indexResponse8) - val failureReasons9 = checkShardsFailure(indexResponse9) - if ( - failureReasons != null || - failureReasons2 != null || - failureReasons3 != null || - failureReasons4 != null || - failureReasons5 != null || - failureReasons6 != null || - failureReasons7 != null || - failureReasons8 != null || - failureReasons9 != null - ) { - log.info("failed because $failureReasons") - log.info("failed because $failureReasons2") - log.info("failed because $failureReasons3") - log.info("failed because $failureReasons4") - log.info("failed because $failureReasons5") - log.info("failed because $failureReasons6") - log.info("failed because $failureReasons7") - log.info("failed because $failureReasons8") - log.info("failed because $failureReasons9") - return + dataPoints.forEach { clusterMetricsDataPoint -> + try { + val request = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) + .source( + clusterMetricsDataPoint.toXContent( + XContentFactory.jsonBuilder(), + ToXContent.MapParams(mapOf("with_type" to "true")) + ) + ) + val indexResponse: IndexResponse = client.suspendUntil { client.index(request, it) } + val failureReasons = checkShardsFailure(indexResponse) + if (failureReasons != null) { + log.info("failed because $failureReasons") + } + } catch (t: Exception) { + log.info("Unable to get index response, $t") } - } catch (t: Exception) { - log.info("Unable to get index response, $t") } +// val indexrequestStatus = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .source(clusterstatusData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) +// val indexrequestShards = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .source(unassignedShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) +// val indexrequestCpu = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .source(cpuUsageData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) +// val indexrequestJvm = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .source(jvmDataPoint.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) +// val indexrequestPending = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .source(pendingTasksData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) +// val indexrequestActive = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .source(activeShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) +// val indexrequestRelocating = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .source(relocatingShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) +// val indexrequestNodes = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .source(nodesData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) +// val indexrequestDatanodes = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) +// .source(dataNodesData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) +// +// try { +// val indexResponse: IndexResponse = client.suspendUntil { client.index(indexrequestStatus, it) } +// val indexResponse2: IndexResponse = client.suspendUntil { client.index(indexrequestShards, it) } +// val indexResponse3: IndexResponse = client.suspendUntil { client.index(indexrequestCpu, it) } +// val indexResponse4: IndexResponse = client.suspendUntil { client.index(indexrequestJvm, it) } +// val indexResponse5: IndexResponse = client.suspendUntil { client.index(indexrequestPending, it) } +// val indexResponse6: IndexResponse = client.suspendUntil { client.index(indexrequestActive, it) } +// val indexResponse7: IndexResponse = client.suspendUntil { client.index(indexrequestRelocating, it) } +// val indexResponse8: IndexResponse = client.suspendUntil { client.index(indexrequestNodes, it) } +// val indexResponse9: IndexResponse = client.suspendUntil { client.index(indexrequestDatanodes, it) } +// val failureReasons = checkShardsFailure(indexResponse) +// val failureReasons2 = checkShardsFailure(indexResponse2) +// val failureReasons3 = checkShardsFailure(indexResponse3) +// val failureReasons4 = checkShardsFailure(indexResponse4) +// val failureReasons5 = checkShardsFailure(indexResponse5) +// val failureReasons6 = checkShardsFailure(indexResponse6) +// val failureReasons7 = checkShardsFailure(indexResponse7) +// val failureReasons8 = checkShardsFailure(indexResponse8) +// val failureReasons9 = checkShardsFailure(indexResponse9) +// if ( +// failureReasons != null || +// failureReasons2 != null || +// failureReasons3 != null || +// failureReasons4 != null || +// failureReasons5 != null || +// failureReasons6 != null || +// failureReasons7 != null || +// failureReasons8 != null || +// failureReasons9 != null +// ) { +// log.info("failed because $failureReasons") +// log.info("failed because $failureReasons2") +// log.info("failed because $failureReasons3") +// log.info("failed because $failureReasons4") +// log.info("failed because $failureReasons5") +// log.info("failed because $failureReasons6") +// log.info("failed because $failureReasons7") +// log.info("failed because $failureReasons8") +// log.info("failed because $failureReasons9") +// return +// } +// } catch (t: Exception) { +// log.info("Unable to get index response, $t") +// } } private fun checkShardsFailure(response: IndexResponse): String? { val failureReasons = StringBuilder() @@ -318,130 +388,5 @@ class ClusterMetricsCoordinator( ) log.info("deleted ${it.metricName} data from $documentAge ago") } -// DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) -// .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .filter(QueryBuilders.rangeQuery("cluster_status.timestamp").lte("now-$documentAge")) -// .execute( -// object : ActionListener { -// override fun onResponse(response: BulkByScrollResponse) { -// } -// -// override fun onFailure(t: Exception) { -// } -// } -// ) -// log.info("deleted clusterStatus data from $documentAge/$unitTime ago, now-$documentAge") -// -// DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) -// .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .filter(QueryBuilders.rangeQuery("cpu_usage.timestamp").lte("now-$documentAge")) -// .execute( -// object : ActionListener { -// override fun onResponse(response: BulkByScrollResponse) { -// } -// -// override fun onFailure(t: Exception) { -// } -// } -// ) -// log.info("deleted cpu_usage data from $documentAge/$unitTime ago") -// -// DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) -// .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .filter(QueryBuilders.rangeQuery("jvm_pressure.timestamp").lte("now-$documentAge")) -// .execute( -// object : ActionListener { -// override fun onResponse(response: BulkByScrollResponse) { -// } -// -// override fun onFailure(t: Exception) { -// } -// } -// ) -// log.info("deleted jvm_pressure data from $documentAge/$unitTime ago") -// -// DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) -// .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .filter(QueryBuilders.rangeQuery("unassigned_shards.timestamp").lte("now-$documentAge")) -// .execute( -// object : ActionListener { -// override fun onResponse(response: BulkByScrollResponse) { -// } -// -// override fun onFailure(t: Exception) { -// } -// } -// ) -// log.info("deleted unassigned_shards data from $documentAge/$unitTime ago") -// -// DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) -// .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .filter(QueryBuilders.rangeQuery("number_of_pending_tasks.timestamp").lte("now-$documentAge")) -// .execute( -// object : ActionListener { -// override fun onResponse(response: BulkByScrollResponse) { -// } -// -// override fun onFailure(t: Exception) { -// } -// } -// ) -// log.info("deleted number of pending tasks from $documentAge/$unitTime ago") -// -// DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) -// .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .filter(QueryBuilders.rangeQuery("active_shards.timestamp").lte("now-$documentAge")) -// .execute( -// object : ActionListener { -// override fun onResponse(response: BulkByScrollResponse) { -// } -// -// override fun onFailure(t: Exception) { -// } -// } -// ) -// log.info("deleted active shards from $documentAge/$unitTime ago") -// -// DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) -// .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .filter(QueryBuilders.rangeQuery("relocating_shards.timestamp").lte("now-$documentAge")) -// .execute( -// object : ActionListener { -// override fun onResponse(response: BulkByScrollResponse) { -// } -// -// override fun onFailure(t: Exception) { -// } -// } -// ) -// log.info("deleted number of relocating shards from $documentAge/$unitTime ago") -// -// DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) -// .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .filter(QueryBuilders.rangeQuery("number_of_nodes.timestamp").lte("now-$documentAge")) -// .execute( -// object : ActionListener { -// override fun onResponse(response: BulkByScrollResponse) { -// } -// -// override fun onFailure(t: Exception) { -// } -// } -// ) -// log.info("deleted number of nodes from $documentAge/$unitTime ago") -// -// DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) -// .source(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .filter(QueryBuilders.rangeQuery("number_of_data_nodes.timestamp").lte("now-$documentAge")) -// .execute( -// object : ActionListener { -// override fun onResponse(response: BulkByScrollResponse) { -// } -// -// override fun onFailure(t: Exception) { -// } -// } -// ) -// log.info("deleted number of data nodes from $documentAge/$unitTime ago") } } From 7f9204da693291d43642ff2306410f82aba5e34f Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 10 Aug 2022 16:53:10 -0700 Subject: [PATCH 199/215] Including validator logic for settings. Signed-off-by: Richard --- .../alerting/settings/AlertingSettings.kt | 79 ++++++++++++++++--- .../util/ClusterMetricsCoordinator.kt | 46 +++++------ 2 files changed, 88 insertions(+), 37 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index 9cab638e8..8b10e3d22 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -13,6 +13,8 @@ import java.util.concurrent.TimeUnit /** * settings specific to [AlertingPlugin]. These settings include things like history index max age, request timeout, etc... */ +private val log = org.apache.logging.log4j.LogManager.getLogger(AlertingSettings::class.java) + class AlertingSettings { companion object { @@ -20,6 +22,17 @@ class AlertingSettings { const val MONITOR_MAX_INPUTS = 1 const val MONITOR_MAX_TRIGGERS = 10 const val DEFAULT_MAX_ACTIONABLE_ALERT_COUNT = 50L + val METRICS_EXECUTION_FREQUENCY_DEFAULT = Setting.positiveTimeSetting( + "plugins.alerting.cluster_metrics.execution_frequency", + TimeValue(15, TimeUnit.MINUTES), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + val METRICS_STORE_TIME_DEFAULT = Setting.positiveTimeSetting( + "plugins.alerting.cluster_metrics.metrics_history_max_age", + TimeValue(7, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + val MINIMUM_TIME_VALUE = TimeValue(1, TimeUnit.SECONDS) val ALERTING_MAX_MONITORS = Setting.intSetting( "plugins.alerting.monitor.max_monitors", @@ -157,16 +170,60 @@ class AlertingSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ) - val METRICS_STORE_TIME = Setting.positiveTimeSetting( - "plugins.alerting.cluster_metrics.metrics_history_max_age", - TimeValue(7, TimeUnit.DAYS), - Setting.Property.NodeScope, Setting.Property.Dynamic - ) - - val METRICS_EXECUTION_FREQUENCY = Setting.positiveTimeSetting( - "plugins.alerting.cluster_metrics.execution_frequency", - TimeValue(15, TimeUnit.MINUTES), - Setting.Property.NodeScope, Setting.Property.Dynamic - ) + val METRICS_STORE_TIME = Setting.timeSetting( + METRICS_STORE_TIME_DEFAULT.key, + METRICS_STORE_TIME_DEFAULT, + StorageValidator(), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + + val METRICS_EXECUTION_FREQUENCY = Setting.timeSetting( + METRICS_EXECUTION_FREQUENCY_DEFAULT.key, + METRICS_EXECUTION_FREQUENCY_DEFAULT, + ExecutionValidator(), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + internal class ExecutionValidator : Setting.Validator { + override fun validate(value: TimeValue) {} + + override fun validate(value: TimeValue, settings: Map, Any>) { + val storageTime = settings[METRICS_STORE_TIME] as TimeValue + log.info("THIS IS STORAGETIME $storageTime") + validateSettings(value, storageTime) + } + + override fun settings(): MutableIterator> { + val settings = mutableListOf>( + METRICS_EXECUTION_FREQUENCY, + METRICS_STORE_TIME + ) + return settings.iterator() + } + } + + internal class StorageValidator : Setting.Validator { + override fun validate(value: TimeValue) {} + + override fun validate(value: TimeValue, settings: Map, Any>) { + val executionTime = settings[METRICS_EXECUTION_FREQUENCY] as TimeValue + log.info("THIS IS EXECUTIONTIME $executionTime") + validateSettings(executionTime, value) + } + + override fun settings(): MutableIterator> { + val settings = mutableListOf>( + METRICS_EXECUTION_FREQUENCY, + METRICS_STORE_TIME + ) + return settings.iterator() + } + } + private fun validateSettings(executionFrequency: TimeValue, storageTime: TimeValue) { + if (executionFrequency > storageTime) { + throw java.lang.IllegalArgumentException( + "The execution frequency should be less than the storage time." + ) + } + } } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index f002bc8a3..0bfcbc2e1 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -76,34 +76,28 @@ class ClusterMetricsCoordinator( } override fun clusterChanged(event: ClusterChangedEvent?) { - if (metricsStoreTime > metricsExecutionFrequency) { - val scheduledJobCollection = Runnable { - launch { - createDocs(client as NodeClient, clusterService) - deleteDocs(client) - } - } - if (isCollectionUpdated || isDeletionUpdated) { - dataPointCollectionDeletionJob?.cancel() - log.info("Cancelled data collection and deletion jobs") - isRunningFlag = false - log.info("detected changes to settings, resetting running, deletion and collection flags to false") - isDeletionUpdated = false - isCollectionUpdated = false - } - if (event!!.localNodeMaster() && !isRunningFlag) { - log.info("cluster changed metricsExecutionFrequency = $metricsExecutionFrequency") - dataPointCollectionDeletionJob = threadPool.scheduleWithFixedDelay( - scheduledJobCollection, - metricsExecutionFrequency, - ThreadPool.Names.SYSTEM_WRITE - ) - isRunningFlag = true + val scheduledJobCollection = Runnable { + launch { + createDocs(client as NodeClient, clusterService) + deleteDocs(client) } - } else { - throw java.lang.IllegalArgumentException( - "Execution frequency ($metricsExecutionFrequency) can not be greater than the storage time ($metricsStoreTime). " + } + if (isCollectionUpdated || isDeletionUpdated) { + dataPointCollectionDeletionJob?.cancel() + log.info("Cancelled data collection and deletion jobs") + isRunningFlag = false + log.info("detected changes to settings, resetting running, deletion and collection flags to false") + isDeletionUpdated = false + isCollectionUpdated = false + } + if (event!!.localNodeMaster() && !isRunningFlag) { + log.info("cluster changed metricsExecutionFrequency = $metricsExecutionFrequency") + dataPointCollectionDeletionJob = threadPool.scheduleWithFixedDelay( + scheduledJobCollection, + metricsExecutionFrequency, + ThreadPool.Names.SYSTEM_WRITE ) + isRunningFlag = true } } From 7fea94ec039cb3509d806f38d9cc1e067250610c Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 10 Aug 2022 17:04:19 -0700 Subject: [PATCH 200/215] Including validator logic for settings. Modified IT to reflect these changes. Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 40c18ace7..ab8c1d8da 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -117,8 +117,8 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { // Reset the settings back to default, delete the created index. fun cleanup() { // reset settings - client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "15m") client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "7d") + client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "15m") client().makeRequest("DELETE", ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) } private fun generateData() { From 5a2d1f552cc5283d81e5e3dee80c65ae0f757c6a Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 11 Aug 2022 11:11:58 -0700 Subject: [PATCH 201/215] Including validator logic for settings. Modified IT to reflect these changes. Signed-off-by: Richard --- .../alerting/settings/AlertingSettings.kt | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index 8b10e3d22..343fa7a8c 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -19,6 +19,8 @@ class AlertingSettings { companion object { + val MINIMUM_EXECUTION_FREQUENCY = TimeValue(1, TimeUnit.SECONDS) + val MINIMUM_STORAGE_TIME = TimeValue(1, TimeUnit.MINUTES) const val MONITOR_MAX_INPUTS = 1 const val MONITOR_MAX_TRIGGERS = 10 const val DEFAULT_MAX_ACTIONABLE_ALERT_COUNT = 50L @@ -32,7 +34,6 @@ class AlertingSettings { TimeValue(7, TimeUnit.DAYS), Setting.Property.NodeScope, Setting.Property.Dynamic ) - val MINIMUM_TIME_VALUE = TimeValue(1, TimeUnit.SECONDS) val ALERTING_MAX_MONITORS = Setting.intSetting( "plugins.alerting.monitor.max_monitors", @@ -206,6 +207,7 @@ class AlertingSettings { override fun validate(value: TimeValue, settings: Map, Any>) { val executionTime = settings[METRICS_EXECUTION_FREQUENCY] as TimeValue + log.info("THIS IS EXECUTIONTIME AS MILLIS ${executionTime.millis()}") log.info("THIS IS EXECUTIONTIME $executionTime") validateSettings(executionTime, value) } @@ -220,9 +222,17 @@ class AlertingSettings { } private fun validateSettings(executionFrequency: TimeValue, storageTime: TimeValue) { if (executionFrequency > storageTime) { - throw java.lang.IllegalArgumentException( + throw IllegalArgumentException( "The execution frequency should be less than the storage time." ) + } else if (executionFrequency < MINIMUM_EXECUTION_FREQUENCY) { + throw IllegalArgumentException( + "The execution frequency can not be less than 1 second." + ) + } else if (storageTime < MINIMUM_STORAGE_TIME) { + throw IllegalArgumentException( + "The storage time can not be less than 1 minute." + ) } } } From 6f5c6f17ac86f756d6ae7335d3810220a7c4d84e Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 11 Aug 2022 13:12:16 -0700 Subject: [PATCH 202/215] Added IT for validator. Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index ab8c1d8da..376a2e838 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -113,6 +113,36 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { assertEquals(diff, 2) } + fun `test update storage time to less than minimum storage time`() { + assertThrows(IllegalArgumentException::class.java) { + client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "30s") + } + } + + fun `test update frequency to less than minimum frequency`() { + assertThrows(IllegalArgumentException::class.java) { + client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1ms") + } + } + + fun `test when execution frequency greater than storage time`() { + assertThrows(IllegalArgumentException::class.java) { + client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "20m") + client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "25m") + } + } + + fun `test successful client setting update`() { + client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "400m") + client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "12m") + val response = getSettings() + val persistentMap = response["persistent"] as Map + val executionFrequency = persistentMap["plugins.alerting.cluster_metrics.execution_frequency"].toString() + val storageTime = persistentMap["plugins.alerting.cluster_metrics.metrics_history_max_age"].toString() + assertEquals(executionFrequency, "12m") + assertEquals(storageTime, "400m") + } + @After // Reset the settings back to default, delete the created index. fun cleanup() { From b061938f68df94ae815db90a2f3ce8cf7ce16dfe Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 11 Aug 2022 13:29:20 -0700 Subject: [PATCH 203/215] Added IT for validator. Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 376a2e838..eeba28144 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -15,6 +15,7 @@ import org.opensearch.alerting.model.ClusterMetricsDataPoint import org.opensearch.alerting.opensearchapi.string import org.opensearch.alerting.util.ClusterMetricsVisualizationIndex import org.opensearch.client.Response +import org.opensearch.client.ResponseException import org.opensearch.common.xcontent.XContentFactory.jsonBuilder import org.opensearch.common.xcontent.XContentType import org.opensearch.rest.RestStatus @@ -115,7 +116,12 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { fun `test update storage time to less than minimum storage time`() { assertThrows(IllegalArgumentException::class.java) { - client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "30s") + try { + client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "30s") + } catch (t: ResponseException) { + logger.info("LOG THE RESPONSE EXCEPTION OBJECT $t") + } + } } From 127ed3e5064d810b1b61339c490bafd019a3bb22 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 11 Aug 2022 13:30:48 -0700 Subject: [PATCH 204/215] Added IT for validator. Signed-off-by: Richard --- .../clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index eeba28144..0f4045225 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -121,7 +121,6 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { } catch (t: ResponseException) { logger.info("LOG THE RESPONSE EXCEPTION OBJECT $t") } - } } From db96e4a61634ce8d23dc8ebb33a8fa9677ade6e0 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 11 Aug 2022 14:06:23 -0700 Subject: [PATCH 205/215] Added IT for validator. Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 0f4045225..6dce6029d 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -119,7 +119,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { try { client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "30s") } catch (t: ResponseException) { - logger.info("LOG THE RESPONSE EXCEPTION OBJECT $t") + logger.info("LOG THE RESPONSE EXCEPTION OBJECT ${t.response}") } } } From d090416219222d067575829fc19224cf9d394877 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 11 Aug 2022 14:09:00 -0700 Subject: [PATCH 206/215] Added IT for validator. Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 6dce6029d..bcea461e0 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -119,7 +119,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { try { client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "30s") } catch (t: ResponseException) { - logger.info("LOG THE RESPONSE EXCEPTION OBJECT ${t.response}") + logger.info("LOG THE RESPONSE EXCEPTION OBJECT ${t.cause}") } } } From 293d6a7b29cacafabe0f3728a031e4bfa4c8e0a9 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 11 Aug 2022 14:11:17 -0700 Subject: [PATCH 207/215] Added IT for validator. Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index bcea461e0..1a5c618ea 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -119,7 +119,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { try { client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "30s") } catch (t: ResponseException) { - logger.info("LOG THE RESPONSE EXCEPTION OBJECT ${t.cause}") + logger.info("LOG THE RESPONSE EXCEPTION OBJECT ${t.stackTrace}") } } } From 5dda938c3d176bbcd116641e7fc45182a749f327 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 11 Aug 2022 14:13:40 -0700 Subject: [PATCH 208/215] Added IT for validator. Signed-off-by: Richard --- .../org/opensearch/alerting/settings/AlertingSettings.kt | 8 +++++--- .../ClusterMetricsCoordinatorIT.kt | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index 343fa7a8c..03fecb835 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -18,17 +18,19 @@ private val log = org.apache.logging.log4j.LogManager.getLogger(AlertingSettings class AlertingSettings { companion object { - - val MINIMUM_EXECUTION_FREQUENCY = TimeValue(1, TimeUnit.SECONDS) - val MINIMUM_STORAGE_TIME = TimeValue(1, TimeUnit.MINUTES) const val MONITOR_MAX_INPUTS = 1 const val MONITOR_MAX_TRIGGERS = 10 const val DEFAULT_MAX_ACTIONABLE_ALERT_COUNT = 50L + + val MINIMUM_EXECUTION_FREQUENCY = TimeValue(1, TimeUnit.SECONDS) + val MINIMUM_STORAGE_TIME = TimeValue(1, TimeUnit.MINUTES) + val METRICS_EXECUTION_FREQUENCY_DEFAULT = Setting.positiveTimeSetting( "plugins.alerting.cluster_metrics.execution_frequency", TimeValue(15, TimeUnit.MINUTES), Setting.Property.NodeScope, Setting.Property.Dynamic ) + val METRICS_STORE_TIME_DEFAULT = Setting.positiveTimeSetting( "plugins.alerting.cluster_metrics.metrics_history_max_age", TimeValue(7, TimeUnit.DAYS), diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 1a5c618ea..761c22eb8 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -119,7 +119,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { try { client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "30s") } catch (t: ResponseException) { - logger.info("LOG THE RESPONSE EXCEPTION OBJECT ${t.stackTrace}") + logger.info("LOG THE RESPONSE EXCEPTION OBJECT ${t.message}") } } } From f29bf7339fc7ef76e231b8bb3f5d33f2e17f2dce Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 11 Aug 2022 14:16:24 -0700 Subject: [PATCH 209/215] Added IT for validator. Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 761c22eb8..749b46af9 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -119,7 +119,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { try { client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "30s") } catch (t: ResponseException) { - logger.info("LOG THE RESPONSE EXCEPTION OBJECT ${t.message}") + logger.info("LOG THE RESPONSE EXCEPTION OBJECT ${t.suppressed}") } } } From b26b3379da4c024b29590c2372b227dbbbdb4017 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 11 Aug 2022 14:19:14 -0700 Subject: [PATCH 210/215] Added IT for validator. Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 749b46af9..d2465793f 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -119,7 +119,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { try { client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "30s") } catch (t: ResponseException) { - logger.info("LOG THE RESPONSE EXCEPTION OBJECT ${t.suppressed}") + logger.info("LOG THE RESPONSE EXCEPTION OBJECT ${t.localizedMessage}") } } } From cf13c25de7bb6e3eb438baceb374ba34c9b05f52 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 11 Aug 2022 15:44:13 -0700 Subject: [PATCH 211/215] Testing IT, fixed some suggestions that Thomas had commented. Signed-off-by: Richard --- .../org/opensearch/alerting/AlertingPlugin.kt | 5 +- .../util/ClusterMetricsCoordinator.kt | 120 +----------------- .../ClusterMetricsCoordinatorIT.kt | 5 +- 3 files changed, 7 insertions(+), 123 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index c003f058a..033a5eb90 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -112,7 +112,7 @@ import java.util.function.Supplier * It also adds [Monitor.XCONTENT_REGISTRY], [SearchInput.XCONTENT_REGISTRY], [QueryLevelTrigger.XCONTENT_REGISTRY], * [BucketLevelTrigger.XCONTENT_REGISTRY], [ClusterMetricsInput.XCONTENT_REGISTRY] to the [NamedXContentRegistry] so that we are able to deserialize the custom named objects. */ -private val log = LogManager.getLogger(AlertingPlugin::class.java) + internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, ReloadablePlugin, SearchPlugin, PercolatorPluginExt() { override fun getContextWhitelists(): Map, List> { @@ -245,9 +245,6 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R clusterMetricsCoordinator = ClusterMetricsCoordinator(settings, client, clusterService, threadPool) this.threadPool = threadPool this.clusterService = clusterService - // create a cluster metrics visualization index upon initialization of alerting plugin - // clusterMetricsVisualization = ClusterMetricsVisualizationIndex(client, clusterService, threadPool) - // ClusterMetricsVisualizationIndex(client, clusterService) return listOf( sweeper, scheduler, diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 0bfcbc2e1..577a94f73 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -219,57 +219,6 @@ class ClusterMetricsCoordinator( numDataNodes ) ) - -// val clusterstatusData = ClusterMetricsDataPoint( -// ClusterMetricsDataPoint.MetricType.CLUSTER_STATUS, -// currentTime, -// clusterStatus -// ) -// val unassignedShardsData = ClusterMetricsDataPoint( -// ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS, -// currentTime, -// unassignedShards -// ) -// val cpuUsageData = ClusterMetricsDataPoint( -// ClusterMetricsDataPoint.MetricType.CPU_USAGE, -// currentTime, -// avgCPU, -// minimumCPU, -// maximumCPU -// ) -// val jvmDataPoint = ClusterMetricsDataPoint( -// ClusterMetricsDataPoint.MetricType.JVM_PRESSURE, -// currentTime, -// avgJVM, -// minimumJVM, -// maximumJVM -// ) -// val pendingTasksData = ClusterMetricsDataPoint( -// ClusterMetricsDataPoint.MetricType.NUMBER_OF_PENDING_TASKS, -// currentTime, -// numPending -// ) -// val activeShardsData = ClusterMetricsDataPoint( -// ClusterMetricsDataPoint.MetricType.ACTIVE_SHARDS, -// currentTime, -// activeShards -// ) -// val relocatingShardsData = ClusterMetricsDataPoint( -// ClusterMetricsDataPoint.MetricType.RELOCATING_SHARDS, -// currentTime, -// relocatingShards -// ) -// val nodesData = ClusterMetricsDataPoint( -// ClusterMetricsDataPoint.MetricType.NUMBER_OF_NODES, -// currentTime, -// numNodes -// ) -// val dataNodesData = ClusterMetricsDataPoint( -// ClusterMetricsDataPoint.MetricType.NUMBER_OF_DATA_NODES, -// currentTime, -// numDataNodes -// ) - dataPoints.forEach { clusterMetricsDataPoint -> try { val request = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) @@ -282,75 +231,12 @@ class ClusterMetricsCoordinator( val indexResponse: IndexResponse = client.suspendUntil { client.index(request, it) } val failureReasons = checkShardsFailure(indexResponse) if (failureReasons != null) { - log.info("failed because $failureReasons") + log.info("Failed to index ${clusterMetricsDataPoint.metric}.", failureReasons) } } catch (t: Exception) { - log.info("Unable to get index response, $t") + log.info("Failed to index ${clusterMetricsDataPoint.metric}.", t) } } -// val indexrequestStatus = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .source(clusterstatusData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) -// val indexrequestShards = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .source(unassignedShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) -// val indexrequestCpu = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .source(cpuUsageData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) -// val indexrequestJvm = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .source(jvmDataPoint.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) -// val indexrequestPending = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .source(pendingTasksData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) -// val indexrequestActive = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .source(activeShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) -// val indexrequestRelocating = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .source(relocatingShardsData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) -// val indexrequestNodes = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .source(nodesData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) -// val indexrequestDatanodes = IndexRequest(ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) -// .source(dataNodesData.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) -// -// try { -// val indexResponse: IndexResponse = client.suspendUntil { client.index(indexrequestStatus, it) } -// val indexResponse2: IndexResponse = client.suspendUntil { client.index(indexrequestShards, it) } -// val indexResponse3: IndexResponse = client.suspendUntil { client.index(indexrequestCpu, it) } -// val indexResponse4: IndexResponse = client.suspendUntil { client.index(indexrequestJvm, it) } -// val indexResponse5: IndexResponse = client.suspendUntil { client.index(indexrequestPending, it) } -// val indexResponse6: IndexResponse = client.suspendUntil { client.index(indexrequestActive, it) } -// val indexResponse7: IndexResponse = client.suspendUntil { client.index(indexrequestRelocating, it) } -// val indexResponse8: IndexResponse = client.suspendUntil { client.index(indexrequestNodes, it) } -// val indexResponse9: IndexResponse = client.suspendUntil { client.index(indexrequestDatanodes, it) } -// val failureReasons = checkShardsFailure(indexResponse) -// val failureReasons2 = checkShardsFailure(indexResponse2) -// val failureReasons3 = checkShardsFailure(indexResponse3) -// val failureReasons4 = checkShardsFailure(indexResponse4) -// val failureReasons5 = checkShardsFailure(indexResponse5) -// val failureReasons6 = checkShardsFailure(indexResponse6) -// val failureReasons7 = checkShardsFailure(indexResponse7) -// val failureReasons8 = checkShardsFailure(indexResponse8) -// val failureReasons9 = checkShardsFailure(indexResponse9) -// if ( -// failureReasons != null || -// failureReasons2 != null || -// failureReasons3 != null || -// failureReasons4 != null || -// failureReasons5 != null || -// failureReasons6 != null || -// failureReasons7 != null || -// failureReasons8 != null || -// failureReasons9 != null -// ) { -// log.info("failed because $failureReasons") -// log.info("failed because $failureReasons2") -// log.info("failed because $failureReasons3") -// log.info("failed because $failureReasons4") -// log.info("failed because $failureReasons5") -// log.info("failed because $failureReasons6") -// log.info("failed because $failureReasons7") -// log.info("failed because $failureReasons8") -// log.info("failed because $failureReasons9") -// return -// } -// } catch (t: Exception) { -// log.info("Unable to get index response, $t") -// } } private fun checkShardsFailure(response: IndexResponse): String? { val failureReasons = StringBuilder() @@ -380,7 +266,7 @@ class ClusterMetricsCoordinator( } } ) - log.info("deleted ${it.metricName} data from $documentAge ago") + log.info("Deleted ${it.metricName} data from $documentAge ago.") } } } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index d2465793f..ee5ad5b3f 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -119,7 +119,8 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { try { client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "30s") } catch (t: ResponseException) { - logger.info("LOG THE RESPONSE EXCEPTION OBJECT ${t.localizedMessage}") + logger.info("LOG THE RESPONSE EXCEPTION OBJECT ${t.response.asMap()}") + t } } } @@ -130,7 +131,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { } } - fun `test when execution frequency greater than storage time`() { + fun `test update execution frequency greater than storage time`() { assertThrows(IllegalArgumentException::class.java) { client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "20m") client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "25m") From 07181f38e036a0bcf37bd7d8941a4729880593f6 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 11 Aug 2022 15:45:51 -0700 Subject: [PATCH 212/215] Testing IT, fixed some suggestions that Thomas had commented. Signed-off-by: Richard --- .../src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 033a5eb90..bb9f8828f 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -5,7 +5,6 @@ package org.opensearch.alerting -import org.apache.logging.log4j.LogManager import org.opensearch.action.ActionRequest import org.opensearch.action.ActionResponse import org.opensearch.alerting.action.AcknowledgeAlertAction From 976e0b886acc562338e7becd2ba7cd0ac2d58308 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 11 Aug 2022 15:54:08 -0700 Subject: [PATCH 213/215] Testing IT, fixed some suggestions that Thomas had commented. Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index ee5ad5b3f..1a93c804a 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -115,13 +115,12 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { } fun `test update storage time to less than minimum storage time`() { - assertThrows(IllegalArgumentException::class.java) { - try { - client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "30s") - } catch (t: ResponseException) { - logger.info("LOG THE RESPONSE EXCEPTION OBJECT ${t.response.asMap()}") - t - } + try { + client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "30s") + } catch (t: ResponseException) { + val responseMap = t.response.asMap() + val errMap = responseMap["error"] as Map + assertEquals("illegal_argument_exception", errMap["type"]) } } From 8f79701950a587de8d0898ceda76bf53dc877896 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 11 Aug 2022 16:00:00 -0700 Subject: [PATCH 214/215] Testing IT, fixed some suggestions that Thomas had commented. Signed-off-by: Richard --- .../ClusterMetricsCoordinatorIT.kt | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 1a93c804a..9e9f2c92a 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -125,15 +125,23 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { } fun `test update frequency to less than minimum frequency`() { - assertThrows(IllegalArgumentException::class.java) { + try { client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "1ms") + } catch (t: ResponseException) { + val responseMap = t.response.asMap() + val errMap = responseMap["error"] as Map + assertEquals("illegal_argument_exception", errMap["type"]) } } fun `test update execution frequency greater than storage time`() { - assertThrows(IllegalArgumentException::class.java) { + try { client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "20m") client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "25m") + } catch (t: ResponseException) { + val responseMap = t.response.asMap() + val errMap = responseMap["error"] as Map + assertEquals("illegal_argument_exception", errMap["type"]) } } From a607e5b7a43e3103ea3b3bff9525b91e94c35dfd Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 11 Aug 2022 16:54:24 -0700 Subject: [PATCH 215/215] Changed code based on feedback. Signed-off-by: Richard --- .../alerting/settings/AlertingSettings.kt | 11 ++-- .../util/ClusterMetricsCoordinator.kt | 17 ++++--- .../ClusterMetricsCoordinatorIT.kt | 50 ++++++++++++++++--- 3 files changed, 60 insertions(+), 18 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index 03fecb835..846f483b7 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -25,15 +25,18 @@ class AlertingSettings { val MINIMUM_EXECUTION_FREQUENCY = TimeValue(1, TimeUnit.SECONDS) val MINIMUM_STORAGE_TIME = TimeValue(1, TimeUnit.MINUTES) + val METRICS_EXECUTION_FREQUENCY_DEFAULT_VALUE = TimeValue(15, TimeUnit.MINUTES) + val METRICS_STORE_TIME_DEFAULT_VALUE = TimeValue(7, TimeUnit.DAYS) + val METRICS_EXECUTION_FREQUENCY_DEFAULT = Setting.positiveTimeSetting( "plugins.alerting.cluster_metrics.execution_frequency", - TimeValue(15, TimeUnit.MINUTES), + METRICS_EXECUTION_FREQUENCY_DEFAULT_VALUE, Setting.Property.NodeScope, Setting.Property.Dynamic ) val METRICS_STORE_TIME_DEFAULT = Setting.positiveTimeSetting( "plugins.alerting.cluster_metrics.metrics_history_max_age", - TimeValue(7, TimeUnit.DAYS), + METRICS_STORE_TIME_DEFAULT_VALUE, Setting.Property.NodeScope, Setting.Property.Dynamic ) @@ -229,11 +232,11 @@ class AlertingSettings { ) } else if (executionFrequency < MINIMUM_EXECUTION_FREQUENCY) { throw IllegalArgumentException( - "The execution frequency can not be less than 1 second." + "The execution frequency can not be less than $MINIMUM_EXECUTION_FREQUENCY" ) } else if (storageTime < MINIMUM_STORAGE_TIME) { throw IllegalArgumentException( - "The storage time can not be less than 1 minute." + "The storage time can not be less than $MINIMUM_STORAGE_TIME." ) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt index 577a94f73..53ef1f7b1 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/ClusterMetricsCoordinator.kt @@ -109,19 +109,19 @@ class ClusterMetricsCoordinator( ClusterMetricsVisualizationIndex.initFunc(client, clusterService) - val unassignedShards = clusterHealth["unassigned_shards"].toString() + val unassignedShards = clusterHealth[ClusterMetricsDataPoint.MetricType.UNASSIGNED_SHARDS.metricName].toString() log.info("This is unassigned shards value: $unassignedShards") val clusterStatus = clusterHealth["status"].toString() log.info("This is cluster status value: $clusterStatus") - val numPending = clusterHealth["number_of_pending_tasks"].toString() + val numPending = clusterHealth[ClusterMetricsDataPoint.MetricType.NUMBER_OF_PENDING_TASKS.metricName].toString() log.info("This is the number of pending tasks: $numPending") - val activeShards = clusterHealth["active_shards"].toString() + val activeShards = clusterHealth[ClusterMetricsDataPoint.MetricType.ACTIVE_SHARDS.metricName].toString() log.info("This is active shards $activeShards") - val relocatingShards = clusterHealth["relocating_shards"].toString() + val relocatingShards = clusterHealth[ClusterMetricsDataPoint.MetricType.RELOCATING_SHARDS.metricName].toString() log.info("This is relocating shards $relocatingShards") - val numNodes = clusterHealth["number_of_nodes"].toString() + val numNodes = clusterHealth[ClusterMetricsDataPoint.MetricType.NUMBER_OF_NODES.metricName].toString() log.info("This is number of nodes $numNodes") - val numDataNodes = clusterHealth["number_of_data_nodes"].toString() + val numDataNodes = clusterHealth[ClusterMetricsDataPoint.MetricType.NUMBER_OF_DATA_NODES.metricName].toString() log.info("this is number of data nodes $numDataNodes") val nodesMap = nodeStats["nodes"] as Map @@ -231,13 +231,14 @@ class ClusterMetricsCoordinator( val indexResponse: IndexResponse = client.suspendUntil { client.index(request, it) } val failureReasons = checkShardsFailure(indexResponse) if (failureReasons != null) { - log.info("Failed to index ${clusterMetricsDataPoint.metric}.", failureReasons) + log.error("Failed to index ${clusterMetricsDataPoint.metric}.", failureReasons) } } catch (t: Exception) { - log.info("Failed to index ${clusterMetricsDataPoint.metric}.", t) + log.error("Failed to index ${clusterMetricsDataPoint.metric}.", t) } } } + private fun checkShardsFailure(response: IndexResponse): String? { val failureReasons = StringBuilder() if (response.shardInfo.failed > 0) { diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt index 9e9f2c92a..cbaf24088 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/util/clustermetricscoordinatortest/ClusterMetricsCoordinatorIT.kt @@ -6,6 +6,7 @@ package org.opensearch.alerting.util.clustermetricscoordinatortest import org.apache.http.entity.ContentType +import org.apache.http.entity.ContentType.APPLICATION_JSON import org.apache.http.entity.StringEntity import org.junit.After import org.junit.Before @@ -13,6 +14,7 @@ import org.opensearch.alerting.AlertingRestTestCase import org.opensearch.alerting.makeRequest import org.opensearch.alerting.model.ClusterMetricsDataPoint import org.opensearch.alerting.opensearchapi.string +import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.util.ClusterMetricsVisualizationIndex import org.opensearch.client.Response import org.opensearch.client.ResponseException @@ -37,14 +39,14 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { assertEquals(storageTime, "10m") } - fun `test checkName`() { + fun `test check name of index`() { // WHEN + THEN, check whether the created index exists and has the name '.opendistro-alerting-cluster-metrics' val index = ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX val response = client().makeRequest("HEAD", index) assertEquals("Index $index does not exist.", RestStatus.OK, response.restStatus()) } - fun `test numberDocs`() { + fun `test check that number of documents is correct`() { // Check that the total number of documents found is divisible by the total number of metric types. val response = getResponse() val hits = getHits(response) @@ -70,7 +72,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { assertEquals(mapCheck.values.toSet().size, 1) } - fun `test deleteDocs`() { + fun `test deleting docs from index`() { val time = getTime() createDoc(time) Thread.sleep(60000) @@ -93,7 +95,7 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { assertTrue(flag) } - fun `test frequency`() { + fun `test execution frequency of job`() { client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "2m") Thread.sleep(300000) @@ -156,12 +158,48 @@ class ClusterMetricsCoordinatorIT : AlertingRestTestCase() { assertEquals(storageTime, "400m") } + fun `test simultaneously changing execution frequency and storage time where execution less than storage time`() { + val settings = jsonBuilder() + .startObject() + .startObject("persistent") + .field("plugins.alerting.cluster_metrics.execution_frequency", "5m") + .field("plugins.alerting.cluster_metrics.metrics_history_max_age", "10m") + .endObject() + .endObject() + .string() + val response = client().makeRequest("PUT", "_cluster/settings", StringEntity(settings, APPLICATION_JSON)) + assertEquals(RestStatus.OK, response.restStatus()) + } + fun `test simultaneously changing execution frequency and storage time where execution greater than storage time`() { + val settings = jsonBuilder() + .startObject() + .startObject("persistent") + .field("plugins.alerting.cluster_metrics.execution_frequency", "10m") + .field("plugins.alerting.cluster_metrics.metrics_history_max_age", "5m") + .endObject() + .endObject() + .string() + try { + client().makeRequest("PUT", "_cluster/settings", StringEntity(settings, APPLICATION_JSON)) + } catch (t: ResponseException) { + val responseMap = t.response.asMap() + val errMap = responseMap["error"] as Map + assertEquals("illegal_argument_exception", errMap["type"]) + } + } + @After // Reset the settings back to default, delete the created index. fun cleanup() { // reset settings - client().updateSettings("plugins.alerting.cluster_metrics.metrics_history_max_age", "7d") - client().updateSettings("plugins.alerting.cluster_metrics.execution_frequency", "15m") + client().updateSettings( + "plugins.alerting.cluster_metrics.metrics_history_max_age", + AlertingSettings.METRICS_STORE_TIME_DEFAULT_VALUE + ) + client().updateSettings( + "plugins.alerting.cluster_metrics.execution_frequency", + AlertingSettings.METRICS_EXECUTION_FREQUENCY_DEFAULT_VALUE + ) client().makeRequest("DELETE", ClusterMetricsVisualizationIndex.CLUSTER_METRIC_VISUALIZATION_INDEX) } private fun generateData() {