From f7c5e944f32decf088812e456513c1b86f682213 Mon Sep 17 00:00:00 2001 From: travisbenedict Date: Mon, 18 Apr 2022 18:27:16 +0000 Subject: [PATCH 1/2] Change bulk operation for metrics and index templates to support OpenSearch 2.0 as a data store Signed-off-by: travisbenedict --- osbenchmark/metrics.py | 20 +- osbenchmark/resources/metrics-template.json | 160 ++++++++------- osbenchmark/resources/results-template.json | 184 +++++++++--------- .../resources/test-executions-template.json | 114 ++++++----- 4 files changed, 229 insertions(+), 249 deletions(-) diff --git a/osbenchmark/metrics.py b/osbenchmark/metrics.py index 7c651120b..d4ed17e0b 100644 --- a/osbenchmark/metrics.py +++ b/osbenchmark/metrics.py @@ -27,6 +27,7 @@ import json import logging import math +import opensearchpy.helpers import os import pickle import random @@ -54,7 +55,6 @@ def __init__(self, client, cluster_version=None): self._cluster_version = cluster_version self._cluster_distribution = None - # TODO #653: Remove version-specific support for metrics stores before 7.0.0. def probe_version(self): info = self.guarded(self._client.info) try: @@ -72,15 +72,7 @@ def probe_version(self): self._cluster_distribution = "elasticsearch" def put_template(self, name, template): - # TODO #653: Remove version-specific support for metrics stores before 7.0.0 (also adjust template) - if (self._cluster_version[0] > 6 and self._cluster_distribution == "elasticsearch") or \ - self._cluster_distribution == "opensearch": - return self.guarded(self._client.indices.put_template, name=name, body=template, params={ - # allows to include the type name although it is not allowed anymore by default - "include_type_name": "true" - }) - else: - return self.guarded(self._client.indices.put_template, name=name, body=template) + return self.guarded(self._client.indices.put_template, name=name, body=template) def template_exists(self, name): return self.guarded(self._client.indices.exists_template, name) @@ -102,13 +94,7 @@ def refresh(self, index): return self.guarded(self._client.indices.refresh, index=index) def bulk_index(self, index, doc_type, items): - # TODO #653: Remove version-specific support for metrics stores before 7.0.0. - # pylint: disable=import-outside-toplevel - import opensearchpy.helpers - if self._cluster_version[0] > 6: - self.guarded(opensearchpy.helpers.bulk, self._client, items, index=index, chunk_size=5000) - else: - self.guarded(opensearchpy.helpers.bulk, self._client, items, index=index, doc_type=doc_type, chunk_size=5000) + self.guarded(opensearchpy.helpers.bulk, self._client, items, index=index, chunk_size=5000) def index(self, index, doc_type, item, id=None): doc = { diff --git a/osbenchmark/resources/metrics-template.json b/osbenchmark/resources/metrics-template.json index d9d797def..503e75bb3 100644 --- a/osbenchmark/resources/metrics-template.json +++ b/osbenchmark/resources/metrics-template.json @@ -9,91 +9,89 @@ } }, "mappings": { - "_doc": { - "date_detection": false, - "dynamic_templates": [ - { - "strings": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "keyword" - } + "date_detection": false, + "dynamic_templates": [ + { + "strings": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "keyword" } } - ], - "_source": { - "enabled": true - }, - "properties": { - "@timestamp": { - "type": "date", - "format": "epoch_millis" - }, - "relative-time-ms": { - "type": "float" - }, - "test-execution-id": { - "type": "keyword" - }, - "test-execution-timestamp": { - "type": "date", - "format": "basic_date_time_no_millis", - "fields": { - "raw": { - "type": "keyword" - } + } + ], + "_source": { + "enabled": true + }, + "properties": { + "@timestamp": { + "type": "date", + "format": "epoch_millis" + }, + "relative-time-ms": { + "type": "float" + }, + "test-execution-id": { + "type": "keyword" + }, + "test-execution-timestamp": { + "type": "date", + "format": "basic_date_time_no_millis", + "fields": { + "raw": { + "type": "keyword" } - }, - "environment": { - "type": "keyword" - }, - "workload": { - "type": "keyword" - }, - "test_procedure": { - "type": "keyword" - }, - "provision-config-instance": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "value": { - "type": "float" - }, - "min": { - "type": "float" - }, - "max": { - "type": "float" - }, - "mean": { - "type": "float" - }, - "median": { - "type": "float" - }, - "unit": { - "type": "keyword" - }, - "sample-type": { - "type": "keyword" - }, - "task": { - "type": "keyword" - }, - "operation": { - "type": "keyword" - }, - "operation-type": { - "type": "keyword" - }, - "job": { - "type": "keyword" } + }, + "environment": { + "type": "keyword" + }, + "workload": { + "type": "keyword" + }, + "test_procedure": { + "type": "keyword" + }, + "provision-config-instance": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "value": { + "type": "float" + }, + "min": { + "type": "float" + }, + "max": { + "type": "float" + }, + "mean": { + "type": "float" + }, + "median": { + "type": "float" + }, + "unit": { + "type": "keyword" + }, + "sample-type": { + "type": "keyword" + }, + "task": { + "type": "keyword" + }, + "operation": { + "type": "keyword" + }, + "operation-type": { + "type": "keyword" + }, + "job": { + "type": "keyword" } } } -} \ No newline at end of file +} diff --git a/osbenchmark/resources/results-template.json b/osbenchmark/resources/results-template.json index e022128d9..b108b9b8e 100644 --- a/osbenchmark/resources/results-template.json +++ b/osbenchmark/resources/results-template.json @@ -9,104 +9,102 @@ } }, "mappings": { - "_doc": { - "date_detection": false, - "dynamic_templates": [ - { - "strings": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "keyword" - } + "date_detection": false, + "dynamic_templates": [ + { + "strings": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "keyword" } } - ], - "_source": { - "enabled": true - }, - "properties": { - "test-execution-id": { - "type": "keyword" - }, - "test-execution-timestamp": { - "type": "date", - "format": "basic_date_time_no_millis", - "fields": { - "raw": { - "type": "keyword" - } + } + ], + "_source": { + "enabled": true + }, + "properties": { + "test-execution-id": { + "type": "keyword" + }, + "test-execution-timestamp": { + "type": "date", + "format": "basic_date_time_no_millis", + "fields": { + "raw": { + "type": "keyword" } - }, - "active": { - "type": "boolean" - }, - "benchmark-version": { - "type": "keyword" - }, - "benchmark-revision": { - "type": "keyword" - }, - "environment": { - "type": "keyword" - }, - "workload": { - "type": "keyword" - }, - "test_procedure": { - "type": "keyword" - }, - "provision-config-instance": { - "type": "keyword" - }, - "node-count": { - "type": "short" - }, - "plugins": { - "type": "keyword" - }, - "distribution-flavor": { - "type": "keyword" - }, - "distribution-version": { - "type": "keyword" - }, - "distribution-major-version": { - "type": "short" - }, - "task": { - "type": "keyword" - }, - "operation": { - "type": "keyword" - }, - "job": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "value": { - "type": "object", - "properties": { - "single": { - "type": "double" - }, - "min": { - "type": "double" - }, - "mean": { - "type": "double" - }, - "median": { - "type": "double" - }, - "max": { - "type": "double" - } + } + }, + "active": { + "type": "boolean" + }, + "benchmark-version": { + "type": "keyword" + }, + "benchmark-revision": { + "type": "keyword" + }, + "environment": { + "type": "keyword" + }, + "workload": { + "type": "keyword" + }, + "test_procedure": { + "type": "keyword" + }, + "provision-config-instance": { + "type": "keyword" + }, + "node-count": { + "type": "short" + }, + "plugins": { + "type": "keyword" + }, + "distribution-flavor": { + "type": "keyword" + }, + "distribution-version": { + "type": "keyword" + }, + "distribution-major-version": { + "type": "short" + }, + "task": { + "type": "keyword" + }, + "operation": { + "type": "keyword" + }, + "job": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "value": { + "type": "object", + "properties": { + "single": { + "type": "double" + }, + "min": { + "type": "double" + }, + "mean": { + "type": "double" + }, + "median": { + "type": "double" + }, + "max": { + "type": "double" } } } } } -} \ No newline at end of file +} diff --git a/osbenchmark/resources/test-executions-template.json b/osbenchmark/resources/test-executions-template.json index 140d0fd75..240d166dd 100644 --- a/osbenchmark/resources/test-executions-template.json +++ b/osbenchmark/resources/test-executions-template.json @@ -9,70 +9,68 @@ } }, "mappings": { - "_doc": { - "date_detection": false, - "dynamic_templates": [ - { - "strings": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "keyword" - } + "date_detection": false, + "dynamic_templates": [ + { + "strings": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "keyword" } } - ], - "_source": { - "enabled": true + } + ], + "_source": { + "enabled": true + }, + "properties": { + "test-execution-id": { + "type": "keyword" }, - "properties": { - "test-execution-id": { - "type": "keyword" - }, - "test-execution-timestamp": { - "type": "date", - "format": "basic_date_time_no_millis", - "fields": { - "raw": { - "type": "keyword" - } + "test-execution-timestamp": { + "type": "date", + "format": "basic_date_time_no_millis", + "fields": { + "raw": { + "type": "keyword" } - }, - "benchmark-version": { - "type": "keyword" - }, - "benchmark-revision": { - "type": "keyword" - }, - "environment": { - "type": "keyword" - }, - "pipeline": { - "type": "keyword" - }, - "workload": { - "type": "keyword" - }, - "test_procedure": { - "type": "keyword" - }, - "provision-config-instance": { - "type": "keyword" - }, - "node-count": { - "type": "short" - }, - "plugins": { - "type": "keyword" - }, - "results": { - "properties": { - "op_metrics": { - "type": "nested" - } + } + }, + "benchmark-version": { + "type": "keyword" + }, + "benchmark-revision": { + "type": "keyword" + }, + "environment": { + "type": "keyword" + }, + "pipeline": { + "type": "keyword" + }, + "workload": { + "type": "keyword" + }, + "test_procedure": { + "type": "keyword" + }, + "provision-config-instance": { + "type": "keyword" + }, + "node-count": { + "type": "short" + }, + "plugins": { + "type": "keyword" + }, + "results": { + "properties": { + "op_metrics": { + "type": "nested" } } } } } -} \ No newline at end of file +} From e8c0bcecb8db52ec87f57c2d9ebb9b4e16c99d84 Mon Sep 17 00:00:00 2001 From: travisbenedict Date: Mon, 18 Apr 2022 22:35:47 +0000 Subject: [PATCH 2/2] Fix import for linter Signed-off-by: travisbenedict --- osbenchmark/metrics.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/osbenchmark/metrics.py b/osbenchmark/metrics.py index d4ed17e0b..7cf66cca2 100644 --- a/osbenchmark/metrics.py +++ b/osbenchmark/metrics.py @@ -27,7 +27,6 @@ import json import logging import math -import opensearchpy.helpers import os import pickle import random @@ -37,7 +36,7 @@ import zlib from enum import Enum, IntEnum from http.client import responses - +import opensearchpy.helpers import tabulate from osbenchmark import client, time, exceptions, config, version, paths