diff --git a/manifests/2.16.1/opensearch-2.16.1-test.yml b/manifests/2.16.1/opensearch-2.16.1-test.yml new file mode 100644 index 0000000000..c181699bad --- /dev/null +++ b/manifests/2.16.1/opensearch-2.16.1-test.yml @@ -0,0 +1,143 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 + args: -e JAVA_HOME=/opt/java/openjdk-21 +components: + - name: alerting + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + plugins.destination.host.deny_list: + - 10.0.0.0/8 + - 127.0.0.1 + bwc-test: + test-configs: + - with-security + - name: anomaly-detection + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - name: flow-framework + integ-test: + test-configs: + - with-security + - without-security + - name: asynchronous-search + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - name: cross-cluster-replication + integ-test: + topology: + - cluster_name: leader + data_nodes: 2 + - cluster_name: follower + data_nodes: 2 + test-configs: + - with-security + - without-security + - name: geospatial + integ-test: + test-configs: + - with-security + - without-security + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: + - /tmp + bwc-test: + test-configs: + - with-security + - name: k-NN + integ-test: + test-configs: + - with-security + - without-security + - name: ml-commons + integ-test: + test-configs: + - with-security + - without-security + - name: neural-search + integ-test: + test-configs: + - with-security + - without-security + - name: notifications + working-directory: notifications + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - name: opensearch-observability + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - name: opensearch-reports + integ-test: + test-configs: + - with-security + - without-security + - name: security + integ-test: + test-configs: + - with-security + - name: security-analytics + integ-test: + test-configs: + - with-security + - without-security + - name: sql + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + script.context.field.max_compilations_rate: 1000/1m + plugins.query.datasources.encryption.masterkey: 4fc8fee6a3fd7d6ca01772e5 + bwc-test: + test-configs: + - with-security + - name: custom-codecs + integ-test: + test-configs: + - with-security + - without-security + - name: skills + integ-test: + test-configs: + - with-security + - without-security + - name: query-insights + integ-test: + test-configs: + - with-security + - without-security diff --git a/manifests/3.0.0/opensearch-3.0.0-test.yml b/manifests/3.0.0/opensearch-3.0.0-test.yml index 6a63f55349..15006c19fe 100644 --- a/manifests/3.0.0/opensearch-3.0.0-test.yml +++ b/manifests/3.0.0/opensearch-3.0.0-test.yml @@ -3,7 +3,7 @@ schema-version: '1.0' name: OpenSearch ci: image: - name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 args: -e JAVA_HOME=/opt/java/openjdk-21 components: - name: alerting diff --git a/manifests/3.0.0/opensearch-dashboards-3.0.0-test.yml b/manifests/3.0.0/opensearch-dashboards-3.0.0-test.yml index 7fcfac61a0..f5b041c241 100644 --- a/manifests/3.0.0/opensearch-dashboards-3.0.0-test.yml +++ b/manifests/3.0.0/opensearch-dashboards-3.0.0-test.yml @@ -13,6 +13,12 @@ components: additional-cluster-configs: vis_builder.enabled: true data_source.enabled: true + savedObjects.maxImportPayloadBytes: 10485760 + server.maxPayloadBytes: 1759977 + logging.json: false + data.search.aggs.shardDelay.enabled: true + csp.warnLegacyBrowsers: false + ci-groups: 9 - name: alertingDashboards integ-test: test-configs: diff --git a/manifests/templates/opensearch-dashboards/1.x/manifest-test.yml b/manifests/templates/opensearch-dashboards/1.x/manifest-test.yml new file mode 100644 index 0000000000..f653b3fd30 --- /dev/null +++ b/manifests/templates/opensearch-dashboards/1.x/manifest-test.yml @@ -0,0 +1,12 @@ +--- +schema-version: '1.0' +name: OpenSearch Dashboards +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v4 +components: + - name: indexManagementDashboards + integ-test: + test-configs: + - with-security + - without-security diff --git a/manifests/templates/opensearch-dashboards/2.x/manifest-test.yml b/manifests/templates/opensearch-dashboards/2.x/manifest-test.yml new file mode 100644 index 0000000000..bc2310e9b1 --- /dev/null +++ b/manifests/templates/opensearch-dashboards/2.x/manifest-test.yml @@ -0,0 +1,26 @@ +--- +schema-version: '1.0' +name: OpenSearch Dashboards +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-almalinux8-opensearch-dashboards-integtest-v1 +components: + - name: OpenSearch-Dashboards + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + vis_builder.enabled: true + data_source.enabled: true + savedObjects.maxImportPayloadBytes: 10485760 + server.maxPayloadBytes: 1759977 + logging.json: false + data.search.aggs.shardDelay.enabled: true + csp.warnLegacyBrowsers: false + ci-groups: 9 + - name: indexManagementDashboards + integ-test: + test-configs: + - with-security + - without-security diff --git a/manifests/templates/opensearch-dashboards/3.x/manifest-test.yml b/manifests/templates/opensearch-dashboards/3.x/manifest-test.yml new file mode 100644 index 0000000000..bc2310e9b1 --- /dev/null +++ b/manifests/templates/opensearch-dashboards/3.x/manifest-test.yml @@ -0,0 +1,26 @@ +--- +schema-version: '1.0' +name: OpenSearch Dashboards +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-almalinux8-opensearch-dashboards-integtest-v1 +components: + - name: OpenSearch-Dashboards + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + vis_builder.enabled: true + data_source.enabled: true + savedObjects.maxImportPayloadBytes: 10485760 + server.maxPayloadBytes: 1759977 + logging.json: false + data.search.aggs.shardDelay.enabled: true + csp.warnLegacyBrowsers: false + ci-groups: 9 + - name: indexManagementDashboards + integ-test: + test-configs: + - with-security + - without-security diff --git a/manifests/templates/opensearch-dashboards/default/manifest-test.yml b/manifests/templates/opensearch-dashboards/default/manifest-test.yml new file mode 100644 index 0000000000..bc2310e9b1 --- /dev/null +++ b/manifests/templates/opensearch-dashboards/default/manifest-test.yml @@ -0,0 +1,26 @@ +--- +schema-version: '1.0' +name: OpenSearch Dashboards +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-almalinux8-opensearch-dashboards-integtest-v1 +components: + - name: OpenSearch-Dashboards + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + vis_builder.enabled: true + data_source.enabled: true + savedObjects.maxImportPayloadBytes: 10485760 + server.maxPayloadBytes: 1759977 + logging.json: false + data.search.aggs.shardDelay.enabled: true + csp.warnLegacyBrowsers: false + ci-groups: 9 + - name: indexManagementDashboards + integ-test: + test-configs: + - with-security + - without-security diff --git a/manifests/templates/opensearch/1.x/manifest-test.yml b/manifests/templates/opensearch/1.x/manifest-test.yml new file mode 100644 index 0000000000..17f2390a33 --- /dev/null +++ b/manifests/templates/opensearch/1.x/manifest-test.yml @@ -0,0 +1,20 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 + args: -e JAVA_HOME=/opt/java/openjdk-11 +components: + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + bwc-test: + test-configs: + - with-security diff --git a/manifests/templates/opensearch/2.x/manifest-test.yml b/manifests/templates/opensearch/2.x/manifest-test.yml new file mode 100644 index 0000000000..5c481e2d98 --- /dev/null +++ b/manifests/templates/opensearch/2.x/manifest-test.yml @@ -0,0 +1,20 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 + args: -e JAVA_HOME=/opt/java/openjdk-21 +components: + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + bwc-test: + test-configs: + - with-security diff --git a/manifests/templates/opensearch/2.x/manifest.yml b/manifests/templates/opensearch/2.x/manifest.yml index ce51bbae19..852db1a3df 100644 --- a/manifests/templates/opensearch/2.x/manifest.yml +++ b/manifests/templates/opensearch/2.x/manifest.yml @@ -5,7 +5,7 @@ build: version: 'replace' ci: image: - name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 args: -e JAVA_HOME=/opt/java/openjdk-21 components: - name: OpenSearch diff --git a/manifests/templates/opensearch/3.x/manifest-test.yml b/manifests/templates/opensearch/3.x/manifest-test.yml new file mode 100644 index 0000000000..5c481e2d98 --- /dev/null +++ b/manifests/templates/opensearch/3.x/manifest-test.yml @@ -0,0 +1,20 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 + args: -e JAVA_HOME=/opt/java/openjdk-21 +components: + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + bwc-test: + test-configs: + - with-security diff --git a/manifests/templates/opensearch/3.x/manifest.yml b/manifests/templates/opensearch/3.x/manifest.yml index ce51bbae19..852db1a3df 100644 --- a/manifests/templates/opensearch/3.x/manifest.yml +++ b/manifests/templates/opensearch/3.x/manifest.yml @@ -5,7 +5,7 @@ build: version: 'replace' ci: image: - name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 args: -e JAVA_HOME=/opt/java/openjdk-21 components: - name: OpenSearch diff --git a/manifests/templates/opensearch/default/manifest-test.yml b/manifests/templates/opensearch/default/manifest-test.yml new file mode 100644 index 0000000000..5c481e2d98 --- /dev/null +++ b/manifests/templates/opensearch/default/manifest-test.yml @@ -0,0 +1,20 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 + args: -e JAVA_HOME=/opt/java/openjdk-21 +components: + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + bwc-test: + test-configs: + - with-security diff --git a/manifests/templates/opensearch/default/manifest.yml b/manifests/templates/opensearch/default/manifest.yml index ce51bbae19..852db1a3df 100644 --- a/manifests/templates/opensearch/default/manifest.yml +++ b/manifests/templates/opensearch/default/manifest.yml @@ -5,7 +5,7 @@ build: version: 'replace' ci: image: - name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 args: -e JAVA_HOME=/opt/java/openjdk-21 components: - name: OpenSearch diff --git a/src/manifests_workflow/input_manifests.py b/src/manifests_workflow/input_manifests.py index 187fc200ee..9e6193bcb9 100644 --- a/src/manifests_workflow/input_manifests.py +++ b/src/manifests_workflow/input_manifests.py @@ -10,7 +10,7 @@ import os import re from abc import abstractmethod -from typing import Dict, List, Type, Union +from typing import Dict, List, Tuple, Type, Union import ruamel.yaml from packaging.version import Version @@ -18,6 +18,7 @@ from manifests.input_manifest import InputManifest from manifests.manifests import Manifests +from manifests.test_manifest import TestManifest from manifests_workflow.component_opensearch import ComponentOpenSearch from manifests_workflow.component_opensearch_dashboards_min import ComponentOpenSearchDashboardsMin from manifests_workflow.component_opensearch_min import ComponentOpenSearchMin @@ -122,7 +123,7 @@ def update( self.add_to_versionincrement_workflow(new_version_entry) known_versions.append(new_version_entry) - def create_manifest(self, version: str, branch: str, known_versions: List[str]) -> InputManifest: + def create_manifest(self, version: str, branch: str, known_versions: List[str]) -> Tuple[InputManifest, TestManifest]: # If : No known_versions manifests exist or new version smaller than the min(known_versions), create new manifests from the templates # (1.0.0-3.0.0 based on template 1.x-3.x, 4.0.0+ from default.x, previous behavior) # Else: Create new manifests based on the latest version before the new version @@ -132,35 +133,46 @@ def create_manifest(self, version: str, branch: str, known_versions: List[str]) templates_base_path = os.path.join(self.manifests_path(), "templates") template_version_folder = version.split(".")[0] + ".x" template_full_path = os.path.join(templates_base_path, self.prefix, template_version_folder, "manifest.yml") + template_test_full_path = os.path.join(templates_base_path, self.prefix, template_version_folder, "manifest-test.yml") if not os.path.exists(template_full_path): template_full_path = os.path.join(templates_base_path, self.prefix, "default", "manifest.yml") + template_test_full_path = os.path.join(templates_base_path, self.prefix, "default", "manifest-test.yml") else: previous_versions = [v for v in known_versions if Version(v) < Version(version)] base_version = max(previous_versions, key=version_parse) logging.info(f"Base Version: {base_version} is the highest version before {version}") template_full_path = os.path.join(self.manifests_path(), base_version, f"{self.prefix}-{base_version}.yml") + template_test_full_path = os.path.join(self.manifests_path(), base_version, f"{self.prefix}-{base_version}-test.yml") if not os.path.exists(template_full_path): template_full_path = os.path.join(self.legacy_manifests_path(), base_version, f"{self.prefix}-{base_version}.yml") + template_test_full_path = os.path.join(self.legacy_manifests_path(), base_version, f"{self.prefix}-{base_version}-test.yml") - logging.info(f"Using {template_full_path} as the base manifest") - + # Input Manifest + logging.info(f"Using {template_full_path} as the base input manifest") manifest = InputManifest.from_file(open(template_full_path)) - manifest.build.version = version - for component in manifest.components.select(): component.ref = branch # type: ignore - return manifest + # Test Manifest + logging.info(f"Using {template_test_full_path} as the base test manifest") + manifest_test = TestManifest.from_file(open(template_test_full_path)) + + return (manifest, manifest_test) def write_manifest(self, version: str, branch: str, known_versions: List[str]) -> None: logging.info(f"Creating new version: {version}") - manifest = self.create_manifest(version, branch, known_versions) + manifests = self.create_manifest(version, branch, known_versions) manifest_dir = os.path.join(self.manifests_path(), version) os.makedirs(manifest_dir, exist_ok=True) - manifest_path = os.path.join(manifest_dir, f"{self.prefix}-{version}.yml") - manifest.to_file(manifest_path) - logging.info(f"Wrote {manifest_path} as the new manifest") + for manifest in manifests: + if manifest.__class__.__name__ == 'TestManifest': + manifest_path = os.path.join(manifest_dir, f"{self.prefix}-{version}-test.yml") + logging.info(f"Wrote {manifest_path} as the new test manifest") + else: + manifest_path = os.path.join(manifest_dir, f"{self.prefix}-{version}.yml") + logging.info(f"Wrote {manifest_path} as the new input manifest") + manifest.to_file(manifest_path) # type: ignore[attr-defined] def add_to_cron(self, version: str) -> None: logging.info(f"Adding new version to cron: {version}") diff --git a/tests/tests_manifests_workflow/data/opensearch-2.12.1000-test.yml b/tests/tests_manifests_workflow/data/opensearch-2.12.1000-test.yml new file mode 100644 index 0000000000..f9ffe22604 --- /dev/null +++ b/tests/tests_manifests_workflow/data/opensearch-2.12.1000-test.yml @@ -0,0 +1,160 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 + args: -e JAVA_HOME=/opt/java/openjdk-21 +components: + - name: alerting + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + plugins.destination.host.deny_list: [10.0.0.0/8, 127.0.0.1] + bwc-test: + test-configs: + - with-security + + - name: anomaly-detection + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + + - name: flow-framework + integ-test: + test-configs: + - with-security + - without-security + + - name: asynchronous-search + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + + - name: cross-cluster-replication + integ-test: + topology: + - cluster_name: leader + data_nodes: 2 + cluster_manager_nodes: 0 + - cluster_name: follower + data_nodes: 2 + cluster_manager_nodes: 0 + test-configs: + - with-security + - without-security + + - name: geospatial + integ-test: + test-configs: + - with-security + - without-security + + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + bwc-test: + test-configs: + - with-security + + - name: k-NN + integ-test: + test-configs: + - with-security + - without-security + + - name: ml-commons + integ-test: + test-configs: + - with-security + - without-security + + - name: neural-search + integ-test: + test-configs: + - with-security + - without-security + + - name: notifications + working-directory: notifications + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + + - name: opensearch-observability + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + + - name: opensearch-reports + integ-test: + test-configs: + - with-security + - without-security + + - name: security + integ-test: + test-configs: + - with-security + + - name: security-analytics + integ-test: + test-configs: + - with-security + - without-security + + - name: sql + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + script.context.field.max_compilations_rate: 1000/1m + plugins.query.datasources.encryption.masterkey: 4fc8fee6a3fd7d6ca01772e5 + bwc-test: + test-configs: + - with-security + + - name: custom-codecs + integ-test: + test-configs: + - with-security + - without-security + + - name: skills + integ-test: + test-configs: + - with-security + - without-security + + - name: query-insights + integ-test: + test-configs: + - with-security + - without-security diff --git a/tests/tests_manifests_workflow/data/opensearch-dashboards-2.12.1000-test.yml b/tests/tests_manifests_workflow/data/opensearch-dashboards-2.12.1000-test.yml new file mode 100644 index 0000000000..d1a4216bd2 --- /dev/null +++ b/tests/tests_manifests_workflow/data/opensearch-dashboards-2.12.1000-test.yml @@ -0,0 +1,95 @@ +--- +schema-version: '1.0' +name: OpenSearch Dashboards +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v4 +components: + - name: OpenSearch-Dashboards + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + vis_builder.enabled: true + data_source.enabled: true + savedObjects.maxImportPayloadBytes: 10485760 + server.maxPayloadBytes: 1759977 + logging.json: false + data.search.aggs.shardDelay.enabled: true + csp.warnLegacyBrowsers: false + ci-groups: 9 + - name: alertingDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: anomalyDetectionDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: ganttChartDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: indexManagementDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: observabilityDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: queryWorkbenchDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: reportsDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: securityDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: notificationsDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: customImportMapDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: searchRelevanceDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: securityAnalyticsDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: mlCommonsDashboards + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + ml_commons_dashboards.enabled: true + - name: assistantDashboards + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + assistant.chat.enabled: true diff --git a/tests/tests_manifests_workflow/test_input_manifests.py b/tests/tests_manifests_workflow/test_input_manifests.py index be5aad53fa..75ed969516 100644 --- a/tests/tests_manifests_workflow/test_input_manifests.py +++ b/tests/tests_manifests_workflow/test_input_manifests.py @@ -10,10 +10,12 @@ from unittest.mock import MagicMock, call, mock_open, patch from manifests.input_manifest import InputManifest +from manifests.test_manifest import TestManifest from manifests_workflow.input_manifests import InputManifests class TestInputManifests(unittest.TestCase): + def test_manifests_path(self) -> None: path = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..", "manifests")) self.assertEqual(path, InputManifests.manifests_path()) @@ -44,7 +46,7 @@ def test_create_manifest_opensearch_template(self) -> None: input_manifests = InputManifests("OpenSearch") input_manifest = input_manifests.create_manifest("1.2.3", "1.x", []) self.assertEqual( - input_manifest.to_dict(), + input_manifest[0].to_dict(), { "schema-version": "1.0", "build": {"name": "OpenSearch", "version": "1.2.3"}, @@ -56,16 +58,29 @@ def test_create_manifest_opensearch_template(self) -> None: "checks": ["gradle:publish", "gradle:properties:version"]}] } ) + self.assertEqual( + input_manifest[1].to_dict(), + { + 'schema-version': '1.0', + 'name': 'OpenSearch', + 'ci': {'image': {'name': 'opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3', + 'args': '-e JAVA_HOME=/opt/java/openjdk-11'}}, + 'components': [{'name': 'index-management', + 'integ-test': {'build-dependencies': ['job-scheduler'], + 'test-configs': ['with-security', 'without-security'], + 'additional-cluster-configs': {'path.repo': ['/tmp']}}, 'bwc-test': {'test-configs': ['with-security']}}] + } + ) def test_create_manifest_opensearch_default_template(self) -> None: input_manifests = InputManifests("OpenSearch") input_manifest = input_manifests.create_manifest("0.2.3", "0.x", []) self.assertEqual( - input_manifest.to_dict(), + input_manifest[0].to_dict(), { "schema-version": "1.1", "build": {"name": "OpenSearch", "version": "0.2.3"}, - "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3", + "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1", "args": "-e JAVA_HOME=/opt/java/openjdk-21"}}, "components": [{"name": "OpenSearch", "repository": "https://github.com/opensearch-project/OpenSearch.git", @@ -73,19 +88,32 @@ def test_create_manifest_opensearch_default_template(self) -> None: "checks": ["gradle:publish", "gradle:properties:version"]}] } ) + self.assertEqual( + input_manifest[1].to_dict(), + { + 'schema-version': '1.0', + 'name': 'OpenSearch', + 'ci': {'image': {'name': 'opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1', + 'args': '-e JAVA_HOME=/opt/java/openjdk-21'}}, + 'components': [{'name': 'index-management', + 'integ-test': {'build-dependencies': ['job-scheduler'], + 'test-configs': ['with-security', 'without-security'], + 'additional-cluster-configs': {'path.repo': ['/tmp']}}, 'bwc-test': {'test-configs': ['with-security']}}] + } + ) def test_create_manifest_opensearch_previous_base_version(self) -> None: input_manifests = InputManifests("OpenSearch") input_manifest = input_manifests.create_manifest("2.12.1000", "2.12", ["0.9.2", "1.3.1", "1.3.14", "2.12.0", "2.14.0", "3.0.0"]) # based on 2.12.0 manifest_path = os.path.join(os.path.dirname(__file__), "data", "opensearch-2.12.1000.yml") input_manifest_compare = InputManifest.from_file(open(manifest_path)) - self.assertEqual(input_manifest.to_dict(), input_manifest_compare.to_dict()) + self.assertEqual(input_manifest[0].to_dict(), input_manifest_compare.to_dict()) def test_create_manifest_opensearch_dashboards_template(self) -> None: input_manifests = InputManifests("OpenSearch Dashboards") input_manifest = input_manifests.create_manifest("1.2.3", "1.x", []) self.assertEqual( - input_manifest.to_dict(), + input_manifest[0].to_dict(), { "schema-version": "1.0", "build": {"name": "OpenSearch Dashboards", "version": "1.2.3"}, @@ -95,12 +123,22 @@ def test_create_manifest_opensearch_dashboards_template(self) -> None: "ref": "1.x"}] } ) + self.assertEqual( + input_manifest[1].to_dict(), + { + 'schema-version': '1.0', + 'name': 'OpenSearch Dashboards', + 'ci': {'image': {'name': 'opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v4'}}, + 'components': [{'name': 'indexManagementDashboards', + 'integ-test': {'test-configs': ['with-security', 'without-security']}}] + } + ) def test_create_manifest_opensearch_dashboards_default_template(self) -> None: input_manifests = InputManifests("OpenSearch Dashboards") input_manifest = input_manifests.create_manifest("4.2.3", "4.x", []) self.assertEqual( - input_manifest.to_dict(), + input_manifest[0].to_dict(), { "schema-version": "1.1", "build": {"name": "OpenSearch Dashboards", "version": "4.2.3"}, @@ -111,23 +149,52 @@ def test_create_manifest_opensearch_dashboards_default_template(self) -> None: "checks": ["npm:package:version"]}] } ) + self.assertEqual( + input_manifest[1].to_dict(), + { + 'schema-version': '1.0', + 'name': 'OpenSearch Dashboards', + 'ci': {'image': {'name': 'opensearchstaging/ci-runner:ci-runner-almalinux8-opensearch-dashboards-integtest-v1'}}, + 'components': [{'name': 'OpenSearch-Dashboards', + 'integ-test': {'test-configs': ['with-security', 'without-security'], + 'additional-cluster-configs': {'vis_builder.enabled': True, + 'data_source.enabled': True, + 'savedObjects.maxImportPayloadBytes': 10485760, + 'server.maxPayloadBytes': 1759977, + 'logging.json': False, + 'data.search.aggs.shardDelay.enabled': True, + 'csp.warnLegacyBrowsers': False}, + 'ci-groups': 9}}, + {'name': 'indexManagementDashboards', + 'integ-test': {'test-configs': ['with-security', 'without-security']}}] + } + ) def test_create_manifest_opensearch_dashboards_previous_base_version(self) -> None: input_manifests = InputManifests("OpenSearch-Dashboards") input_manifest = input_manifests.create_manifest("2.12.1000", "2.12", ["0.9.2", "1.3.1", "1.3.14", "2.12.0", "2.14.0", "3.0.0"]) # based on 2.12.0 manifest_path = os.path.join(os.path.dirname(__file__), "data", "opensearch-dashboards-2.12.1000.yml") input_manifest_compare = InputManifest.from_file(open(manifest_path)) - self.assertEqual(input_manifest.to_dict(), input_manifest_compare.to_dict()) + self.assertEqual(input_manifest[0].to_dict(), input_manifest_compare.to_dict()) + @patch("manifests.manifest.Manifest.to_file") @patch("os.makedirs") @patch("manifests_workflow.input_manifests.InputManifests.create_manifest") - def test_write_manifest(self, mock_create_manifest: MagicMock, mock_makedirs: MagicMock) -> None: + def test_write_manifest(self, mock_create_manifest: MagicMock, mock_makedirs: MagicMock, mock_to_file: MagicMock) -> None: + input_manifest = InputManifest.from_file(open(os.path.join(os.path.dirname(__file__), "data", "opensearch-2.12.1000.yml"))) + test_manifest = TestManifest.from_file(open(os.path.join(os.path.dirname(__file__), "data", "opensearch-2.12.1000-test.yml"))) + mock_create_manifest.return_value = (input_manifest, test_manifest) input_manifests = InputManifests("opensearch") - input_manifests.write_manifest('0.1.2', '0.x', []) - mock_create_manifest.assert_called_with('0.1.2', '0.x', []) - mock_makedirs.assert_called_with(os.path.join(InputManifests.manifests_path(), '0.1.2'), exist_ok=True) - mock_create_manifest.return_value.to_file.assert_called_with( - os.path.join(InputManifests.manifests_path(), '0.1.2', 'opensearch-0.1.2.yml') + input_manifests.write_manifest('2.12.1000', '2.x', []) + mock_create_manifest.assert_called_with('2.12.1000', '2.x', []) + mock_makedirs.assert_called_with(os.path.join(InputManifests.manifests_path(), '2.12.1000'), exist_ok=True) + self.assertEqual( + mock_to_file.call_args_list[0][0][0], + os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..", "manifests", "2.12.1000", "opensearch-2.12.1000.yml")) + ) + self.assertEqual( + mock_to_file.call_args_list[1][0][0], + os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..", "manifests", "2.12.1000", "opensearch-2.12.1000-test.yml")) ) def test_jenkins_path(self) -> None: diff --git a/tests/tests_manifests_workflow/test_input_manifests_opensearch.py b/tests/tests_manifests_workflow/test_input_manifests_opensearch.py index fed4f7aa02..b3386bab85 100644 --- a/tests/tests_manifests_workflow/test_input_manifests_opensearch.py +++ b/tests/tests_manifests_workflow/test_input_manifests_opensearch.py @@ -42,7 +42,7 @@ def test_update(self, mock_component_opensearch_min: MagicMock, mock_manifest_to mock_component_opensearch_min.checkout.return_value = MagicMock(version="2.1000.1000") manifests = InputManifestsOpenSearch() manifests.update() - self.assertEqual(mock_manifest_to_file.call_count, 1) + self.assertEqual(mock_manifest_to_file.call_count, 2) calls = [ call( os.path.join( diff --git a/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py b/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py index 327d02841d..395c4c25bd 100644 --- a/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py +++ b/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py @@ -43,7 +43,7 @@ def test_update(self, mock_component_opensearch_dashboards_min: MagicMock, mock_ manifests = InputManifestsOpenSearchDashboards() manifests.update() - self.assertEqual(mock_manifest_to_file.call_count, 1) + self.assertEqual(mock_manifest_to_file.call_count, 2) calls = [ call( os.path.join(