From c93dc735539284db67158397eb8d8dd06dacd4d7 Mon Sep 17 00:00:00 2001 From: Zelin Hao Date: Wed, 2 Aug 2023 14:03:33 -0700 Subject: [PATCH 1/3] Modify the jenkins job to trigger reporting Signed-off-by: Zelin Hao --- .../integ-test.jenkinsfile | 36 +++++++++++-------- jenkins/opensearch/integ-test.jenkinsfile | 7 ++++ 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/jenkins/opensearch-dashboards/integ-test.jenkinsfile b/jenkins/opensearch-dashboards/integ-test.jenkinsfile index a7280e73c9..15f3ce2257 100644 --- a/jenkins/opensearch-dashboards/integ-test.jenkinsfile +++ b/jenkins/opensearch-dashboards/integ-test.jenkinsfile @@ -1,6 +1,6 @@ -lib = library(identifier: 'jenkins@5.4.1', retriever: modernSCM([ +lib = library(identifier: 'jenkins@jenkins_report', retriever: modernSCM([ $class: 'GitSCMSource', - remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', + remote: 'https://github.com/zelinh/opensearch-build-libraries.git', ])) def docker_images = [ @@ -198,22 +198,22 @@ pipeline { switchUserNonRoot: "${switch_user_non_root}" ) String closeCommentMessage = "Closing the issue as the Integration Test passed for ${local_component}
Version: ${version}
Distribution: ${distribution}
Architecture: ${architecture}
Platform: ${platform}

Please check the logs: ${RUN_DISPLAY_URL}

*" - closeGithubIssue( - repoUrl: buildManifestObj.getRepo("${local_component}"), - issueTitle: "[AUTOCUT] Integration Test failed for ${local_component}: ${version} ${distribution} distribution", - closeComment: closeCommentMessage, - label: "autocut,v${version},integ-test-failure" - ) +// closeGithubIssue( +// repoUrl: buildManifestObj.getRepo("${local_component}"), +// issueTitle: "[AUTOCUT] Integration Test failed for ${local_component}: ${version} ${distribution} distribution", +// closeComment: closeCommentMessage, +// label: "autocut,v${version},integ-test-failure" +// ) } } catch (e) { echo "Error running integtest for component ${local_component}, creating Github issue" String issueBodyMessage = "The integration test failed at distribution level for component ${local_component}
Version: ${version}
Distribution: ${distribution}
Architecture: ${architecture}
Platform: ${platform}

Please check the logs: ${RUN_DISPLAY_URL}

* Steps to reproduce: See https://github.com/opensearch-project/opensearch-build/tree/main/src/test_workflow#integration-tests
* See all log files:
- [With security](https://ci.opensearch.org/ci/dbc/${JOB_NAME}/${version}/${buildId}/${platform}/${architecture}/${distribution}/test-results/${BUILD_NUMBER}/integ-test/${local_component}/with-security/${local_component}.yml) (if applicable)
- [Without security](https://ci.opensearch.org/ci/dbc/${JOB_NAME}/${version}/${buildId}/${platform}/${architecture}/${distribution}/test-results/${BUILD_NUMBER}/integ-test/${local_component}/without-security/${local_component}.yml) (if applicable)

" - createGithubIssue( - repoUrl: buildManifestObj.getRepo("${local_component}"), - issueTitle: "[AUTOCUT] Integration Test failed for ${local_component}: ${version} ${distribution} distribution", - issueBody: issueBodyMessage, - label: "autocut,v${version},integ-test-failure" - ) +// createGithubIssue( +// repoUrl: buildManifestObj.getRepo("${local_component}"), +// issueTitle: "[AUTOCUT] Integration Test failed for ${local_component}: ${version} ${distribution} distribution", +// issueBody: issueBodyMessage, +// label: "autocut,v${version},integ-test-failure" +// ) throw new Exception("Error running integtest for component ${local_component}", e) } finally { echo "Completed running integtest for component ${local_component}" @@ -234,6 +234,14 @@ pipeline { } post { always { + createTestRunManifest( + testManifest: "manifests/${TEST_MANIFEST}", + buildManifest: BUILD_MANIFEST_OPENSEARCH, + buildManifestDashboards: BUILD_MANIFEST, + testRunID: "${env.BUILD_NUMBER}", + testType: "integ-test", + ) + sh("cat test-run.yml") postCleanup() } } diff --git a/jenkins/opensearch/integ-test.jenkinsfile b/jenkins/opensearch/integ-test.jenkinsfile index bcd6af52d2..920a931c78 100644 --- a/jenkins/opensearch/integ-test.jenkinsfile +++ b/jenkins/opensearch/integ-test.jenkinsfile @@ -198,6 +198,13 @@ pipeline { } post { always { + createTestRunManifest( + testManifest: "manifests/${TEST_MANIFEST}", + buildManifest: BUILD_MANIFEST, + testRunID: "${env.BUILD_NUMBER}", + testType: "integ-test", + ) + sh("cat test-run.yml") postCleanup() } } From ad2d49708190ec7374be4824370d78e4cf75e0ad Mon Sep 17 00:00:00 2001 From: Zelin Hao Date: Tue, 22 Aug 2023 17:12:41 -0700 Subject: [PATCH 2/3] Add post stage to create test report manifest Signed-off-by: Zelin Hao --- .../component_release_template.md | 1 + .github/ISSUE_TEMPLATE/release_template.md | 162 ++++-- .github/workflows/link-checker.yml | 5 +- Pipfile | 2 +- Pipfile.lock | 7 +- README.md | 70 +-- RELEASE_PROCESS_OPENSEARCH.md | 543 ++++++++++++++++++ assests/distribution_build_os.png | Bin 0 -> 122341 bytes docker/ci/config/windows-servercore-setup.ps1 | 227 ++++++++ ...opensearch-dashboards.x64.arm64.dockerfile | 103 ++++ ...ockylinux8.opensearch.x64.arm64.dockerfile | 4 +- ...uild.windows2019.servercore.x64.dockerfile | 31 + ...lease.centos7.clients.x64.arm64.dockerfile | 14 +- ...opensearch-dashboards.x64.arm64.dockerfile | 0 ...ld.centos7.opensearch.x64.arm64.dockerfile | 2 +- ...tos7.performance-test.x64.arm64.dockerfile | 0 jenkins/check-for-build.jenkinsfile | 8 +- .../perf-test.jenkinsfile | 10 +- jenkins/docker/docker-build.jenkinsfile | 28 +- jenkins/gradle/gradle-check.jenkinsfile | 8 +- ...RunNonSecurityPerfTestScript.groovy.legacy | 0 .../perf-test.jenkinsfile | 26 +- .../bwc-test.jenkinsfile | 1 + .../distribution-build.jenkinsfile | 74 +-- .../integ-test.jenkinsfile | 58 +- jenkins/opensearch/benchmark-test.jenkinsfile | 144 +++-- jenkins/opensearch/bwc-test.jenkinsfile | 1 + .../opensearch/distribution-build.jenkinsfile | 75 +-- jenkins/opensearch/integ-test.jenkinsfile | 32 +- .../maven-publish-1.3.x.jenkinsfile | 2 +- manifests/1.3.12/opensearch-1.3.12-test.yml | 69 +++ manifests/1.3.12/opensearch-1.3.12.yml | 127 +++- .../opensearch-dashboards-1.3.12-test.yml | 52 ++ .../1.3.12/opensearch-dashboards-1.3.12.yml | 40 ++ manifests/1.3.13/opensearch-1.3.13.yml | 16 + ...ensearch-2.10.0-concurrent-search-test.yml | 189 ++++++ manifests/2.10.0/opensearch-2.10.0-test.yml | 133 +++++ .../opensearch-dashboards-2.10.0-test.yml | 82 +++ .../2.10.0/opensearch-dashboards-2.10.0.yml | 43 ++ manifests/3.0.0/opensearch-3.0.0-test.yml | 113 ++++ manifests/3.0.0/opensearch-3.0.0.yml | 25 + .../opensearch-dashboards-3.0.0-test.yml | 44 +- .../3.0.0/opensearch-dashboards-3.0.0.yml | 15 + .../opensearch/1.x/non-os-template-1.1.0.yml | 30 + .../opensearch-release-notes-1.3.12.md | 41 ++ .../opensearch/deb/debian/postinst | 12 +- .../deb/debmake_opensearch_install.sh | 3 +- .../opensearch/rpm/opensearch.rpm.spec | 27 +- src/build_workflow/build_args.py | 9 + ...un_manifest.py => test_report_manifest.py} | 8 +- src/report_workflow/README.md | 24 + src/report_workflow/report_args.py | 2 +- ...st_run_runner.py => test_report_runner.py} | 22 +- src/run_build.py | 10 +- src/run_test_report.py | 8 +- src/sign_workflow/sign_args.py | 2 +- src/sign_workflow/signer_jar.py | 53 ++ src/sign_workflow/signers.py | 2 + src/test_workflow/README.md | 26 +- .../benchmark_test/benchmark_args.py | 17 +- .../benchmark_test/benchmark_test_cluster.py | 1 + .../benchmark_test/benchmark_test_suite.py | 14 +- src/test_workflow/bwc_test/bwc_test_runner.py | 2 +- tests/jenkins/TestDockerBuild.groovy | 5 + .../TestOpenSearchDashboardsIntegTest.groovy | 9 +- tests/jenkins/TestOpenSearchIntegTest.groovy | 43 +- .../TestRunNonSecBenchmarkTestScript.groovy | 6 +- .../TestRunSecureBenchmarkTestScript.groovy | 8 +- tests/jenkins/data/opensearch-3.0.0-build.yml | 23 + tests/jenkins/data/opensearch-3.0.0-test.yml | 132 +++++ .../perf-test.jenkinsfile.txt | 5 - .../bwc-test.jenkinsfile.txt | 2 + .../integ-test.jenkinsfile.txt | 82 ++- .../opensearch/benchmark-test.jenkinsfile.txt | 131 +++-- .../opensearch/bwc-test.jenkinsfile.txt | 2 + .../opensearch/integ-test.jenkinsfile.txt | 124 ++-- .../opensearch/perf-test.jenkinsfile.txt | 2 + .../secure-benchmark-test.jenkinsfile.txt | 139 +++-- tests/test_run_build.py | 57 ++ tests/test_run_test_report.py | 2 +- tests/tests_build_workflow/test_build_args.py | 8 + .../tests_manifests/test_test_run_manifest.py | 6 +- ...n_runner.py => test_test_report_runner.py} | 42 +- tests/tests_sign_workflow/test_signer_jar.py | 61 ++ tests/tests_sign_workflow/test_signers.py | 6 + .../test_benchmark_test_cluster.py | 2 + .../test_benchmark_test_suite.py | 24 +- 87 files changed, 3216 insertions(+), 604 deletions(-) create mode 100644 RELEASE_PROCESS_OPENSEARCH.md create mode 100644 assests/distribution_build_os.png create mode 100644 docker/ci/config/windows-servercore-setup.ps1 create mode 100644 docker/ci/dockerfiles/current/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile create mode 100644 docker/ci/dockerfiles/current/build.windows2019.servercore.x64.dockerfile rename docker/ci/dockerfiles/{current => legacy}/build.centos7.opensearch-dashboards.x64.arm64.dockerfile (100%) rename docker/ci/dockerfiles/{current => legacy}/build.centos7.opensearch.x64.arm64.dockerfile (99%) rename docker/ci/dockerfiles/{current => legacy}/test.centos7.performance-test.x64.arm64.dockerfile (100%) rename tests/jenkins/TestRunNonSecurityPerfTestScript.groovy => jenkins/legacy/TestRunNonSecurityPerfTestScript.groovy.legacy (100%) rename jenkins/{opensearch => legacy}/perf-test.jenkinsfile (75%) create mode 100644 manifests/1.3.12/opensearch-1.3.12-test.yml create mode 100644 manifests/1.3.12/opensearch-dashboards-1.3.12-test.yml create mode 100644 manifests/1.3.12/opensearch-dashboards-1.3.12.yml create mode 100644 manifests/1.3.13/opensearch-1.3.13.yml create mode 100644 manifests/2.10.0/opensearch-2.10.0-concurrent-search-test.yml create mode 100644 manifests/2.10.0/opensearch-2.10.0-test.yml create mode 100644 manifests/2.10.0/opensearch-dashboards-2.10.0-test.yml create mode 100644 manifests/templates/opensearch/1.x/non-os-template-1.1.0.yml create mode 100644 release-notes/opensearch-release-notes-1.3.12.md rename src/manifests/{test_run_manifest.py => test_report_manifest.py} (94%) create mode 100644 src/report_workflow/README.md rename src/report_workflow/{test_run_runner.py => test_report_runner.py} (88%) create mode 100644 src/sign_workflow/signer_jar.py create mode 100644 tests/jenkins/data/opensearch-3.0.0-build.yml create mode 100644 tests/jenkins/data/opensearch-3.0.0-test.yml rename tests/tests_report_workflow/{test_test_run_runner.py => test_test_report_runner.py} (79%) create mode 100644 tests/tests_sign_workflow/test_signer_jar.py diff --git a/.github/ISSUE_TEMPLATE/component_release_template.md b/.github/ISSUE_TEMPLATE/component_release_template.md index 02a01df03b..1c06905ea2 100644 --- a/.github/ISSUE_TEMPLATE/component_release_template.md +++ b/.github/ISSUE_TEMPLATE/component_release_template.md @@ -33,6 +33,7 @@ If including changes in this release, increment the version on `{{RELEASE_VERSIO - [ ] Finalize scope and feature set and update [the Public Roadmap](https://github.com/orgs/opensearch-project/projects/1). - [ ] All the tasks in this issue have been reviewed by the release owner. - [ ] Create, update, triage and label all features and issues targeted for this release with `v{{RELEASE_VERSION}}`. +- [ ] Finalize the code and create the the release branch `{{RELEASE_VERSION}}` from the `{{RELEASE_VERSION_X}}` branch. ### CI/CD diff --git a/.github/ISSUE_TEMPLATE/release_template.md b/.github/ISSUE_TEMPLATE/release_template.md index 0b254c13a4..b41886651e 100644 --- a/.github/ISSUE_TEMPLATE/release_template.md +++ b/.github/ISSUE_TEMPLATE/release_template.md @@ -13,90 +13,130 @@ I noticed that a manifest was automatically created in [manifests/{{ env.VERSION ## This Release Issue -This issue captures the state of the OpenSearch release, its assignee is responsible for driving the release. Please contact them or @mention them on this issue for help. There are linked issues on components of the release where individual components can be tracked. More details are included in the Maintainers [Release owner](https://github.com/opensearch-project/opensearch-build/blob/main/MAINTAINERS.md#release-owner) section. +This issue captures the state of the OpenSearch release, its assignee (Release Manager) is responsible for driving the release. Please contact them or @mention them on this issue for help. There are linked issues on components of the release where individual components can be tracked. For more information check the the [Release Process OpenSearch Guide](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md). -## Release Steps +

+ -There are several steps to the release process, these steps are completed as the whole release and components that are behind present risk to the release. The release owner completes the tasks in this ticket, whereas component owners resolve tasks on their ticket in their repositories. +Please refer to the following link for the release version dates: [Release Schedule and Maintenance Policy](https://opensearch.org/releases.html). -Steps have completion dates for coordinating efforts between the components of a release; components can start as soon as they are ready far in advance of a future release. +### [Preparation](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#preparation) -### Component List +- [ ] [Release manager](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#release-manager) assigned. +- [ ] Existence of label in each component repo. For more information check the [release-label](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#release-label) section. +- [ ] [Increase the build frequency](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#increase-the-build-frequency). +- [ ] [Release Issue](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#release-issue). -To aid in understanding the state of the release there is a table with status indicating each component state. This is updated based on the status of the component issues. +### [Campaigns](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#campaigns) -

- +- [ ] [Component Release Issue](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#component-release-issue). +- [ ] [Release Campaigns](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#release-campaigns). -### Preparation +### [Release Branch and Version Increment](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#release-branch-readiness) - _Ends __REPLACE_RELEASE-minus-14-days__ -- [ ] Assign this issue to a release owner. -- [ ] Declare a pencils down date for new features to be merged. -- [ ] __REPLACE_RELEASE-minus-14-days__ is pencils down date for feature freeze. -- [ ] Update the Campaigns section to include monitoring campaigns during this release. -- [ ] Update this issue so all `__REPLACE_RELEASE-__` placeholders have actual dates. -- [ ] Document any new quality requirements or changes. -- [ ] Finalize scope and feature set and update [the Public Roadmap](https://github.com/orgs/opensearch-project/projects/1). -- [ ] [Create a release issue in every component repo](https://github.com/opensearch-project/opensearch-plugins/blob/main/META.md#create-an-issue-in-all-plugin-repos) based on [component release issue template](https://github.com/opensearch-project/opensearch-build/blob/main/.github/ISSUE_TEMPLATE/component_release_template.md) and link back to this issue, update Components section with these links. -- [ ] Ensure the label is created in each component repo for this new version, and the next minor release. [Create a version label](https://github.com/opensearch-project/opensearch-plugins/blob/main/META.md#create-or-update-labels-in-all-plugin-repos) -- [ ] Ensure that all release issues created above are assigned to an owner in the component team. -- [ ] Increase the build frequency for the this release from once a day (H 1 * * *) to once every hour (H/60 * * * *) in [jenkinsFile](https://github.com/opensearch-project/opensearch-build/blob/main/jenkins/check-for-build.jenkinsfile). +- [ ] [Core Release Branch](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#core). +- [ ] [Core Version Increment](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#core-version-increment). +- [ ] [Components Release Branch](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#components). +- [ ] [Components Version Increment](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#components-version-increment). -### Campaigns +### [Feature Freeze](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#code-complete-and-feature-freeze) - _Ends __REPLACE_RELEASE-minus-12-days__ -__REPLACE with OpenSearch wide initiatives to improve quality and consistency.__ +- [ ] OpenSearch / OpenSearch-Dashboards core and components teams finalize their features. -### Release Branch - _Ends __REPLACE_RELEASE-minus-14-days__ +### [Code Complete](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#code-complete-and-feature-freeze) - _Ends __REPLACE_RELEASE-minus-10-days___ -- [ ] Component versions are auto-incremented to {{ env.VERSION }} version. -- [ ] Plugins team to ensure working CI in component repositories and merge the version increment PRs. -- [ ] OpenSearch / OpenSearch-Dashboards core cut branch `.` early. -- [ ] All component repos create `.` branch for the release. +- [ ] Mark this as done once the [Code Complete](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#code-complete-and-feature-freeze) is reviewed. +- [ ] Create/Verify pull requests to add each component to relase input [manifests/{{ env.VERSION }}/opensearch-{{ env.VERSION }}.yml](/opensearch-project/opensearch-build/tree/main/manifests/{{ env.VERSION }}/opensearch-{{ env.VERSION }}.yml) and [manifests/{{ env.VERSION }}/opensearch-dashboards-{{ env.VERSION }}.yml](/opensearch-project/opensearch-build/tree/main/manifests/{{ env.VERSION }}/opensearch-dashboards-{{ env.VERSION }}.yml). -### Feature Freeze - _Ends __REPLACE_RELEASE-minus-12-days__ +### [Release Candidate Creation and Testing](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#release-candidate-creation-and-testing) - _Ends __REPLACE_RELEASE-minus-6-days___ -- [ ] OpenSearch / OpenSearch-Dashboards core and components teams finalize their features. -- [ ] Create Jenkins workflows that run daily snapshot builds for [OpenSearch](https://build.ci.opensearch.org/job/distribution-build-opensearch/) and [OpenSearch Dashboards](https://build.ci.opensearch.org/job/distribution-build-opensearch-dashboards/). -- [ ] Make pull requests to add each component to [manifests/{{ env.VERSION }}/opensearch-{{ env.VERSION }}.yml](/opensearch-project/opensearch-build/tree/main/manifests/{{ env.VERSION }}/opensearch-{{ env.VERSION }}.yml) and [manifests/{{ env.VERSION }}/opensearch-dashboards-{{ env.VERSION }}.yml](/opensearch-project/opensearch-build/tree/main/manifests/{{ env.VERSION }}/opensearch-dashboards-{{ env.VERSION }}.yml) with the corresponding checks. +- [ ] [Generate Release Candidate](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#release-candidate). +- [ ] [Integ Test TAR](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#integ-test-tar). +- [ ] [Integ Test RPM](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#integ-test-rpm). +- [ ] [Docker Build and Scan](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#docker-build-and-scan). +- [ ] [Backwards Compatibility Tests](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#backwards-compatibility-tests). +- [ ] [Windows Integration Test](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#windows-integration-test). +- [ ] [Broadcast and Communication](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#broadcast-and-communication). +- [ ] [Release Candidate Lock](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#release-candidate-lock). + +### [Performance testing validation](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#benchmark-tests) - _Ends __REPLACE_RELEASE-minus-6-days___ -### Code Complete - _Ends __REPLACE_RELEASE-minus-10-days___ +- [ ] Post the benchmark-tests +- [ ] Longevity tests do not show any issues. -- [ ] Code Complete: Make sure that the code for this specific version of the release is ready and the branch corresponding to this release has been added to this release version manifest. -- [ ] Verify pull requests to add each component to [manifests/{{ env.VERSION }}/opensearch-{{ env.VERSION }}.yml](/opensearch-project/opensearch-build/tree/main/manifests/{{ env.VERSION }}/opensearch-{{ env.VERSION }}.yml) and [manifests/{{ env.VERSION }}/opensearch-dashboards-{{ env.VERSION }}.yml](/opensearch-project/opensearch-build/tree/main/manifests/{{ env.VERSION }}/opensearch-dashboards-{{ env.VERSION }}.yml) have been merged. -- [ ] Gather, review and combine the release notes from components repositories. +### [Pre Release](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#pre-release) - _Ends __REPLACE_RELEASE-minus-1-days___ -### Release testing - _Ends __REPLACE_RELEASE-minus-6-days___ +- [ ] [Release Labeled Issues](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#release-labeled-issues). +- [ ] [Go or No-Go](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#go-or-no-go). +- [ ] [Promote Repos](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#promote-repos). +- [ ] [Promote artifacts](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#promote-artifacts). +- [ ] [Release Notes](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#release-notes). -- [ ] Declare a release candidate build, and provide the instructions with the release candidates for teams on testing (__REPLACE_RELEASE-minus-8-days__). -- [ ] Stop builds for this version of OpenSearch and/or OpenSearch Dashboards in order to avoid accidental commits going in unknowingly. Restart only if necessary else manually run the build workflow and declare new release candidate. -- [ ] After generate the release candidates, raise PR to lock input manifest refs of both OS and OSD builds with the commit ids from distribution manifest of the release candidates. -- [ ] Sanity Testing (__REPLACE_RELEASE-minus-8-days__ - __REPLACE_RELEASE-minus-6-days__): Sanity testing and fixing of critical issues found by teams. Teams test their components within the distribution, ensuring integration, backwards compatibility, and perf tests pass. -- [ ] Publish all test results in the comments of this issue. +### [Release](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#main-release) - _Ends {__REPLACE_RELEASE-day}_ -### Performance testing validation - _Ends __REPLACE_RELEASE-minus-6-days___ +- [ ] [Maven Promotion](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#maven-promotion). +- [ ] [Docker Promotion](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#docker-promotion). +- [ ] [Release Validation](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#release-validation). +- [ ] [Collaboration with the Project Management Team](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#collaboration-with-the-project-management-team). -- [ ] Performance tests do not show a regression. -- [ ] Longevity tests do not show any issues. +### [Release Checklist](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#release-checklist). -### Release - _Ends {__REPLACE_RELEASE-day}_ +
+
Release Checklist +

-- [ ] Verify [all issues labeled `v{{ env.VERSION }}` in all projects](https://github.com/opensearch-project/project-meta#find-labeled-issues) have been resolved. -- [ ] Complete [documentation](https://github.com/opensearch-project/documentation-website) for this release. -- [ ] Author [blog post](https://github.com/opensearch-project/project-website) for this release. -- [ ] __REPLACE_RELEASE-minus-1-day - Publish this release on [opensearch.org](https://opensearch.org/downloads.html). -- [ ] __REPLACE_RELEASE-day - Publish a [blog post](https://github.com/opensearch-project/project-website) - release is launched! +### Pre-Release activities +- [ ] Promote Repos. + - - [ ] OS + - - [ ] OSD +- [ ] Promote Artifacts. + - - [ ] Windows + - - [ ] Linux Debian + - - [ ] Linux RPM + - - [ ] Linux TAR +- [ ] Consolidated Release Notes. + +### Release activities +- [ ] Docker Promotion. +- [ ] Release Validation part 1. + - - [ ] OpenSearch and OpenSearch Dashboard Validation. + - - [ ] Validate the native plugin installation. +- [ ] Merge consolidated release notes PR. +- [ ] Website and Documentation Changes. + - - [ ] Merge staging website PR. + - - [ ] Promote the website changes to prod. + - - [ ] Add website alert. +- [ ] Release Validation part 2. + - - [ ] Validate the artifact download URL's and signatures. +- [ ] Release Validation part 3. + - - [ ] Trigger the validation build (Search for `Completed validation for <>` in the logs). +- [ ] Maven Promotion. +- [ ] Publish blog posts. +- [ ] Advertise on Social Media. +- [ ] Post on public slack and Github Release issue. + +### Post-Release activities +- [ ] Release Tags. +- [ ] Input Manifest Update. +- [ ] Decrease the Build Frequency. +- [ ] OpenSearch Build Release notes. +- [ ] Retrospective Issue. +- [ ] Helm and Ansible Playbook release. +- [ ] Upcoming Release Preparation. -### Post Release +

+
+
+ +### [Post Release](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#post-release) -- [ ] Create [release tags](https://github.com/opensearch-project/opensearch-build/blob/main/jenkins/release-tag/release-tag.jenkinsfile) for each component (Jenkins job name: release-tag-creation). -- [ ] Replace refs in [manifests/{{ env.VERSION }}](/opensearch-project/opensearch-build/tree/main/manifests/{{ env.VERSION }}) with tags and remove checks. -- [ ] If this is a major or minor version release, stop building previous patch version. -- [ ] Generate distribution release notes reviewed by PM team for opensearch-build repository. -- [ ] Increment version for Helm Charts [(sample PR)](https://github.com/opensearch-project/helm-charts/pull/246) for the `{{ env.VERSION }}` release. -- [ ] Increment version for Ansible Charts [(sample PR)](https://github.com/opensearch-project/ansible-playbook/pull/50) for the `{{ env.VERSION }}` release. -- [ ] Prepare [for next patch release](https://github.com/opensearch-project/opensearch-plugins/blob/main/META.md#increment-a-version-in-every-plugin) by incrementing patch versions for each component. -- [ ] Update [this template](https://github.com/opensearch-project/opensearch-build/blob/main/.github/ISSUE_TEMPLATE/release_template.md) with any new or missed steps. -- [ ] Create an issue for a retrospective, solicit feedback, and publish a summary. +- [ ] [Release Tags](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#release-tags). +- [ ] [Input Manifest Update](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#input-manifest-update). +- [ ] [OpenSearch Build Release notes](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#opensearch-build-release-notes). +- [ ] [Decrease the Build Frequency](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#decrease-the-build-frequency). +- [ ] [Retrospective Issue](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#retrospective-issue). +- [ ] [Helm and Ansible Playbook release](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#helm-and-ansible-playbook-release). +- [ ] [Upcoming Release Preparation](https://github.com/opensearch-project/opensearch-build/blob/main/RELEASE_PROCESS_OPENSEARCH.md#upcoming-release-preparation). ### Components @@ -116,4 +156,4 @@ __Replace with links to all component tracking issues.__ | :red_circle: | Missed multiple milestones |

- + \ No newline at end of file diff --git a/.github/workflows/link-checker.yml b/.github/workflows/link-checker.yml index b00a81b406..a4b7c5ca3d 100644 --- a/.github/workflows/link-checker.yml +++ b/.github/workflows/link-checker.yml @@ -1,7 +1,6 @@ --- name: Link Checker -on: - push: +on: [push, pull_request] permissions: contents: read @@ -15,7 +14,7 @@ jobs: id: lychee uses: lycheeverse/lychee-action@v1.5.4 with: - args: --accept=200,403,429 --exclude-mail **/*.html **/*.md **/*.txt **/*.json --exclude-file .lychee.excludes + args: --accept=200,403,429 --exclude-mail **/*.html **/*.md **/*.txt **/*.json --exclude-file .lychee.excludes --exclude-path tests/ fail: true env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} diff --git a/Pipfile b/Pipfile index 9f0798b96c..328cad7677 100644 --- a/Pipfile +++ b/Pipfile @@ -30,7 +30,7 @@ sortedcontainers = "~=2.4.0" cerberus = "~=1.3.4" psutil = "~=5.8" atomicwrites = "~=1.4.1" -validators = "~=0.20.0" +validators = "*" yamlfix = "~=1.0.1" yamllint = "~=1.27.1" pytablewriter = "~=0.64.2" diff --git a/Pipfile.lock b/Pipfile.lock index 8dcb40bfe8..6b78b9f947 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "15c704bcb340f5a90f3d3197c5cd791d86549cc1b7adec55998b0263b675e48e" + "sha256": "c7967cf4ead64b6ca007013a5dd53dd22a044683b5cc7c2e1dd6c375a91e483d" }, "pipfile-spec": 6, "requires": { @@ -649,10 +649,11 @@ }, "validators": { "hashes": [ - "sha256:24148ce4e64100a2d5e267233e23e7afeb55316b47d30faae7eb6e7292bc226a" + "sha256:002ba1552076535176824e43149c18c06f6b611bc8b597ddbcf8770bcf5f9f5c", + "sha256:6ad95131005a9d4c734a69dd4ef08cf66961e61222e60da25a9b5137cecd6fd4" ], "index": "pypi", - "version": "==0.20.0" + "version": "==0.21.2" }, "virtualenv": { "hashes": [ diff --git a/README.md b/README.md index 4272f8c404..ad250ee2b2 100644 --- a/README.md +++ b/README.md @@ -7,33 +7,33 @@ - [Releasing OpenSearch](#releasing-opensearch) - [Releases and Versions](#releases-and-versions) - - [Creating a New Version](#creating-a-new-version) - - [Onboarding a New Plugin](#onboarding-a-new-plugin) - - [Building and Testing an OpenSearch Distribution](#building-and-testing-an-opensearch-distribution) - - [Building from Source](#building-from-source) - - [Assembling a Distribution](#assembling-a-distribution) - - [Building Patches](#building-patches) - - [Min snapshots](#min-snapshots) - - [CI/CD Environment](#cicd-environment) - - [Build Numbers](#build-numbers) - - [Latest Distribution Url](#latest-distribution-url) - - [Testing the Distribution](#testing-the-distribution) - - [Checking Release Notes](#checking-release-notes) - - [Signing Artifacts](#signing-artifacts) - - [PGP](#pgp) - - [Windows](#windows) - - [Signing RPM artifacts](#signing-rpm-artifacts) - - [Making a Release](#making-a-release) - - [Releasing for Linux](#releasing-for-linux) - - [Releasing for FreeBSD](#releasing-for-freebsd) - - [Releasing for Windows](#releasing-for-windows) - - [Releasing for MacOS](#releasing-for-macos) - - [Utilities](#utilities) - - [Checking Out Source](#checking-out-source) - - [Cross-Platform Builds](#cross-platform-builds) - - [Sanity Checking the Bundle](#sanity-checking-the-bundle) - - [Auto-Generating Manifests](#auto-generating-manifests) - - [Deploying Infrastructure](#deploying-infrastructure) + - [Release labels](#release-labels) +- [Onboarding a New Plugin](#onboarding-a-new-plugin) +- [Building and Testing an OpenSearch Distribution](#building-and-testing-an-opensearch-distribution) + - [Building from Source](#building-from-source) + - [Assembling a Distribution](#assembling-a-distribution) + - [Building Patches](#building-patches) + - [Min snapshots](#min-snapshots) + - [CI/CD Environment](#cicd-environment) + - [Build Numbers](#build-numbers) + - [Latest Distribution Url](#latest-distribution-url) + - [Testing the Distribution](#testing-the-distribution) + - [Checking Release Notes](#checking-release-notes) + - [Signing Artifacts](#signing-artifacts) + - [PGP](#pgp) + - [Windows](#windows) + - [Signing RPM artifacts](#signing-rpm-artifacts) +- [Making a Release](#making-a-release) + - [Releasing for Linux](#releasing-for-linux) + - [Releasing for FreeBSD](#releasing-for-freebsd) + - [Releasing for Windows](#releasing-for-windows) + - [Releasing for MacOS](#releasing-for-macos) +- [Utilities](#utilities) + - [Checking Out Source](#checking-out-source) + - [Cross-Platform Builds](#cross-platform-builds) + - [Sanity Checking the Bundle](#sanity-checking-the-bundle) + - [Auto-Generating Manifests](#auto-generating-manifests) +- [Deploying Infrastructure](#deploying-infrastructure) - [Contributing](#contributing) - [Getting Help](#getting-help) - [Code of Conduct](#code-of-conduct) @@ -41,22 +41,22 @@ - [License](#license) - [Copyright](#copyright) -## Releasing OpenSearch +### Releasing OpenSearch -### Releases and Versions +Please refer to the [release process document](./RELEASE_PROCESS_OPENSEARCH.md) for detailed information on how to release the OpenSearch and OpenSearch Dashboards software. + + +#### Releases and Versions The OpenSearch project releases as versioned distributions of OpenSearch, OpenSearch Dashboards, and the OpenSearch plugins. It [follows semantic versioning](https://opensearch.org/blog/technical-post/2021/08/what-is-semver/). Software, such as Data Prepper, clients, and the Logstash output plugin, are versioned independently of the OpenSearch Project. They also may have independent releases from the main project distributions. The OpenSearch Project may also release software under alpha, beta, release candidate, and generally available labels. The definition of when to use these labels is derived from [the Wikipedia page on Software release lifecycle](https://en.wikipedia.org/wiki/Software_release_life_cycle). Below is the definition of when to use each label. -Release labels: +#### Release labels: * **Alpha** - The code is released with instructions to build. Built distributions of the software may not be available. Some features many not be complete. Additional testing and developement work is planned. Distributions will be postfixed with `-alphaX` where "X" is the number of the alpha version (e.g., "2.0-alpha1"). * **Beta** - Built distributions of the software are available. All features are completed. Additional testing and developement work is planned. Distributions will be postfixed with `-betaX` where "X" is the number of the beta version (e.g., "2.0.0-beta1"). * **Release Candidate** - Built distributions of the software are available. All features are completed. Code is tested and minimal validation remains. At this stage the software is potentially stable and will release unless signficant bugs emerge. Distributions will be postfixed with `-rcX` where "X" is the number of the release candidate version (e.g., "2.0.0-rc1"). * **Generally Available** - Built distributions of the software are available. All features are completed and documented. All testing is completed. Distributions for generally available versions are not postfixed with an additional label (e.g., "2.0.0"). -### Creating a New Version - -Each new OpenSearch release process starts when any one component increments a version, typically on the `main` branch. For example, [OpenSearch#1192](https://github.com/opensearch-project/OpenSearch/pull/1192) incremented the version to 2.0. The [version check automation workflow](.github/workflows/versions.yml) will notice this change or it can be triggered [manually](https://github.com/opensearch-project/opensearch-build/actions/workflows/versions.yml), and make a pull request (e.g. [opensearch-build#514](https://github.com/opensearch-project/opensearch-build/pull/514)) that adds a new manifest (e.g. [opensearch-2.0.0.yml](manifests/2.0.0/opensearch-2.0.0.yml). After that's merged, a GitHub issue is automatically opened by [this workflow](.github/workflows/releases.yml) to make a new release using [this release template](.github/ISSUE_TEMPLATE/release_template.md) (e.g. [opensearch-build#566](https://github.com/opensearch-project/opensearch-build/issues/566)). Existing and new components [(re)onboard into every release](ONBOARDING.md) by submitting pull requests to each version's manifest. ### Onboarding a New Plugin @@ -210,9 +210,10 @@ RPM artifacts are signed via a legacy shell script which uses a [macros template See [src/sign_workflow](./src/sign_workflow) for more information. + ### Making a Release -#### Releasing for Linux / Windows +#### Releasing for Linux and Windows The Linux / Windows release is managed by a team at Amazon following [this release template](.github/ISSUE_TEMPLATE/release_template.md) (e.g. [opensearch-build#2649](https://github.com/opensearch-project/opensearch-build/issues/2649)). @@ -277,6 +278,7 @@ Check for updates and create any new manifests. See [src/manifests_workflow](./src/manifests_workflow) for more information. + ### Deploying Infrastructure Storage and access roles for the OpenSearch release process are codified in a [CDK project](./deployment/README.md). diff --git a/RELEASE_PROCESS_OPENSEARCH.md b/RELEASE_PROCESS_OPENSEARCH.md new file mode 100644 index 0000000000..b5e91e13ad --- /dev/null +++ b/RELEASE_PROCESS_OPENSEARCH.md @@ -0,0 +1,543 @@ +- [OpenSearch Releas Process](#opensearch-release-process) + - **[Preparation](#preparation)** + - [Release Terminology and Knowledge Center](#release-terminology-and-knowledge-center) + - [Definitions](#definitions) + - [Types of Manifests](#types-of-manifests) + - [Input Manifest](#input-manifest) + - [Test Manifest](#test-manifest) + - [Build Manifest](#build-manifest) + - [Bundle Manifest](#bundle-manifest) + - [AUTOCUT issues](#autocut-issues) + - [Build Workflows](#build-workflows) + - [Release Workflows](#release-workflows) + - [Creating a New Version](#creating-a-new-version) + - [Release Manager](#release-manager) + - [Release label](#release-label) + - [Release Issue](#release-issue) + - [Release Issue Update](#release-issue-update) + - [Increase the Build Frequency](#increase-the-build-frequency) + - [Update the Maven Publish Workflow](#update-the-maven-publish-workflow) + - **[Campaigns](#campaigns)** + - [Component Release Issues](#component-release-issues) + - [Issue Creation Process Overview](#issue-creation-process-overview) + - [Release Campaigns](#release-campaigns) + - **[Release Branch Readiness](#release-branch-readiness)** + - [Release Branch](#release-branch) + - [Core](#core) + - [Components](#components) + - [Version Increment](#version-increment) + - [Core Version Increment](#core-version-increment) + - [Components Version Increment](#components-version-increment) + - **[Code Complete and Feature Freeze](#code-complete-and-feature-freeze)** + - **[Release Candidate Creation and Testing](#release-candidate-creation-and-testing)** + - [Distribution Build](#distribution-build) + - [Workflow Trigger](#workflow-trigger) + - [Order of Execution](#order-of-execution) + - [OpenSearch](#opensearch) + - [OpenSearch Dashboards](#opensearch-dashboards) + - [Release Candidate](#release-candidate) + - [Sample Build details](#sample-build-details) + - [Integ Test TAR](#integ-test-tar) + - [Integ Test RPM](#integ-test-rpm) + - [Docker Build and Scan](#docker-build-and-scan) + - [Docker RC Freeze](#docker-rc-freeze) + - [Benchmark Tests](#benchmark-tests) + - [Backwards Compatibility Tests](#backwards-compatibility-tests) + - [Windows Integration Test](#windows-integration-test) + - [Broadcast and Communication](#broadcast-and-communication) + - [Release Candidate Lock](#release-candidate-lock) + - **[Release](#release)** + - [Pre-Release](#pre-release) + - [Release Labeled Issues](#release-labeled-issues) + - [Go or No-Go](#go-or-no-go) + - [Promote Repos](#promote-repos) + - [Promote Artifacts](#promote-artifacts) + - [Release Notes](#release-notes) + - [Main Release](#main-release) + - [Maven Promotion](#maven-promotion) + - [Docker Promotion](#docker-promotion) + - [Collaboration with the Project Management Team](#collaboration-with-the-project-management-team) + - [Website and Documentation Changes](#website-and-documentation-changes) + - [Publish blog posts](#publish-blog-posts) + - [Advertise on Social Media](#advertise-on-social-media) + - [Release Validation](#release-validation) + - [Release Checklist](#release-checklist) + - **[Post Release](#post-release)** + - [Release Tags](#release-tags) + - [Input Manifest Update](#input-manifest-update) + - [OpenSearch Build Release notes](#opensearch-build-release-notes) + - [Decrease the build frequency](#decrease-the-build-frequency) + - [Retrospective Issue](#retrospective-issue) + - [Helm and Ansible Playbook release](#helm-and-ansible-playbook-release) + - [Upcoming Release Preparation](#upcoming-release-preparation) + - **[Communication Templates](#communication-templates)** + - [Release Announcement](#release-announcement) + - [Release Readiness](#release-readiness) + - [Release Candidate Announcement](#release-candidate-announcement) + - [Release Complete](#release-complete) + +# OpenSearch Release Process + +## Preparation +### Release Terminology and Knowledge Center + +#### Definitions + +**OpenSearch Project**: OpenSearch is a community-driven, Apache 2.0-licensed open source search and analytics suite that makes it easy to ingest, search, visualize, and analyze data.
+ +**OpenSearch/OpenSearch Dashboards Bundle**: An OpenSearch/OpenSearch Dashboards bundle refers to a compressed file that encompasses both the OpenSearch/OpenSearch Dashboards distribution and accompanying plugins. This compressed file acts as a container for the OpenSearch/OpenSearch Dashboards software and its associated extensions, enabling users to conveniently package and deploy the entire system.
+ +**OpenSearch/OpenSearch Dashboards Components/Plugins**: OpenSearch/OpenSearch Dashboards components/plugins are extensions that can be added to OpenSearch/OpenSearch Dashboards, to add new features or functionality. There are a wide variety of plugins available refer [OpenSearch available plugins](https://opensearch.org/docs/latest/install-and-configure/plugins/#available-plugins) and [OpenSearch Dashboards available plugins](https://opensearch.org/docs/latest/install-and-configure/install-dashboards/plugins/#available-plugins) sections.
+ +#### Types of Manifests + +##### Input Manifest + +These manifests serve as the initial input for driving the build workflow. These manifests are located in [build repository](https://github.com/opensearch-project/opensearch-build/tree/main/manifests) and are generated by the [manifest workflow](https://github.com/opensearch-project/opensearch-build/tree/main/src/manifests_workflow). Once the manifest is created, the maintainers of the build repository review and merge it into the `main` branch. The entire build repository operates exclusively on the `main` branch. + +##### Test Manifest + +These manifests are integral to the comprehensive testing of the core and components/plugins, as they contain numerous settings and configurations that facilitate various tests within the [test_workflow](https://github.com/opensearch-project/opensearch-build/tree/main/src/test_workflow). + +###### Sample Test Manifests + +| OpenSearch | OpenSearch Dashboards | +| --- | --- | +| [opensearch-2.8.0-test.yml](https://github.com/opensearch-project/opensearch-build/blob/main/manifests/2.8.0/opensearch-2.8.0-test.yml) | [opensearch-dashboards-2.8.0-test.yml](https://github.com/opensearch-project/opensearch-build/blob/main/manifests/2.8.0/opensearch-dashboards-2.8.0-test.yml) + +##### Build Manifest + +Output of the [build workflow](https://github.com/opensearch-project/opensearch-build/tree/main/src/build_workflow), used as input to the [assemble workflow](https://github.com/opensearch-project/opensearch-build/tree/main/src/assemble_workflow). + + ###### Sample Generated Build Manifests + + | deb | rpm | tar | windows | + | --- | --- | --- | ------- | + | [os-arm64](https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.10/7848/linux/arm64/deb/builds/opensearch/manifest.yml), [os-x64](https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.10/7848/linux/x64/deb/builds/opensearch/manifest.yml) | [os-arm64](https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.10/7848/linux/arm64/rpm/builds/opensearch/manifest.yml), [os-x64](https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.10/7848/linux/x64/rpm/builds/opensearch/manifest.yml) | [os-x64](https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.10/7848/linux/x64/tar/builds/opensearch/manifest.yml), [os-arm64](https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.10/7848/linux/arm64/tar/builds/opensearch/manifest.yml) | [os-x64](https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.10/7848/windows/x64/zip/builds/opensearch/manifest.yml) | + + +##### Bundle Manifest + +The final output of the assemble workflow and manifest that is added to the final distribution, this has the commitID (can be used for reproducible builds) and the artifact file links. This final bundle manifest incorporates the assembled components and ensures traceability through the commit ID and accessibility to the artifact files. + + ###### Sample Generated Bundle Manifests + + | deb | rpm | tar | windows | + | --- | --- | --- | ------- | + | [os-arm64](https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.10/7848/linux/arm64/deb/dist/opensearch/manifest.yml), [os-x64](https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.10/7848/linux/x64/deb/dist/opensearch/manifest.yml) | [os-arm64](https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.10/7848/linux/arm64/rpm/dist/opensearch/manifest.yml), [os-x64](https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.10/7848/linux/x64/rpm/dist/opensearch/manifest.yml) | [os-x64](https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.10/7848/linux/x64/tar/dist/opensearch/manifest.yml), [os-arm64](https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.10/7848/linux/arm64/tar/dist/opensearch/manifest.yml) | [os-x64](https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.10/7848/windows/x64/zip/dist/opensearch/manifest.yml) | + + +#### AUTOCUT issues + +These are the issues created by automation with the distribution build and integ-test workflows failure, the automation detects the component failure and raises an issue in the respective component repo. Sample [integ-test failure AUTOCUT issue](https://github.com/opensearch-project/k-NN/issues/914) and [distribution build failure AUTOCUT issue](https://github.com/opensearch-project/k-NN/issues/732). The created `AUTOCUT` issues will have the updated information with latest build failure details, the automation also detects if the component build has passed and closes the issues automatically. For more details refer the [closeBuildSuccessGithubIssue.groovy](https://github.com/opensearch-project/opensearch-build-libraries/blob/main/vars/closeBuildSuccessGithubIssue.groovy) and [createGithubIssue.groovy](https://github.com/opensearch-project/opensearch-build-libraries/blob/main/vars/createGithubIssue.groovy) libraries part of the distribution build and integ-test worklows. + + +#### Build Workflows + +| Wokflow | Description | +| ----------------------------------------------------------------------------------------------------------------------------- | ------------------- | +| [Check for Build](https://build.ci.opensearch.org/job/check-for-build/) | Workflow that peridically triggers the distribution workflows using parameterized cron. | +| [OpenSearch Distribution Build](https://build.ci.opensearch.org/job/distribution-build-opensearch/) | Workflow that is responsible to build/assemble the OpenSearch and its components. | +| [OpenSearch Dashboards Distribution Build](https://build.ci.opensearch.org/job/distribution-build-opensearch-dashboards/) | Workflow that is responsible to build/assemble the OpenSearch Dashboards and its components. | +| [OpenSearch Integ Test](https://build.ci.opensearch.org/job/integ-test/) | Workflow that runs integ tests for OpenSearch and its components. | +| [OpenSearch Dashboards Integ Test](https://build.ci.opensearch.org/job/integ-test-opensearch-dashboards/) | Workflow that runs integ tests for OpenSearch Dashboards and its components. | +| [Benchmark Tests](https://build.ci.opensearch.org/job/benchmark-test/) | Workflow that runs Performance tests using [opensearch-benchmark](https://github.com/opensearch-project/opensearch-benchmark) on a cluster created with a given version. | +| [BWC Tests](https://build.ci.opensearch.org/job/bwc-test/) | Workflow that runs backward compatibility tests on a cluster created with a given version. | +| [RPM Validation](https://build.ci.opensearch.org/job/rpm-validation/) | Workflow that validates the RPM distribution | +| [Docker Build](https://build.ci.opensearch.org/job/docker-build/) | Workflow that builds the OpenSearch and OpenSearch Dashboards docker images | +| [Docker Copy](https://build.ci.opensearch.org/job/docker-copy/) | Workflow that copies the created docker images to multiple DockerHub and ECR repositories | +| [Docker Scan](https://build.ci.opensearch.org/job/docker-scan/) | Workflow that checks vulnerabilities for a given docker image as an input. | +| [1.x Maven Publish](https://build.ci.opensearch.org/job/snapshot-maven-publish-1.x/) | Workflow that published snapshot maven artifcats, used only for 1.3.x versions. Fore more details check https://github.com/opensearch-project/job-scheduler/issues/319. | + + +#### Release Workflows + +| Wokflow | Description | +| ---------------------------------------------------------------------------------------- | ------------------- | +| [Release Notes Tracker](https://build.ci.opensearch.org/job/release-notes-tracker/) | Workflow that identifies if a component has a release notes added based on the commit history. | +| [Promote Repos](https://build.ci.opensearch.org/job/distribution-promote-repos/) | Workflow that signs and promotes the APT/YUM repos to the production buckets accessed via the cloudfront. | +| [Promote artifacts](https://build.ci.opensearch.org/job/distribution-promote-artifacts/) | Workflow that signs and promotes all the release artifcats to the production buckets accessed via the cloudfront. | +| [Publish to Maven](https://build.ci.opensearch.org/job/publish-to-maven/) | Workflow that signs and publishes to the central maven repository. | +| [Docker Promotion](https://build.ci.opensearch.org/job/docker-promotion/) | Workflow that promotoes the docker images to production docker repositories. | +| [Validation Workflow](https://build.ci.opensearch.org/job/distribution-validation) | Workflow that validates the released distribution. | + + +#### Creating a New Version + +Each new OpenSearch release process starts when any one component increments a version, typically on the `main` branch. For example, [OpenSearch#1192](https://github.com/opensearch-project/OpenSearch/pull/1192) incremented the version to 2.0. The [version check automation workflow](.github/workflows/versions.yml) will notice this change or it can be triggered [manually](https://github.com/opensearch-project/opensearch-build/actions/workflows/versions.yml), and make a pull request (e.g. [opensearch-build#514](https://github.com/opensearch-project/opensearch-build/pull/514)) that adds a new manifest (e.g. [opensearch-2.0.0.yml](manifests/2.0.0/opensearch-2.0.0.yml). After that's merged, a GitHub issue is automatically opened by [this workflow](.github/workflows/releases.yml) to make a new release using [this release template](.github/ISSUE_TEMPLATE/release_template.md) (e.g. [opensearch-build#566](https://github.com/opensearch-project/opensearch-build/issues/566)). Existing and new components [(re)onboard into every release](ONBOARDING.md) by submitting pull requests to each version's manifest. + +### Release Manager + +The release manager to a specific OpenSearch release will be assigned through volunteer model. The request for a release manager will be posted in [OpenSearch public Slack #releases channel](https://opensearch.slack.com/archives/C0561HRK961) and selected on first come first served (FCFS) model. +Note: The release manager should be a maintainer of a repo under OpenSearch GitHub organization. + +### Release Label + +The release label creation is part of the version increment workflows running in the build repo [OpenSearch Version Increment Workflow](https://github.com/opensearch-project/opensearch-build/blob/main/.github/workflows/os-increment-plugin-versions.yml), +[OpenSearch Dashboards Version Increment Workflow](https://github.com/opensearch-project/opensearch-build/blob/main/.github/workflows/osd-increment-plugin-versions.yml). These workflows not only raise a pull request for a version increment but also verify if the release label exists for a given version. If it doesn't, they proceed to create it. + +### Release Issue + +This issue captures the state of the OpenSearch release, its assignee is responsible for driving the release. Please contact them or @mention them on this issue for help. There are linked issues on components of the release where individual components can be tracked. More details are included in the Maintainers Release owner section. + + +#### Release Issue Update + +The release issue is created by an [automation workflow](https://github.com/opensearch-project/opensearch-build/blob/main/.github/workflows/releases.yml). Once the release manager is finalized, the release manager should be updating the created release issue. Sample [Release Issue 2.8.0](https://github.com/opensearch-project/opensearch-build/issues/3434). Update the release issue issue so all `__REPLACE_RELEASE-__` placeholders have actual dates. + +### Increase the Build Frequency + +Increase the build frequency for the this release from once a day (H 1 * * *) to once every hour (H/60 * * * *) in [check-for-build.jenkinsfile](https://github.com/opensearch-project/opensearch-build/blob/main/jenkins/check-for-build.jenkinsfile). This will ensure the [Distribution Build](#distribution-build) workflow is called every hour to build and detect the components failure early that are part of the [Input Manifest](#input-manifest). + +### Update the Maven Publish Workflow + +This step is necessary solely for the `1.3.x` release process; you can find more information in the [issue](https://github.com/opensearch-project/job-scheduler/issues/319). Please make sure to update and execute the `1.x Maven Publish workflow`; you can find the workflow specifics in the [Build Workflows](#build-workflows) section. Additionally, you can refer to a relevant [sample PR](https://github.com/opensearch-project/opensearch-build/pull/3838). + +## Campaigns + +This section is not required for a patch release. + +### Component Release Issues + +The component release issues are auto created by the workflows part of the build repo [OpenSearch components](https://github.com/opensearch-project/opensearch-build/blob/main/.github/workflows/os-release-issues.yml), [OpenSearch Dashboards components](https://github.com/opensearch-project/opensearch-build/blob/main/.github/workflows/osd-release-issues.yml). These workflows creates the release issues based on the template [component_release_template.md](https://github.com/opensearch-project/opensearch-build/blob/main/.github/ISSUE_TEMPLATE/component_release_template.md) and links back the global release issue part of the build. Sample component release issues created for [2.10.0](https://github.com/issues?q=is%3Aopen+is%3Aissue+user%3Aopensearch-project+%5BRELEASE%5D+Release+version+2.10.0+in%3Atitle+) + +#### Issue Creation Process Overview + +Inside the template [component_release_template.md](https://github.com/opensearch-project/opensearch-build/blob/main/.github/ISSUE_TEMPLATE/component_release_template.md), replace the fields `RELEASE_VERSION`, `RELEASE_BRANCH_X`, `RELEASE_BRANCH` and `RELEASE_ISSUE` to desired release values before creating the release issues across the component/plugin repos. Once the fields are replaced use the `meta` and `gh` cli to create the issues. Find the list of components/plugins from the [opensearch-plugins](https://github.com/opensearch-project/opensearch-plugins) repo (for [OpenSearch](https://github.com/opensearch-project/opensearch-plugins/tree/main/plugins), for [OpenSearch Dashboards](https://github.com/opensearch-project/opensearch-plugins/tree/main/dashboards-plugins)) and use the `meta` cli to create the release issues. For more details check the [create-an-issue-in-all-plugin-repos](https://github.com/opensearch-project/opensearch-plugins/blob/main/META.md#create-an-issue-in-all-plugin-repos) section. + +``` +meta exec "gh issue create --label v2.8.0 --title 'Release version 2.8.0' --body-file /tmp/opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md" +``` + +### Release Campaigns + +If exists any release specific issues/campaigns link it back to the release issue. Sample linked [issues/campaigns](https://github.com/opensearch-project/opensearch-build/issues/3434#issuecomment-1552138916) + +## Release Branch Readiness + +The `Release Branch Readiness date` is determined as the release date minus 14 days. + +### Release Branch + +Not applicable for patch release. + +#### Core + +This step requires both OpenSearch and OpenSearch Dashboards to create a release branch that will be used for the release. + +#### Components + +This step requires that every team participating in a release has their release branch created for the corresponding release by the date listed on this step. The [Distribution Build](#distribution-build) workflow will also start using the release branch to create release candidate instead of `.x` branches. + + +### Version Increment + +Versions are incremented as soon as development starts on a given version to avoid confusion. Following is the example that depicts the version increment. + +* OpenSearch: `main` = 3.0.0, `2.x` = 2.9.0, and `2.8` = 2.8.1 +* common-utils: `main` = 3.0.0.0, `2.x` = 2.9.0.0 and `2.8` = 2.8.1.0 + +#### Core Version Increment + +To ensure the version incrementation process is handled correctly, it is important to follow to increment the version of the release branch. Currently, the version incrementation is being done manually by an individual from the core repositories. Sample OpenSearch Version Increment [PR](https://github.com/opensearch-project/OpenSearch/pull/7864/) for the release branch. + + +#### Components Version Increment + +The next step is to identify the pending component version increment pull requests. The creation of component version increment pull requests is automated with workflows running in the build repo [OpenSearch Version Increment Workflow](https://github.com/opensearch-project/opensearch-build/blob/main/.github/workflows/os-increment-plugin-versions.yml), +[OpenSearch Dashboards Version Increment Workflow](https://github.com/opensearch-project/opensearch-build/blob/main/.github/workflows/osd-increment-plugin-versions.yml). Sample component version increment [PR](https://github.com/opensearch-project/job-scheduler/pull/398). + +The objective is to merge these pull requests in order to synchronize all the components with the core version. It is not as simple as submitting a pull request and having it merged. When a pull request is submitted, CI checks are triggered for each repository. These CI checks assess the dependencies. Thus, prior to merging the pull requests that involve version increments, the release manager must confirm that the dependencies have been updated, built, and pushed to Maven and S3. This crucial stage requires the release manager to coordinate with the component teams, address any dependency issues, and ultimately merge the pull requests that involve version increments. + +## Code Complete and Feature Freeze + +Coordinate with the Core and component teams to ensure that the code for this particular release version is fully prepared and that the corresponding branch has been included in the release version [Input Manifest](#input-manifest). Update Jenkins workflows that execute daily snapshot builds for both OpenSearch and OpenSearch Dashboards (Ref [Increase the build frequency](#increase-the-build-frequency)). Submit pull requests to incorporate each component into the respective version level [Input Manifest](#input-manifest) along with the necessary checks. Sample [PR](https://github.com/opensearch-project/opensearch-build/pull/3501/files). The `Feature Freeze date` and the `Code Complete date` is determined as the release date minus 4 days. + +## Release Candidate Creation and Testing + +### Distribution Build + +Refer the [Build Workflows](#build-workflows) section to get the details about the distribution build job for OpenSearch and OpenSearch Dashboards. This section covers how to sequentially handle the distribution build job during the release. + +#### Workflow Trigger + +Ensure the proper inputs are used to initiate the distribution. For instance, here's an example for OpenSearch. It's important that the OpenSearch Dashboard distribution aligns accordingly. + +![Alt Text](./assests/distribution_build_os.png) + +**COMPONENT_NAME**: To trigger a specific component, this includes standalone OpenSearch or specific plugin.
+ +**INPUT_MANIFEST**: The release input manifest that drives the workflow.
+ +**TEST_MANIFEST**: The release test input manifest that is used for the integ tests.
+ +**INTEG_TEST_JOB_NAME**: The integ test job name. Default already added to this input `integ-test` for OpenSearch and `integ-test-opensearch-dashboards` for OpenSearch Dashboards.
+ +**BUILD_PLATFORM**: The input used to build for a specific platform, followed by its own distributions within the `platform`.
+ +**BUILD_DISTRIBUTION**: Input to build selected distribution related artifacts, choices include 'tar', 'rpm', 'deb', 'zip'. Can combine multiple distributions with space in between (docker is only available on tar).
+ +**BUILD_DOCKER**: Input with a dropdown that has 3 options `build_docker`, `build_docker_with_build_number_tag`, `do_not_build_docker`, the release manager has to take a call with right inputs.
+ +**UPDATE_LATEST_URL**: To update the `/latest` CFN URL, Visit [latest-distribution-url](https://github.com/opensearch-project/opensearch-build#latest-distribution-url) for more details.
+ +##### Order of Execution + +Following is the order of execution of the distribution build to address the components dependencies. + +Note: The execution order specified is necessary only for versions up to `1.3.x`. For `2.x` and above Maven dependencies are published through each component repository using the GH workflow. For more details check the [META issue](https://github.com/opensearch-project/opensearch-build/issues/3185). + +###### OpenSearch +``` +OpenSearch +OpenSearch + common-utils + job-scheduler +OpenSearch + common-utils + job-scheduler + performance-analyzer +OpenSearch + common-utils + job-scheduler + performance-analyzer + security +OpenSearch + common-utils + job-scheduler + ml-commons + performance-analyzer + security +All components (which are ready after completion of version increment) +``` + +###### OpenSearch Dashboards +``` +OpenSearch Dashboards +All components (which are ready after completion of version increment) +``` + +### Release Candidate + +Now once all the version increment PR’s are completed and all the components are part of the input manifest, now its time to generate the RC. Use the [Distribution Build](#distribution-build) to generate the release candidate. Use the following section as the reference to generate the RC, validate it and broadcast it for a given release. The process of `Release Candidate Generation and Testing` should commence at least 6 days prior to the release date. + +#### Sample Build details + +Following is the generated build number after triggering the [Distribution Build](#distribution-build) workflow. The distribution build number denotes the RC, now with the example below the finalized RC’s are `OS: 7848`, `OSD: 6126`. + +| OpenSearch | OpenSearch Dashboards | +|----------|----------| +| [build_7848](https://build.ci.opensearch.org/job/distribution-build-opensearch/7848/console) | [build_6126](https://build.ci.opensearch.org/job/distribution-build-opensearch-dashboards/6126/) | + + +#### Integ Test TAR + +Following are the TAR integ test jobs for a given RC based on the above section [Sample Build details](#sample-build-details). The integ tests are executed for artifacts generated as part of the builds `OS: 7848`, `OSD: 6126`. + +| Integ Test Tar | x64 | arm64 | +|----------|----------|----------| +| OpenSearch | [x64](https://build.ci.opensearch.org/job/integ-test/4906/console) | [arm64](https://build.ci.opensearch.org/job/integ-test/4897/console) | +| OpenSearch Dashboards | [x64](https://build.ci.opensearch.org/job/integ-test-opensearch-dashboards/3531/console) | [arm64](https://build.ci.opensearch.org/job/integ-test-opensearch-dashboards/3530/console) | + +#### Integ Test RPM + +Following are the RPM integ test jobs for a given RC based on the above section [Sample Build details](#sample-build-details). The integ tests for rpm are executed for artifacts generated as part of the builds `OS: 7848`, `OSD: 6126`. + +| Integ Test RPM | x64 | arm64 | +|----------|----------|----------| +| OpenSearch | [x64](https://build.ci.opensearch.org/job/integ-test/4908/console) | [arm64](https://build.ci.opensearch.org/job/integ-test/4907/console) | +| OpenSearch Dashboards | [x64](https://build.ci.opensearch.org/job/integ-test-opensearch-dashboards/3532/console) | [arm64](https://build.ci.opensearch.org/job/integ-test-opensearch-dashboards/3533/console) | + +#### Docker Build and Scan + +Following are the details for the docker image build and scan. The docker images are built using the TAR artifact generated as part of the [Distribution Build](#distribution-build) (From the above example `OS: 7848`, `OSD: 6126`). The [Distribution Build](#distribution-build) workflow with input `BUILD_DOCKER` (Ref [Workflow Trigger](#workflow-trigger)) triggers the [docker-build] workflow as dowmstream. + +| Docker | build | scan | +|----------|----------|----------| +| OpenSearch | [Build](https://build.ci.opensearch.org/job/docker-build/3371/) | [Scan](https://build.ci.opensearch.org/job/docker-scan/1558/artifact/scan_docker_image.txt) | +| OpenSearch Dashboards | [Build](https://build.ci.opensearch.org/job/docker-build/3370/) | [Scan](https://build.ci.opensearch.org/job/docker-scan/1557/artifact/scan_docker_image.txt) | + + +##### Docker RC Freeze + +This to ensure that [check-for-build.jenkinsfile](https://github.com/opensearch-project/opensearch-build/blob/main/jenkins/check-for-build.jenkinsfile) wont re-build periodically and override the docker, the RC docker is created with build number. This step can be skipped if the input `BUILD_DOCKER: build_docker_with_build_number_tag` (Ref [Workflow Trigger](#workflow-trigger) used in the [Distribution Build](#distribution-build). + +| Docker Freeze | copy | +|----------|----------| +| OpenSearch | [docker-copy](https://build.ci.opensearch.org/job/docker-copy/466/console) | +| OpenSearch Dashboards | [docker-copy](https://build.ci.opensearch.org/job/docker-copy/467/console) | + +#### Benchmark Tests + +For running the benchmark tests, use the `benchmark-test` job part of the [Build Workflows](#build-workflows). For more details in running the benchmark tests refer [Benchmarking Tests](https://github.com/opensearch-project/opensearch-build/tree/main/src/test_workflow#benchmarking-tests) section part of the [test workflow](https://github.com/opensearch-project/opensearch-build/tree/main/src/test_workflow). This job offers multiple options to test the performance of a specific version cluster using various metrics. The benchmark performance results can be accessed via the [OpenSearch Performance Benchmarks dashboard](https://opensearch.org/benchmarks). Sample benchmark tests for [2.9.0 release](https://github.com/opensearch-project/opensearch-build/issues/3616#issuecomment-1642764515). + + +#### Backwards Compatibility Tests + +For more details in running the BWC tests refer [Backwards Compatibility Tests](https://github.com/opensearch-project/opensearch-plugins/blob/main/TESTING.md#bwc-tests-on-distribution-bundle-level) section. +On board the components/plugins to the test [Test Manifest](#test-manifest) with the `bwc-test` setting. Example as follows. +``` + - name: index-management + bwc-test: + test-configs: + - with-security +``` + +#### Windows Integration Test + +Currently, the windows integration tests for a release is manual. The manually tested windows zip is being evaluated and approved by the plugin teams for sign off. In order to test the windows distribution, two instances need to be created: one with security features enabled and another without security. Afterward, API calls should be tested by launching the OpenSearch and Dashboard processes through direct execution of the `.bat` file. + +#### Broadcast and Communication + +Broadcast the release candidate in OpenSearch public slack workspace and the release GitHub issue using format [sample broadcast mesaage](https://github.com/opensearch-project/opensearch-build/issues/3434#issuecomment-1571201919) to gather votes. + +As a release manager it is essential to ensure the successful completion of all the above mentioned jobs. In the event of failures during integration tests or scans, the release manager should collaborate with the component teams and initiate a re-run to ensure that all jobs are executed successfully. + +Post all the job related failures in the `Release issue`, Sample [post](https://github.com/opensearch-project/opensearch-build/issues/3331#issuecomment-1550461519). + +Note: Sometimes the integ-test jobs are flaky and might not pass due to several reasons with the component code, in that case coordinate with the respective component team and get a manual sign off. Sample [manual sign off](https://github.com/opensearch-project/opensearch-build/issues/3331#issuecomment-1552191673). +All the failed logs are in s3 accessed through the cloudfront. Sample [link](https://github.com/opensearch-project/opensearch-build/issues/3331#issuecomment-1552148546). + + +#### Release Candidate Lock + +Stop builds for this version of OpenSearch and/or OpenSearch Dashboards in order to avoid accidental commits going in unknowingly. Restart only if necessary else manually run the build workflow and declare new release candidate. + +Once the RC is finalized, in order to exclude the release from running periodically, at this point it is necessary for the release manager to lock the input manifest and update the `check-for-build.jenkins` to remove it from the scheduled execution, sample [PR](https://github.com/opensearch-project/opensearch-build/pull/3523/files). + + +## Release + +### Pre-Release + +Ensure that all pre-release activities listed below are completed before proceeding with the final release. These activities hold great significance as they determine whether the release should progress further and, if approved, they contribute to saving a substantial amount of time during the release window. + +#### Release Labeled Issues + +Verify all issues labeled with with this release have been resolved. Coordinate with the core/components team to close the gaps in resolving the issues labeled with with this release. + +#### Go or No-Go + +Get the Go / No-Go votes from project management committee (PMC) before staging the release artifacts for production publishing process, + +#### Promote Repos + +| Repo | OpenSearch (Sample Runs) | OpenSearch Dashboards (Sample Runs) | +| ---- | ---------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | +| YUM | [yum-os](https://build.ci.opensearch.org/job/distribution-promote-repos/46/) | [yum-osd](https://build.ci.opensearch.org/job/distribution-promote-repos/48/) | +| APT | [apt-os](https://build.ci.opensearch.org/job/distribution-promote-repos/) | [apt-osd](https://build.ci.opensearch.org/job/distribution-promote-repos/49/) | + + +#### Promote Artifacts + +| Artifacts | OpenSearch (Sample Runs) | OpenSearch Dashboards (Sample Runs) | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Windows | [os-windows-zip-x64](https://build.ci.opensearch.org/job/distribution-promote-artifacts/233/) | [osd-windows-zip-x64](https://build.ci.opensearch.org/job/distribution-promote-artifacts/234/) | +| Debian | [os-deb-arm64](https://build.ci.opensearch.org/job/distribution-promote-artifacts/235/), [os-deb-x64](https://build.ci.opensearch.org/job/distribution-promote-artifacts/236/) | [osd-deb-arm64](https://build.ci.opensearch.org/job/distribution-promote-artifacts/237/), [osd-deb-x64](https://build.ci.opensearch.org/job/distribution-promote-artifacts/238/) | +| TAR | [os-tar-arm64](https://build.ci.opensearch.org/job/distribution-promote-artifacts/243/), [os-tar-x64](https://build.ci.opensearch.org/job/distribution-promote-artifacts/246/) | [osd-tar-x64](https://build.ci.opensearch.org/job/distribution-promote-artifacts/244/), [osd-tar-arm64](https://build.ci.opensearch.org/job/distribution-promote-artifacts/245/) | +| RPM | [os-rpm-arm64](https://build.ci.opensearch.org/job/distribution-promote-artifacts/239/), [os-rpm-x64](https://build.ci.opensearch.org/job/distribution-promote-artifacts/240/) | [osd-rpm-arm64](https://build.ci.opensearch.org/job/distribution-promote-artifacts/241/), [osd-rpm-x64](https://build.ci.opensearch.org/job/distribution-promote-artifacts/242/) | + + +#### Release Notes + +Coordinate with the plugin teams and create a consolidates release notes. Sample [PR](https://github.com/opensearch-project/opensearch-build/pull/3532). Release manager can check if a plugin team has created a release notes or not using the [release notes tracker tool](https://github.com/opensearch-project/opensearch-build/tree/main/src/release_notes_workflow). Sample [run](https://build.ci.opensearch.org/job/release-notes-tracker/). + + +### Main Release + +Release the artifacts to production distribution channels, update the website and inform the community of the release. + +#### Maven Promotion +Promote OpenSearch to maven, trigger the `publish-to-maven worfklow` (Ref [Release Workflows](#release-workflows)), sample [run](https://build.ci.opensearch.org/job/publish-to-maven/17/console). + +#### Docker Promotion +Publish the images to docker and ECR, trigger the `docker promotion workflow` (Ref [Release Workflows](#release-workflows)), sample [run](https://build.ci.opensearch.org/job/docker-promotion/32/console). + +#### Collaboration with the Project Management Team + +While not directly under the responsibility of a Release Manager, the following sections require coordination with the Project Management Team to ensure that the tasks are successfully fulfilled. + +##### Website and Documentation Changes + +Coordinate with the documentation website team to ensure the changes are in place and close the gaps in promoting the website changes to prod. At this point the maintainers of the repos [documentation-website](https://github.com/opensearch-project/documentation-website) and [project-website](https://github.com/opensearch-project/project-website) will handle this task. + +##### Publish blog posts + +[Sample Blog Post](https://opensearch.org/blog/opensearch-2.8.0-released/) + +##### Advertise on Social Media + +Coordinate with the project management team to ensure the social media advertisement is completed + +#### Release Validation + +Use the validation workflow (Ref [Release Workflows](#release-workflows)) to validate the published artifcats, sample [validation workflow run](https://build.ci.opensearch.org/job/distribution-validation/3/console). + +### Release Checklist + +Please update the checklist either in the release issue body or as a new comment to the release issue. By following and updating the release checklist, we can ensure the success of the release. Sample release checklist for [2.9.0 release](https://github.com/opensearch-project/opensearch-build/issues/3616#issuecomment-1646312725). + +## Post Release + +Once the release is completed following are the activities that needs to be completed by the release manager: + +### Release Tags + +Create release tags for each OpenSearch and Dashboard components. Sample [OpenSearch](https://build.ci.opensearch.org/job/distribution-release-tag-creation/78/), [OpenSearch Dashboards](https://build.ci.opensearch.org/job/distribution-release-tag-creation/77/) workflow runs. + +### Input Manifest Update + +Replace `refs` in input manifest with tags. The `refs` can be identified from the bundle or build manifest, sample [PR](https://github.com/opensearch-project/opensearch-build/pull/3534) that updates the input manifest. + +### OpenSearch Build Release notes + +Generate distribution release notes for opensearch-build repository, sample [1.3.10](https://github.com/opensearch-project/opensearch-build/releases/tag/1.3.10) release details. + +### Decrease the build frequency + +Lower the frequency of builds for the release version of OpenSearch and/or OpenSearch Dashboards, sample [PR](https://github.com/opensearch-project/opensearch-build/pull/3523). + +### Retrospective Issue + +Create an issue for a retrospective, solicit feedback, and publish a summary. Sample [retro issue](https://github.com/opensearch-project/opensearch-build/issues/3535). + +### Helm and Ansible Playbook release + +Update and release the [Helm chart](https://github.com/opensearch-project/helm-charts) and [ansible playbook](https://github.com/opensearch-project/ansible-playbook) with the new OpenSearch and Dashboard version. Sample helm chart [PR](https://github.com/opensearch-project/helm-charts/pull/431/files) and ansible [PR](https://github.com/opensearch-project/ansible-playbook/pull/131). + +### Upcoming Release Preparation + +The release manager for the current release should ensure that a release manager is assigned for the upcoming release. This can be achieved by coordinating in a Slack channel and [@opensearch-project/engineering-effectiveness](https://github.com/orgs/opensearch-project/teams/engineering-effectiveness) team should assist in assigning the user to the upcoming [Release issue](#release-issue). + +## Communication Templates + +Please utilize the provided communication templates to effectively coordinate with the teams involved in the release process. These templates are designed to assist you in communicating through the GitHub release issue and the public Slack channel called `#release`. + +### Release Announcement + +``` +We are starting the process of release. +Release Issue: +Release Manager: + +Kindly review the following information provided below regarding the release dates: + +Release Branch and Version Increment: +Feature freeze: +Code Complete: +RC creation and : +Pre Release: +Release date: +Post Release: +``` + +### Release Readiness + +``` +The version increment pull requests are still awaiting attention. We kindly request the plugin/component owners to review them and make an effort to merge them promptly. + +OpenSearch: https://github.com/pulls?q=is%3Aopen+is%3Apr+org%3Aopensearch-project+increment++in%3Atitle+ + +OpenSearch Dashboards: https://github.com/pulls?q=is%3Aopen+is%3Apr+org%3Aopensearch-project+increment+.0+in%3Atitle+ +``` + + +### Release Candidate Announcement + +Refer the [sample rc announcement](https://github.com/opensearch-project/opensearch-build/issues/3331#issuecomment-1549876307) from past `1.3.10` release. + +### Release Complete + +``` +OpenSearch 2.8.0 version has been released to public :tada:! + +Thanks everyone for the help to bring release out. +Component repo owners please create a github release based on the tags of . +Presenting the retrospective for the release. Please feel free to provide your valuable feedback for further improvements in the upcoming release. +``` diff --git a/assests/distribution_build_os.png b/assests/distribution_build_os.png new file mode 100644 index 0000000000000000000000000000000000000000..23fcb9f7937ba465253908ab61cc5c047abdabb1 GIT binary patch literal 122341 zcmeFZXIxWD+disDL_tIaL_x|1Q2_~x()(6KdhZ0JiL_8d4+v~U1wm9mL+ctcYt5`#bI;5@_kCU0@YKjai=CC1 z_0XY1?Amv4n;bd>sy}q-2>sYm;7XLcNYPPa>*(xy+daV7 z%Q4jG#-T%3l3_3Ebz6+Mz8V&r^eR3%$$vrm>8-(j)*!zJ5?q?s%s|DaZzGR;M_d#$ z@#C}<+ZMEHw)Xw>#LU;v_tb5c)A)CaVl@v!2x(?x!i3gYhWxvLTDvAbqw^A5Ojj0D zll8vjq6aZE;jwX9rbYM5KYqB%y3fFp*ZVdJhEJ^Zu|cOs4eipp!D&HROr}u`$PnWD zTV4inP~-Xevy;hBQ@rDk6GR2BKM?y-G+Ect8Z!V!C}pL%yxUH->Sy(MT%&!pKxrd_ zCGD9j5*<52Knz)?eZ4?3?sVk5bncmvaWpe%Dr#>aQ7uu@-}0UjIEm?M`)%#hoaJBE zd*3^rk@h>LL3&XWK#t@<(!YCqFLS@~_U)D$8~gK!eCGRCL*rv1%;$IyK}mWtW?H28 zTXX(R^yAmwFg;$rQsFrF`H$f6s1pm}Q+v)QR`{8MQr1gFx+%Pm5YZA079KH=I@6;X z9l$tPxLRr7*VjKJ4tzg$=y06-A!gv)Vc^3He1PG4`S=hE@cRt#xt-1Q@4KM->?8mF zeuVzBpoZx!ZEfJUsk6VUD>UGNPhibO?=av|BkpEafmZr@iq1aXU`H1pCs%NYx9`s? zhm=DUfv?`KfsR5U-d@lE#Sj(Yze^|r-+x|~5*GTqNT8>Ru$8`%&@CT-S0P#OHSjfI zRaPM(A!UD;`-&#F@BB}7;D0K@4*~;y6{VztgM-1rH^4srZc^736cnVcNlQsfO9CY% z145yJjv<&jzY0|GcVbUxCt^a3L^8i=>TRz@Eqk*dbXKenb-T%J%KNXdwe)j#pZ1FEc|9us( zv?{Bz)PF`zmGz@;``)2L>W8#%YnX)`Ucw)*W8XYzT}`_Bs8X0d7UQDJrdg@Yw%aFc zc7_sfL7_Kl##@Lc%L@PXOegQbEq7xv-DkZ{Pur+!%gdDTx8YRl<@&%(`mU;-TlZCB z`f@h7uT7X>u=n*G&%L_-U02$5C(tKb@fBh38#e?GOuG_#{Aa6r#A9W z{`RIz!Xb_2gwYf(=HE5(Fk24xJJYKRIRnOV>qO-W{c>!7Hp^UJc;vS?&x@${Dme#< zf;4{D1TggczZ&(wul?`P|9=|#rXax$do)RkzTT4pa*DfZwW1q6k{BrKH2v*)Rmkeg zz_Z_Rp==*UUy3%4J!a=yN531_z5l3oX}m+#N$K=2M&%3}m#%W?xBKo$v;+$WU!U~kW{%BPgNfJHC9joCB@TWnCl=v-HZ_e>kl(qwGL{fq_~ z!rp4D3E#DQpE@vMn?=KU9q!N?x1M;)kflR~taoA+kNvqC8H>yUWw*}l{%=W9~UcxRn1Y5(%K^SwSjVRu0 z{pP`q9%VP2LsW5!ltZ`b{x<#s3~ZP_?>H5*3Ffe#v$|<9szzU83t#Vg3bh|PKa}aR zn5?xs?hu6xs(=LJQ>f<*s!j~B_B48vHJF==`_L@EsI1(E-ZVS2MZ*HsN-JBy)29x%jeS;L}e_^WhMv5%bq zJMUmM21W~L`Mw|TCJUVi;gh4XSBDf^NLK0eD)q@4X6?N%j`djGW6&xnO-J}dCE8`v zQQ6hRs0P9;?U{25BVMZESxI%ss*@O~9ucv2p3b1DP0T^Uy_Y9{nD^}w%+|*5Fpd8R znYg3rDD#h1ZuEqi)xt%VxwOga=^0Tk#_neRp+cL0#`W%tO0%z_TWcLX*6Gqpwgm>4 zSq|gUHC}|i%JlnI_&%?il3j)NAu4&dy+|twm^Wp$Yf{R(Qy&IfA{9$j_o!w#!guXX z`&9ikO^jIQ$C6PE&@4WmI@#H5T9kY0gN#xPN!qp=T2E<=%4FhPF{nAR=I7)H4p)_a z$^DB>qABXC(Y;s#8&BkN41wgcyNM%KhA+c=ElRcr^7ZKxcaj7=fw6i_X`n;57IHZD zTK}>x_Q6sA$}UdfWc^Z=TX7m^7Z9pgF;A`o^_4_2!ej#J?vnf1wRdR$7ESv$ezzHU)hBdN{B%QlHwrOi$ z>#s>3+YD5ie+Qk?_iYIsO`PqptT{W+dAY$&ZC{?qE34cDty_J2f!zBfbUm_?P*1OM zf6gZt$OxmKcZgVZ4WE8qZFew6LTMiEvp}A#&Qif@?LJW5nd#nI?_pyIZ%qV9jbfJ# z$}74_P6u=c?7jMf)JBKKaLUen9Eg#XRB@8upghBe%31lW$NlX0CB$JEU%&x_V|-~e zKp$Ul0UlT^bbx9GCI5h)gfQ2xcZcIYZBg>l>`#6F6W#z!i%e=gCZKk$T~~5cws;7G z=`zO92udXenQ;84efrk8we0;)vW5c@ai^-lx*`_O&thI=_N-IRYotxcS9PL(d5C<$ zlwaW5y%)vRdAe!-u$5hMDJ#jjuuXdL zE>T*o5HN?K9br+0&bFsK*TYgVwsxU6e>}asn@)26P_A|M1bZEg;1Yb86w8s%9Blu( zERkPnM3ZNewJRChJfVmyGH;>QOcSyt5T(fbUA!pN!jBPw@TDy8TYFpx9h)wsW#!7ivO z{;|U}cD*mn0%^nT*`Mp@YfpK%0)1XhjFbzx-+V1Y75=fY#g?T{ zy%3m}tBZB(V6<`0x7}*0xg<3Om#b|Wlf8S$(jY}2-@Uy7%-R^CZ6=<hi3Es$@al`)OGrY32U zQ@N2NaNmRF#)F_&KT=fc-U((Y@8nE=_&S=en3i&uMPr>V4F5vk>4$eAE5J|eB38MR z8SCRdjTxjg@Y{rQs}Z>le%?N%_GvX(t-Y#3sKiQf(~7PXd$~xW{Di(vuo~$NwRb9E z^3T4j2y~%T5XA>_weOZnIITfmS~AlaGXKHUunV(2f5QqBm|m>aZp-nZeljK?z>B6z zAFSeWxIoRik?knDSxsnQxXT7plc@Fk13{7dJ82p?0S^QW3M*U+|w2P%Xx88jm6FoY_6=)zTmc4X@5cVVod@?~$P0`<2 zBj01RIB?us?l#jzSky)zlCm}KQ+ZbME!n&3X2=Q~YG`NFE%-WG+)%Voc}y_P&cA^_ zbmnxj=IgYm5my(NXGa#jDti3)_Yg4_gKl@2iV6^f?*b7kP^GyyBKWIpJYavxwxL}` zMfMBZUZ?&-Ii@*8zV}Qa7Zj^X^uEp5SS%R8Lymapm!{GhR2O4{bsikD)z}9-d#f)18|~2zs!7o4YjoHMHqrb5#czTeC}e-|{qB=`h#3sFh|dk&GNH z#~|e8%Pk_C)QJ<|l8I^X!$Qz@y4#k2^h_Jm!HL{*!PZ26!Q&z1?pKDYp*^6!V)J4W zcw*r~Hazz=wQPTT+7OF{RF-5pWV7sw_m;5ik%Zgu!D^@MMyv|HkG`a$K3e%n(MqDH zlhlSmjj%s@S?V~{>a0(-HFV4tVlM|yp~7kF_zJFaSFXBRI+Y&lOut5JvUQlhnM9R; z!GHW8A=jxZSD3!Cwm<$deiA*LdCk%3lyW->{O7uiPvnET(TeAm*B&%I)GE=>$iZ)e zAqEvy-UstcH$A$rVVfQj5KQ2yUI9vsiDvv+v|~#ayUmh?-@0IA$a*$5_UzU?LLwHb zV9#b_voB@4zgTDNyLPa@e73p`h-FQnmv9~TIJTMc{beN8%B`NpIJ-EJG&bNj9+@sbQSE$6Ia?fnB z_ZCNDr7TS3&x6c}HEj)DM%*pummJjkU}QEu@p>{;9aTH`ZPMur8&m^E_UqSfz1(xr zBQdaQRH}%@q3Gn5LS0^vc8VEdtXfUw7}NlESR01^K=u8sjenlNM|2_g z3-e%>mSnSEOh1>0fsvUdv2n~tMPuScArnEWt6v7FKN71fdCESEMVtUa^C1nKn!a7; z_$SAljvEtbgAj$s|IBWpy(NDJi7>Lj10`yW^U6v)cYgLWL)MZj+b?xYggRL2WEwH* zMr3ao?fs#V66*?mWV#yN>v4yHq$MI8`k<-={%N4+mA4bGkAd=PP1^*?xyEn$D9XX^ zx!ED(Xtl<~39mS*q%{eP602S(bo^0>NmQ4IE6SqgyUD9d&+QQScEje5(uTM0tot2{ z{kl?}!*Jd5y3YfKN=VafZ=DtoNj@sgMk7LS!@@V%RuohLHz9VaiWwCT53`0zz_+Zs zvuS0m0_k0Pa36YhHOGaXES1X(1|N_;vg5^=m&R;(Uz0-vKsOqIkfT}ixrGJ~Z)y5V z3~mL2tU0~2hv0ZCZVNfKcG$cgmkBD1A+$Vx{)rD-1F4v~8!#8)%4#$Jb!U5e+(n{- z)U7Xvr-H96{D=W_eatvb?6<*pY*5Cf$|XtFgo4F^13WR(PMp39Bpod9M@hc%fy
KBtoe+&8FwEAyD$Ib)v+#`BVSfQ5XC0B;l+Gsx@Xv-_e|QVg}TqJw2`?(G#}^y z6}z13ZHyi{aaqL*oJr|zIo+Z?o}WtA@hGGUZY<0{b)71BNR(&Zr+ORdxK}ZGO3QYN{oHyX zHsw=w@>)1<*mBOaIzHqhcV*0%eWzGrVo2+*9C2^_ay3FOg)xFAyEi{7>50W^+WV0& zsD#}?b zv=4&EcvV>zh0p)m34>%8kZ}!#dDSs^);0dX-e?Wt;3DV*kEG4J7KIBn7QOjiqbf+3 z?=Qi3^U{ZAHRQb~YW{ej0GW7hA_r=i4GziLJ7*BA^Z{p$vQxl)utNUeHZwh?6QugJ zd}ZK+!_bfhHmDUFw_IqxSLu!bX|#N$IwkGgyCSf0ano@>vJq3yi#Sr-*w!-2Jz8xQq}EahG;&8qb@v0aU~krCyVv%x(oWwsi)QU|%#AFPvi z3ViEPW`#~0Gfnp)Reqsm$L9s@OD(ch-#c%5qp+YKDaNe`6Z>~L`ep}=hQ3UW*^V?= zuxSl9=wPVCT4bU1K=EL>9y(Te{p0x87%W?P)#dRoT!fF|xmtqtY9Us$n&1&+Dp@fo z@US5x#UZed+X7{0jhlxn&R>Al_UG$|yA{VF>N!AYq=K*e$uoq1GvjezAd6A9|@c0gD!RQmAg;@|dLtEFLuGUXY#f zBo%y&iYrV_Zjka4e78QZwk}gT{yr5Mz6?=X82b@!`a(~#(iYuLan zu+naQyMHFHP9A_7!Zm^~;oc-Vic&5MMp#OOWa{$X98vRkyvq@6);KpU~^Dd5R|YXR!VD$Rc#+Q z9@a5(`8*`V(eOaFW|H;+@7JnAWiV(54(<56n$q|F>2^+C5n@{0G&WY3RUGOvDxQbp zyJ2|tkB!qPNO;X~yNx~kK{1e^7hA+(EkPf{HZg-LuZ@t8e?kEd97q3HsNOk&J=l-K zE8DI&)CO`t{FAV_j zGyJ|~k!ppw=J>(kCKA^wC^(sz!D+C-rOY=e8hqiS{cEv`fQy1rOSu6hboB=5z82}I z-9@+RgRIcG+O{LoA`*9{dq4Iarbef{)u-#tS_I38XeAz37Vt;*!#b;>cxi0|y*vLL zMpl%1wD)4BpbKjP>9A*--OGCOavM#xotZ|BC14#~Ss(|chF{6BF((;D+oLNTa7!)QC}w{Z3tzl>|zzCHTUq5+90wh;&Z_mslYw8KsZolGQ9Y?SyVjQV7Qq5Kq@&ZoqW9wnu=_Nb zmJQqYhDPkH%}(GsFl@qd+8p2OC{<(;j(lJ}BZueJ_P0%EpwWSAJ@M<;TQ4kwS4#jE87cGid&PhscoJJc1}P@>n}x(SJ4t z$8skatE8POY*$Cr_7Z^ zG_?$ld$=o;L>=FI>lvoSG+j8?5%DW*;!N8@DuG+v>=TcD$hXbo;6PCMW_A^jFJ4;^ z*&|x?k!)@^{|Um921n)2d@(oDmJ}d{IOhW3A|Ba5T*L5_0`_X>kJflwL@nsBj*y#Q ztB$0WoN9OO&t26h#&PO%oPab&#(LHxEy`JS4Bp9iZb=q5dbFo7U^2$08X)lB!rTY*Qpa9Hj6>EOau}*&$)F=LxR3a zePd$3c!7-Ry&1W8C3s?ojh8jXlPE5f1?~UZ^!^r023UOYoMr!Y(`snu&9R)w-dB z;OD$UYZIj*{X^LHhg)@ZrYgfQJBb&lID|*zyi6fS0E|6{ zrG+GX@)r?S8*}vb=`VU$ARkW@GH2~Ad7S5k@{?M9$hbPMH~~y8jeuUO`<^1s7}t?E zzVG1*sSiWnu9yjXYcmCeuLSKhT*Uq#iP~-_@3&E83ot5$UbpP$jaP1@R&n^ri#Oa#Z^CM0*J@YB_VUyRqa=maVu_n~!# z!ao<>{}i{i7MivRE2cjt)Jp|v4(SnqGr?teMee`1IKvep0J>GWud@B#yXx|e9ggSO}fzv(*} zvXu1;)6>tNq#4%j$V6@})m8;AO3QV0rW3~iWW}jl6~3GBX1aQ~EwG~jT@@SgPvqop zfG#@bqS3t=g8thJ(C54|6_42~_HULn0zm!v!^5$ZF1gEdzP5m0goMt*s(`B(2AqC2 zSrW>qIeg@(<}m!bsO9Jf&3Hy`8#!M;W2thWJE4wv_Q+x?+}3NL%BmGWm3-odKArsa z$c`9*o^ijX?C-6MK~1xM5}5$Ux0Gs&(#5y~+4m<7)sP#D5uxeteTdk#@@)&-s{36? zhb?5)YG~W`F4+ei(7l)%cfpG5pJ3z96(wz*-Bh_@@OZCcd3E;c{D#qIfKqt`JuP(KA`co$a@zVMkRh6Eo+CEty!qS}Q=$gE3 zV`M#_J>abYSok<}yIW})jkC{qAyNvkbsaWUUd~+^g08n7*S&Ve+^O3d+s+o<-2~W| z%o2bl$18g%M1DElTHEMbJnI`N)v#U8ANIgg?%{eY3 z-Bikzb{AfiXvRBJ%38CY&5&7M<6t&S%?>I9JxtSMzO*4COl z+&T`h=F{JZ{vD7Y9E7I8ma+_x4OMJXOWtu8`o|dcduw;)5Ni#)BtC$^r4DB|zaMTq zPzRulNT|Y2vBdd2c0O5~WdeL+yUN?J%1||(^bKHDzU++2Ce73o>)p% zJo^RtZ;?Rk&!@EYTxc6FtW~vKcSG#hokk~M_YaI-h(3Zq+t+vdAv@r5VkF5~0G|x* z5T7g-g37zwAXMeX-&}7gx$~SfT)o^f^*wn?#&bWYnI=a#=N?WmC$LLA+#j}ew)<2M z!4AYw?M{9wGM8}cSZ-z!M2qT12JDpB*MH-x+;pF9HwPvcZ*|@Byz+}2!atS$@O%q zXEW$Ad$Y3S(y(o5HNg5_KDMOh7<*L3Etm+mJ#{RqEre>`b{G})#n38-{~p2PtxT2@r#M{;RI z`|Bi0#YQTAV7WmgW%IR^1NhivsSND2w0l2`MmD2;@^f?JVIS|D{?l5OzK4C-J#8zW z>+mbSTZkEpWof(z%;oosOfiO(aX0+|`km*zRu%VIMgjgy4w7KhxYU5&c8ufq0BGT$ zW|B^d_#Lp`Ab=OuJS-b@rG4Q%GV$nf0Ybp*CtFTxjF{_O-YXI=eLBY${-v1Dd+dUD zFd;PHqT12l%xF5#2sDP^F+_P~!%4RojhxV6k@1uK$URce*uuu%^IpmhUj6ZvA#73k zz!;bESnxgS#_PReK|uxv@Yg&yY11i&b!H_ajRSx~aRKPZ8=&yb;1+hv@$0RXRSSDu z6C6&9$y@{hbZ=sciFHy8P-kDe)a zr6pWFYkvn{VHdfBM%mGqOqHaQj(%QWgeMDIn;Y?P?5RrHR9eXv6l8^cDKaa}9j(Z_ zz5WRI$^+k)s^IgRj$*lfcJzB75l(~SE^h#&SlN75xZ;BJ^uU|=g!Ah*7kiNwZ7hNj z*%_t&v(IFKq*v~Ok2S(E=MFn$pJZz1KGRl0mEGG2DQ&$FkjwT_Fe)M=J8G>8+*bUs zx@S2_t=vWtP&6pMNll5b(v@uHJ~do7kPkmAMQ_^cBGT5)R5uKEP~LKK z(Y&HNQwcbBX}$+UE`VwG7%8I^k=68DVeH`lEOq&22uFq;7_Qt!yb8dId{y%*S^QaP6mjj zAp|}b4vzp5qy(1&s3Nc*rf2J>okhWFc8?mS{y=JVh%F$g+Ji6cB(rJ6Mrh*FAglvm z5d}9y8`@!RYkTX4*w8^&J^EHb2Gc6Z>5@H(Q197mxGRccOH+hx0cidipZ$eZIs9NN zg$*<@RIqcfpEg^v6Sg7WQA=Ascp>^$>YZC1=K8-hNzVadI(};O{4a18VCFGzmEHf{h5J^Q!S#i!_rPmc@yeJhfxr6OxINC;PW;h9Pv-!=;0I?-t$Oavi&2j zK$O=a-dobL+~vDhg;?Vz8HZh!@mLbvutmjPy`R^(Z^OUw2qNxSyH~d_LkQcr*X{K6 zneFNih0@Ryfb8K|XA~8Dw8S0>OWhi+wN!a%g0H+NQBjmqWzF-4e@0~{De(@z{;yZ> zK115SWJuK|Ja!&p_jdb$Ig`n<{Od3%`yQK1(xtOY-umujljG$@Savij(SSAg8?T@W zBC2$!rP)$Vb+kC9rZ3t~GN&#=NO|}0E7qBUj33C>QW1;Dv7?|bDIbmycr8EtmG7#L zqlaim&zNm#*3HVO5`ykzv}Q;tE?bS?YErU!O8aHzw5d^<^QupIZy(5LT~H0 z&5Rp7%%)XU+}HnY30rHRgvHTI;lGajfcfKnS-m%W+`8h@Z%ZJr0wpAly!x$0nyLT| zay_NQn)7!yYqB^Clt_vGF7WFGeoFn``0-O4grMP)U!?MXr)BE*B-nrw?M*JfKtTSb z;Qd4a^tL`qp2I+{{A-JRN&x{STmBCw6B}$s5p|CinjofDD&8^AtnqhX7?xVP@36D-h6$%fSJaqeXp~%1^Cd z@(!a(1~>75<$~{Bmj?e?k2m|gH{}P%D&2hm%@x^AW;S5lJJ@z0UlItkF+odZ6z`1% zzvM;J0)y&H`X?&{Bj0eIuXZ1}l$Q3*J#?0jQIHO791qdn;pIStD_g!)fH0^yE13ve zW;t>0wX?!(0FZoW&Ls(~ki&OolSeOQ!nZ=V`gL0>o|Ey|B#CM;@Am7WjCOuVen59^ zuUm`6Z^p(*i1``!6XY>jCm@XqnoCzfy7f?HAM|npV%sy=$(RmEJQd)Uzen!*HNdD7 zD%)h2pQ=1b2&NMd+XYp3J~=Hla#D$_3`kji#`5FlmJko1LXO1P+N=a1W8wK37E$Yc z{@DM*$zeVHja^@39$+!b0IAo2f$Gjh>|(wW<+7*0r*|-?=KzjzJL(qrwTV#S(c@=Rda15bUqV)R>7XJi^7nODsA+}#xzCA$n^+D zU9AAZ60JE0Kzq!m;GYnF(5>_;W|Dp#YSmHjBi~;Nf*B-)&Trn$0wSzuCvN<17IHC1 zGY$090qI2sEK`P&I3l9F>XOs#zsRn)}SiYSA7bV_w8-~9SVlROibtdJw^mORG?Ny-+yfGpQ743J`stW4$jcfNdP zn=Gt!yFG_eVuRL6j5rtlo5jAD#ss>wz!3LmYC`*AL8 zC23c!rH@@u6J*9CssYjQ#d6e2(`0kccxEtxynpoMg)`}cpQa9PO~wX~1AT;#(0R;Rg}PG267d#9Lbv|-!j5kwTKQ`SKo>hYc0k&Cm6rhBEY9sQ?c z`j_wd_OnmO$ozPo9+LRQ)aJ+RWd%GE7M(Y{3Ph5;Tmkh{ARJmuKBI5BqO6z+q>SXj zLeF0R+NUX3YhPu;gDlgMFxp}^H^WP%fL!BwAmqoovBWB%g=_|6J_ess-ycXytSbuV z2DNTh4;WZOklRlS-uajFM~2+ME|q((QKH}3#9_=rH~I>eDta1Liv3ID&q`K_nW~<| z1h{jZOZz148YGL}I|;mw9yK5A6p_9mZhPi;Oa28jtKOgqKK(CSFE;Jwu|iWdHk-?c z2LJ|gET;BG`6y!(=aC^pW|8YMH0s58;s$;SV8_Z`J7QHEdsbeR*JlNErw2_&KQD3m z>Itb92V_a?n1d0u1p$|e!lByyo!rP8G`zvVS8czfqT3gl8NN}|mh`MH3lLI|u07G) zcFP7Zjio0dV_&?}`BkFCmxtm)5UQ5UhxgX6P7t?#;s9p>t?L=)*rJve#$fqzs3 zLk-k21t20?(B&3tME$onJC7DWTnDB(MBN5iasSJPPx=g@f zSlo)DmVo>lw@VXzKpI!F05o{^bO49(*XzG^adT4VfJY@(AYtbVC*wy@(ixSgqUE2e zsh=v;pIX9XO|FdwI(0lI{^%<_vC@5+1vwSv$1#)8$#vplfGJL@K{;&b$TjU>@1PIW zHO^eQrs0F0iCp_0*?j9ForNR4#D~5dWz;L#X3*Rs>=SGs);621VK_3`EXB-`?RBn_ z3O%c=w`RyEyJT%@DLiFX<^;k?RmY5x+C?%q+C<>^7G7JlBEe&Dgovv7p$Ir)PkoIY zD{aQJV%y|}!GCRe58E6z0_y?M%8P8DWi}`M_Dy;8ov5WU7B3n;I3P4)y(@U@r&K@p z)!3c265||9_^E}qWNXPL#$PT_(t)yFes?C8t{M6Zwaec-GV5RJ$=RFKRHib~_s_Ry z)4u+c2-l?I-uERpJ47+MKoz>p;ti6y>X&?#>zO)Q!fBYL?{l72qZ^Lj&{}f)? zSKLQ-rLR{2k{_`tuyy4^^}Vd;3Uk&Kb3Onzo2T&YGH;pJC<4Hs8f&a0_e0A7l2?Jc zlp*I+24YM7@~$C>f~myu-^htLm(CX#VI^rT^huvNFu`Z+_4b8i2flLGc;H;oYR~Z< zv;e}eJOF`^QnR1V)LIu=rpRzm$cRXr8CeGed$9%bJMP0(28{LP>|0LRvu3OP&w;%00`=(Q zVZce8^hHDWmdET{y|->HkxSiJT1JnIvEGb}?E*Y&HMfNS{=IY=&j!GQ%FNb(5TbD! zo?we1u;Haev>IHa_D-u^`vjaj3Evv+yT0!5Y0?rnfWhHp(9x4D2;b1`U+Rb5ef!4^ zmt6G?KeP&$N%S)SP<8?K5`0lcaZ?kUR6+%o8rHtQd~kH+Mo)rX2(*p-P4Df^F64Xz zJ*$jGAY^#1r-!v}PKJ29qg5&ClFcQ~g06Q8m(Lust@6ryer`2?(W!OjJ7T<=*xiK_ zF10}`Zohl;?!WChG42!fW;o0geNKi%vE%2Pq9m#E?5tfJy9Xj}WIEz8>_G>NSk5k+ zh5Se4K+K~9DmGNsWAHX0d`C=_y`ZkWRZC%{T_uLV6#D$96f-_{14@pxt=Tl<<#vw@ zy@C1ktrWx6ZiOlTozMU}uPW7~OJuM9MzW{)&Z9<}rxmdf*08C~GoKY1_E&iE@`5E~ zNN#+71qcM^@)vKcedTH0`6^r*OjHqS^NG(0n$I*G;}3RWxLZXP5m#TT#tDYbJi74e zU_U$)AF-YT>@zW51K$!trQ}&u(!A}&UBmWw%An^{49DlI^-DP~M=s?*YIWZRUOsqw z)7E|DlN{5l!$CbNhDs7qhqZ|2H+=%rH{p-}lgM#higp6!ee&R^43sv4fzxG@&OmVa zT97+5Q$~WU>oHWsEj;bSe$(9qw~^*y!nT~RPaaLy68Ld;5l=5M<6-p70r2`IJ{s;ZWgnw0gPMs)NM zt+BkcpfZy@ojLMu)UJD(K0*r~3LyBF}srhbd3)Xsk!VV8TyQ-ZzQfQs+*XVptz zOXm+?as*`KO17irSdd04Rl%*j;JS>5*-55nK3E~_a9Ua=x@X;XT)2)=tH8r8VKK;+ z&>jZK;Au~=m9;AfFqH=lzuxnNu5C-Dj*h`(8#^{y1ts+8MUjkp&ik2&SysW|{xooHyH!_AM$M#UqK=;ksXR2*yB`l%AmJMy~`{#{>=nZt#GXVpv_;^yf=I`7nX~ zYGUdraHMA(2$KDgSHm_HW!69@4*r0YNLQ{Hs;#m)I6bkF=Fsgg3EKcwK(HeCq{luD z>3o%A_LB_@g>1|qp1rib0RvCZ@TIb1hrF>C{eX=0IBHV!s+vH;psaf_Mw*i3`$L z$*H6}DZogU(6Ea8&z+9GC5Dl2(U&TG00}0)_W{HC;6WcHZY(i${p?W(qpn27qICX{ zdltP04tfRpbnyjM$9P*n&Tos^1jI`5L4a`mXlRtPs1BFh^HK=5i2dBP&t1i-S199x z?m*sFIn{Ex=8z6>@K4XR#ICmP5&w@_(X`Vdm)<}N%wH^j;j|rZpj#wuKTx06N{mo? z0Nah{ma^{z9LE=4In?xUfyJj3af&N3O08#WthfYg^*;V=L!AfvMIQJf%DO_QDq{1J z@hA}dyPFn7db?$f5&O~2wx(fX=pIT%& z+Y_@AE7^}XydiL2IRnYRf{#VIbp51cu<#=ID=*t2e=V4Xnm-9gygVYPRddhf!{$u_fG5NV~e{0B6 z?Ts=Z0C&frfped+F=yY90lw$H!a5W`#^Op8fimvqsd~VP5jR?qrY(iT=;NH1SYY>4 z*{;Yz?QoHFZ}%DH4-p?;MQ%Vx%SRIx@;%f6X*MmbE&qb}aJUfw!Un0GP`MQ;h`5x^s914OU4} z9~9p%$(z@kF-ncz+xfoh3}_9_xdU55R;#jhkCOFT{aYT>XGs^h2tjHkipB8K^^g5c z(WT7v*>k4xhG^+wh-z1cToBo;qa|L7`hy2n%GwE>!D&U2kvb{upvT@Z_dKbpI<(P( zJEyLuyTu8N35_d$_8~+?VU(hPvonXm$8v_}fuyD?!?o>&CDMC56@@)>5XxgOOZe37 zU}Mqh2DmWW`R>aj-AZZxtpQj0GFzOiKZ?$+Oijo3L=RQ{FmR4;+%jO$O|T zNSNloc2{AoTnxB#xHeTX<0!(D9nbGJr3fE!e9n1&=FKJCkFW9Ao{0KOI;-9Kk?A4x zn`S-ub}eE>V4v}P>lP)6wZYKrWOw)tJncnJqT(QuM5$I|^$H3mIywzm55duiDe)&{ zd=~rj_U_-YceeI)*do*3LHxCOEQwsNhv@NOMG<+~v@goIxJJ(l{&!m@Kg{8%?>>YT zEr;9K^O(U1#+kjrn0PR0sp=NOFiCCS#7OF(^43X(CLnB^07b)Ve^7!YOTsQn?G&l% zWhvuPwfS1P5xmEu)bw7bIb7KdFZfohl;ehg51j?uIRPxlu?987BuVe8+R)JL#g!J0 zN~$+4MqY|WeaVz6I)}#6)%Rsd={@+2VOF59W5Wa{=oXc;@joh;j zbGM}3>NYf;obJ8FhfxDWZ#Nm&%scLae7_~aab2i~?;`ia7PVf)4AqAt&R5Q;4l;U! zIrn!A)z&thjnvL?uUMFn&w#Ljw#xufE?9YDkz^Ajey>p7N`5geHqy%;HZ)3YO<*c0 zd&vZ4O|%Q>QV41WWyV^ZH^qllE&;xUXNK4Xy0LLg0yGCw(99dadw|m&XOJ;rykc9E>eWZVr+#R+xo#2R+%*k%= zeeW}~5T#;0ItycL&bzXONCc$hxqQ!LN~3BRW#_XEzYB+6#_p1+;JYYWc{GPZWN6HI z0nUD0#J6b4X6M09Qq^>#ry~x6llU~gM0$Q=zRPwp@|wrN;^)UUG;z`Or{;1$hDJl^;sn20 zS)pwy`+Dngm%&s(PODM10}y!kQVumkLX3=>IBT6MyK>!LkIFVV|%LQCPI)}u&E#M zJ-+i!kV6(7eG7_|`v%i${2aRk<3f{ATD>sg1K{4KcBV6NrV7Clg6J{4cQs@}wO>;l z(rc=2*&5p&ttwW7@>?a9fMe3cDk_RA3J{SI6VP}kYx0L@dJ&dd0drT6@ft17l}u-n zTZl4=@*fK6P;wg1trv<-@oP|AzTyKf0Jzn~JwS+DfDPz+Er!6deV_9GI=;**U;Nc4 zNq`>$NlpUmX%?)6irCvvxqfI2_IPXar|0-)`iCAAkWVyAp6QT!XIbJ1A71~zN#xCiVIy>}z6^ZzYcM!uf3?*CB-z^#6nz!_ne{R#{iNc;Asj}t z+ESv8OJXtZeGGXw8YPwQfh-t?v7VCkE}D(6Jye~FxG`UF_bfZ_68SNC7|HK$&X>O3 zqn>kzUTXm*`55nro#K}G+OCW2*bvb}#&De*NSav4Ro*-TksL&KVL+CW!zqIv4LT|H z6O|a4r)Fff;oF-k+pD#GwZNaNNJ5G3e9&xk>rk{F9XexAPDqJ}4?|0r+Sh;Z49y!h z9TONj0(tk*ILBm0D;fOefx}MjaPF9cP9e0INyNkW<7^OkmA^+hWp^8J;%=UqJW{2;$s4iek`=DS*?-ne->geu z(gGHnvcC8|{>Xm_#)-egOz+nS71=x#6BEjIRu?0Od|ZOQAHJS9ZKrV^@iJ4( zoX2_Szy)@5)URk$!Z$Y+E?r_(7U6rsWr$}86qWEmWicyuW`C_?8jrBKJFjKpd1p3H zDV%HJqtSaWRK<`cALOR?>p-aKimo=#x$!FR)6C8D*n4_N8}lDof}FF3vw>H|3#r{1 zX`6@UZ+F}_T#vr#kOT{U!SrXJm|B&ZHGWW$5xG)|xw1&Bw})pN!_DZc9*DM;Dxu8#tFL z3}tBPMeEIbJMj|=n#dyVFN5=S-RwFTaw{TsLuDT4uufpIn9son6TfD5v;?=-azEaW z4~DXKona1gj_SEPJAsTC?6;fu5gjT7G!hDy94BzWhd^wudGTds4IG!shh?|dYpCkI zbeH+vm08xjCan@3OuzB_J>uNQhZ?;K-f)Y|{ zl7wQLaWcK6fs=yu>B=_WT@%{QtU-P2(4F4Q)FDa+PF+%5#L_04b7G+S_EimV7A)ZC zQ6CpV*Qe{jvYrRqZ03USn=N<{SGR5S{ZMj6m0o&(GeoY#u48XS`Dr^>mLc(UpdMXK z&(@tuppf(P`BI=-TY{25z2e;$A5rNm*ZR=%*Q}wuesX9Y|Kvg zmk35fT~-h_xCNG48^eY7@~o$JRF5!}SM7(GjH;Vous)iJylOx}3=MJlv zy|hUr$5{kEL)Y0^Zm4VQzCX1^V94d}E&#Kp|Q?}NhB!*tIh_A#sAZ8-( zWx_1tyyCTda@vdibO;VDV|NZj?A90J9#)$+KJzH#Ht5^&*zp~0K~2rxyg;3nUn0e! zmp`vR2pPzaVtfuQC`Y#U%<~gc1BW^7Rh2IgQcKERcP_0zMXPcCy_4HzS^Zz^y=PQY zYq$Pwizuk5fQXb}L7IS4rNb6b0jbhURHRqwEkwYE2#A144TAJuLJ3Vo0jW|$4Im)B zCxj#ZFEYpuJ?d(PjyuEyelvqRYS-;k`?QMD=UeuZkhOiEA$2(xqL5RZ<#eDoG4c<7<|yuZNk@ z(N8AH^Ap{ovGX;F8Vgv;^hD}(5a;z9LN{S`&eq7W29%WsH@$qJdRhdNd`EpwBCS)F zeKx@QZd7LJwDrB#nZ1kZ>Aa_tEN3g<(g?ez(XTSXfvsp2N-XS>moFb*1xc>nLjIi` zI;VT_OHh{dt|wfO3Xf|%w#nYew)o_%&U78?nA_fRC++Jk+0}Me4tH>K=WBkCHE7I% z4s%o8x{;cq_mJ5E2;yc4tSMG|wZc=!=N(-0H9__k&uW%=gbGEu!+}=Km;03u{AhcB4NE%lAVHX;%=-R4<)o>Rh*10m{zAje=grDOX|^FdU&z-JHN6e5`tp1deDNYm z@berv8Gh`NtamcwG_I9e45~)LkoXNXOL#z>VIeweeL700nl2Gdr_21wy*X18z0Erw zYwp_-)}w6fp_jFMBGcQFwks|4MRBs_r%m%b!J;yA$Y&3YUSt5q#sr`6@~?av=UcAR zWE1y;1KknSIiZU0;s$2V=R!Yn=wwrK8uPQF1akEgNb6bWjEn?^%cr-s(`nvir$_Bh zrVAc;lx~d5f#e@9b%OT^w4=GSVQ>3O)W;IGXFOP4I^vBe9Zjvx5xR>v#?LA#o`+2L z3meN9eQ6MrN7NXg__||#P}{l+x6KrkxFHdJi1K&kX7U?-U%0hL=`32XhDAgqVBEGOnvY_vgMt(ZpLR~S`9bwueqM0t2V`;SQx$N!3`+0J; zU5j5FKUNqDv*XQ`(}3%+YZDH1XRCz0E}(9|H;*qEKn|;*up8f@o1~Aj_5-Gvv)cGS z^2hprYBWDJRwa$ZP+YqUbu&9pkP_ITlFUFV%9lA$E4K;ECU~!Yu6_eZ*vui%;djjk zkI$|yW*cYlf;=n7(RjPp2D@&$w{^{|z3WC3gZt3;1uXHI51zjhtfAm^_1moaKaL|{ zXN$eaUT@SX++DA%?a!Ie zOauI4o)b%-bDs0eUSZ5l#_wW{dg?>SqPv`ak592(f1b3**&(IbO#PRn6B}MTbKBjU zB9GGD%jG3zQwKw5UXmN#3u^DI&vvPq zqAp|vpGdb%!-ZPYZ^O;eEqrt^<<}fL`BB>1T~2%LzM0Ik!#R^gic5_YyPSj_*4txX zHVxzAb$)VvFHrUQnFQ92>c%!Z!&!X~uFOw$)BVq*1=n&rbJ9^kXj<>ZpIWJ(R_&?P zZw@v39(d@kGZt>0f8f(jKTWx@_~IWbQnS7Sg_}(vpLm2>+{3_NV9Y9~aDgBz|E_L! zt|m=-clu+f8!_g~xb)OaT#FD^cyitN_H1YKZM4}&FHZIr*_8QUZ9;}^BFDg7m-25R}Cjd&cZ?R)UPj!%y-ao2gv`bBu-R-J9u~=3N)Jr=tolh0c|# zF&{mfKG&6^v?+Y@&yqc8JX$)_vYeuX)-$h=Sh%QDx4Zl}Aswxc`+w6C%gJ1tvI?%Q($r{0*$H2ZnPY^*hQLF7*X9SRDS@)8aGT1?{X zEOxPZR(GXMSKuL2^d|ianQkZhDGKtK(3k&sSD`1BL~DJ$f#^BD0FyNDuubs=4lv{+L}}o!)9a;;l(KNwPnJ zkFu@mA-i6#chCvNp(CGIq63|yHhZ8wE}JLeneRMuyyXm!4;Y9iMK6J@}ZNecFXwfsas|)TC*p{S6#|u zo?+Sii=1L$kG;kzWs7{uW}{hjXCc3SE*p%c76$b63xh6pcDJ(d%^~~UER@kMwb3*Q z$QCJU=J0A$G%Lb0(4A}ku6GD*jOJMS#h+MR%`g|Q^0~?piwH#Uk%bJq(Mp2p_2P$( zQUQ@Ix$TVpy^^M6*sQ(mXR$;^e_b_)W&0#(pG0*yeFw8y7j8`gv98oM=GN8g*?Vu6 z27it0<62L#s}$5@Jf<^jjl8h(N3Pt6YZT+QE;Cf_Gh%CqyV~LG&{yLwa7LUXjrSAv zn$v<>iglz$kO^+uc^(-e8qEzL7Yaf5`txq@HYy_lFK;4H`9V_Pru8Y8=Eo&n7xOhwo^mg~ z9oXW{yvkq%^?pE^Fl0YH8tI-fqMk(8BZCZ*E5kmMB5DM?D?n5_%W)$}qu zA5mv<1FX+Y63*DKlovl{18!!I@WKoDc~+u2gV9;(p?A7@lZ2ng_A8f4ulfrMt1_#4 zNF;{eg45{MW2I;o$<>|TLv(P~TB$y9e-)=62Ps5OwB28_3WU_SlTzcBXdbzr>C3qt zI5v7EUo%hu((scDwWOPid_0m)#90WSE%OFGjzunc64prl3xng23YL4xa|3;oM23@X zx8>iTMZNhx>^UYJ&y%^-i?e`Ps!qR4D6xjA{q!1sp+xQ!N?Itl^JC(9rIB60U6Uxt zmNk-;TPVl9eWShsb?THR#>L6MOua;k)m}DaOBc5z<;QdE!^o5>@tcUIKD6H}J^bT) zf!$gKMt&j3w1wOt8Tp;-@;#_Xp>`oQ;tb1O14!* zD&22ITM{TLOdTpI?qn$5vX@$XWzdycAwA&NpKFMhm+%nPr+#k&nHENVO^AdAo+j5f zWwS`Wu*q28QqN55$2EOgO!`{UFZ`-ULBK>{!^*)XyRh8%b?C~}muH-EKS|}g1-QSq zW&rpuR0t|BD^Vz{F>4x~;GrbMuv*#Tn53(wj_SmGGaKVNy(q#Xfn?bjfNb5Df2lY7 zsNWXGJagLlro$&XE?+|=Lgkis{0mgqqojl{o8P1ho29&7#%?wa-3PBw{vQ2F8w9j5WTb`0m&T^!X(vxJQX$z zRuER?wC;0~=GAiwI5Eq0JrmDmVx80Q%X^cnO@5%@IE@vf;VJ5jRQgW_=Mjc{(E)nO z9MMg8Yfk2Dt)?YwnLpnqR}NkFHE3-TX?;b=ooAbgf2N7M+q22|Wm9@Jhoc`K{wkIv zDwk|j;@|&M-3!j_|@l_$;jlSW{dod1|^52p{}QlRUuS*1sP z=|rD@N2t;HuIP1so31pwy-VhQ@GpFfVR)Np>!~7tR+&=i++q-+}$hOWp=k`H&Qr-e&TJS$#;z;7;_@|M1~2b>juvy?3x8Tr|J`TZ0*gXeA3s_F)318wqpOMrL&%X^cWKpsm>A+FYh~}V zl%M(C;12AAydtJMeM=->)=CM&5_PY*?l*!@^W=dMO1GGtIr=~5&R=v7_&2YC-0RzhCEfH^v34B4)_0V1JqzkgcFX*9`0H-V66OM~32i;fpJ<~4s z*qyAFnZEi}GhjVW5?1AZ z2$Vi(5ZV*|(l51=&DFL*EDmixG6Blg2?Lc+Hrd*2hxSBDOJq;{Z6E#X$B|Nn;-Hds z+IB(ETICr2;(zrsjsDrsvs`9Qqra%Oe-X*^)obc}sEY`y1{tZ5ZfpTY;DL`Fi zqnr^zBXUTu{+t-I)72nR4X9Z!Vs``NX^GQzD?ix~biFtX`-l?UTy~PaS$L#02MiL+n&eFxNyl$`O9kWui1JmEG=SzUQ#P0ltNZ&vo#(n@f&`7 zQAjxmD`FjBpQ}%?Y~@Y_a%7IyN&Q^M-XcIJcUY-gmn^@qQQ!)|LkDFEG7luiN4bUN zuup@AcDu0fr+2l}$xy`0M3Fn4if5Eg9}!8(y_8b&L77pjuev}opKQ9j=w?7z8Jk91 zd38KeU{pGC)cBk+Mv z>=QGwF9$)1(~e7x!`rh6PWEIKTeubb=u>7w`dAw_fIKGSsQrVIUtwMh0M>7Es=QELyD#knuHQUZOrpt`Pn)Y;c7_K}3(M<7T z!6LKSr#mo%Y@Kf%sd8<=6lAhD7RZGMntAA9WyXQ%-~hB#C%rY}GbNh0*P$WkxzF&& zp?P{)oC&$VaHR-Lq+dQlq~{xc^z2fjyF3#d*T8UfgA1|}dCg_KcxdQ3Y^ItciZzln zO6cXzTO>()xe+^Iw4PH!`#ku!-G^v0crtAzc5#_6mq80uX+33m-Cuz1W>I~Tdryeq zOFVoaF}(Vgq@k01PX*En+}-RM^s#| zoqcy|9O#H#rp`v*rY_Y_izGhc>e8tB+k*SokHU92^R0c`s(@a})^t=vlRCteO02(5 zu~G3@Q;r6{S5BVW*-psW(Q6C$G8ES8srZsE`>`sw%wP^QkgW!qx1Jw@zSEk%j0NqW z!$9M+zF0(niFsfr~{VFv5dMek~ z1Kw}j7s@&VC^Do%O?8A!sBg_=lAL;@ySYRFP?=G0M%Z%Yl(BL-VeUzR0IY8Gs%G-p zp>ugawzC7MFcwc(sv@}6x7*{z>{*GR?T$NNDdg-{pKio3Pz0YkG}c^QUWQl11LpgB zI-2)4Vq^bW*4gq8NebedscEQn2+D8smq5T(cVHMo%eQc-m)~)FE%F8sHNe)vt3Rx! zFRI`(9fo?q~i-+p_L6Hnq+NzDb@pJAZoX_W?H8l5hG#!e1w1Zwm$W!w}xDT$lIFs(WPuzYWu#p!~9WHim*Y>^Q z7C2Fp-)`tg?(<@vex79HoP*y)NuJyHii=F(iaS2pZ2JUG3KK5iU@v!Mn7p-I`YqHz z$qrj;T?kZAMuC9Qg`xL?{QL$5#b(u>%tmg2=yvqmTH-Asz0OJHQqvy>VU54X?ME`x zC3gM8&&KXP1|B=D&&V0Q34VkXfg(w?3a#DbkOuKDsSsM5nozylFnVl(jly>Nd z?<2>)@ssJbseRnU?TdVD&Mnm(HG~xY=Oa}RKJ*PS`~ER4$AT~ex5f>~RaUValj)Zv zDjAxu9;c&^cF(?4D6Ri|alo`ErUom)CzxWZ+u=?v5(EQ?K zzBc|ShTq_o$*%`kZ{o}utW3Wrr1_20xLa4PBWRdLxz4m6QSSM<3W7ijAkkAs%hbx= z=WTQCSBJ?Xl%1X$8pd!{y%?~;UjUf?f}G9aRqyQyo?b=$&I|->zHX!N-pMd1!t{|V z^=GK9xMeH-K&tX(eiQLkpA=5up%8&0j6yoNVs^vIf8Py7bg43 zX0+1$Q#lvB}l^_N3|-xn2Jpl;enSwi6KDw^}Xm%j!e{%z=cQOJayS zYB}2lTPTcJpQPT=!z&Pvyc&nk)6|p8V$!nSk_IWe|XA%RTv*kK#ma zUnj7THa})bStZf_1fr@3plkYp#LZYEpyQSAG-iCGje8hC_=kEm(Sq~Xy3fnu(gk5h z66vjE+&Sj`RV|6@&EEz}XN?434PA;E3(qN`ZKr1uZasQxCmj^f6aX_m1^lrPvx|+u zl*Dni)68?V7BeD-F@1dNWq=KUxw>=#?vG4duka)ip3j-Ma~se9ZEYfEqr-&2+Pyi&jG3mX5Q!AbK5 zyk*Z)e>13jk44{!u$p%K))G0j)4nuR%8Y&rWC=Fg!<`1)r4w#g6wq-Bp@6>Eu%>)I zUc|bvrX(}rp7C%OhvPW)rbI$TVBr#S*MEeBBR+@Y``vpTrXMyo4QFSTdo_IAiVU_L zSOHD}&M12fF3_^$E70q=a9kSff*&vsYz3O^#QU_)wPHD#(=B{+ zntd-Yvh&Qvr3mNh{CYYKCa>l}M@>1&uZ*KHQC4HE zw)h1+P?fm13|Q4IMep=`-i!I&a{aHZe!l3vZF6+hj^3?VC z+_Vs6+(`Yd@>Onu-psC$p*rAXLnh!(Pko`QHc1fgfD=O)>ODN#*A_uLdMaxK1I2fZ ze`tLW?6c5S&Y5P(*~CZe(j3U5aZNcQ+HcL}BkO9UPtz@tS4^U&l`e7e5kN?5!TNOb zg@}dz-0jUfBKAiuZJ(T+(%`b|nKVWKsa>Zsf9Y$P7=l0gQdVGn+49 z2|*rV-CSY_IAg!NMmEMsih3?t%W=Ias0VlQByd5o8}!q~^r&Ez0h&w^Pr4kx%wDUW zAl^pWqCA%`v>V7<=qx1_)T%8%V>s1rmwdw_TwWqb(<<>v(_N~+PKwFAr08KI{Gt?# z_|j#hRMz9~X9rOB*7oC6LqyU7+G(&T98MTBC1eMeP(R=o*ryU0a(4q^W9a6=L`k}x z+H_51=@|DlKD1wl6e4^s@dvj`P=M2DrQ|PEULXJZ3iG#6=Dp+StUM%j6E>UNNq;W` zgxI>HjX?u-VXBh2pP`L#7&zrQ8o6nLk&G&S|(W%8SOy+UVRqPZ(HAYHH zF49&jTSvG*Teo-D#)Pn9w)C7Cd&3OYut8c(%hSy)2VHr$>hCIUoMI0r5OP3+)kv4T zvvB%#$<20+6{!*1-i!`$yb=9P9iqFEiwyOzYR-SpzUOD|B*bASoA?K-{fTzWqIIoK zbwpPn({@)bE9=2tvl=krsvdF<5E-PYQ+c!fbv0)w)mPpSXn_4X{DPK;PXXxGRr!Ga z>&%q2pd?y3TdTA~Fxq!xKC>vnmcBMQ(wrM?tS4Q*>T_3iLHkL;>;2zUB9bR6n(Z?W zxk=T%PjIgL0;J9!R$zy_Xr@8uk}UKq02LJlwMLpc@^(JQCP;a_r|$$TrkLk43U0cK z*f{6~DZ??auMKEaNqz}VY$lK0s34)IQ&!{GfTUT4nu~?#s;J_IeZ?4q8p|ful4Wl{e0=4Y2xhp3a5Ss4X>3)*3!+8d9q|^uH(k^>ZRr_$Y|l?W{dtD4?!+9S$Qidd92^+c)++4 zWgz$!g-n=XjVQL@Cf>icP|A_Qlm6mRWpUFjdfvTtxyKtE@<$;8&7`3YuQ87#x{`qo zLP!vJ%INZe^#&J64W84Qgfnv5wO1ztb-&K^!{6aWXKvcC0X?(J!1PPbX#mF=g(&*! z)()1WB%N~HUX)9DDGC$?$7zC?`ufkV8f&=MYaO6;N*B6e9?eASP_Txn`QBO5a&Z94X{_CEPBvXxa-}k3CE1O0B7ZqV z9bb1&mNUwu=zhNOcln!i^5^Y`Ne_)aa@DJt-s*i5_WpbRKQ@l&w z%7@3tr~z%PGAzkz`RKtcV171Z%@0(AM4^M)6T*f+Q6k>*I|lQI&j}b$9!`sQ%uz^P z(9ycvoFM5|hcwHb@?7+E^~OYF#29pm(PD?ol|x_1&9NU53N}Jjy|1duj?pyBmKJR^ z%;v(KRxG_0tSq7>#wr%4+~6)?7aX+fdn|>eE*`b}jag)LX>Ha2tyyVzTI}PdTZ+~* zZu%N?*}B%`@|D5jMgykY@$Tkh^&Ao_tjskdQ{+6Ok$BtsN8jQCs?P>UvgJVr5PjnaiXM2Tzgkctu}yS;SXcFd8+PBxni)CllX2d zm96_jnQiYEgfS-;l{gphQS;qg6pJzAQw+Cep8r!;l^dE(U?ex{gRhybeMRWAi*qAo-nJWSfA5sEjGx9jtkkS0Gu z>R-m*AguJCXS+GuH$G8wY^!^5?dJ0~`x8@=DkBj5^WCe%@;n9^>b6KHZtwh;T)r-m z?Wr#%eQi?QB=l2EMM9YTPu2akv9>+uszyHBiee&BHA<#y~n zD^6#=SgS^jH9n}=Q1*3n(I1S&^2#UJEF2yz*!H_87d&abl_D*v(?Ab`VUtb_O ztWa=+pHz3Y=A)*;5sdG4+aso@nAl*urs^J9WuXxD;F1do-$G+)gvM@Fb*xBbeQ4I< z)n^}=T&817fzCXG>YR3TC`f^Ltws9UF-Ke z<-k_X2Wc#Q_5(X?w$H7geDK>3nIYbcpZXoFA^Be#ChjlXcX|jRi_+4G@=f%!HzH|zM>qhA^=-Z^4*J{ z{`HZ53rGT#Lw66)67wbmV$qi*Z_mhL+DUU<-Bh4)M6l%(H39Ay45^yp*T9*QW;58A z@n*Q#E!e7kIao~O<2UMF9$w3_pK21QqwPr1uQl$gmvlQvj2oHs?N^cQ({eFr*b=I1 zxi|C?E=o40e^n;xIQ+Q-hn&a6nBPK1|1h-C&bc(Ri%I3IDL|wv-|a)dZZ-*}BYDlx z(Q1x8L$kw}_8ldRYY8DV;~uM2=_=(+KNVylxm}$@yHC{LJ!lPgy(LdFKPUjsx%ljm z=XX2-!Hvps61g;{;lz-5%t^CQnc z4_0rZKL^f#zRM)P@tkowU_N~^;A)N)IL+rNH~h3tW?8US<*x+Zy-s#VYcqw+StU2d zwoZTYU>>fD`cBjA*M zXvO(o0z`r zyh*RDESWv$hc?xb2}TXGARRztmLh1A!nKjTt z)KHjA^~3GLst#=i)4&!EceT5foT2BG7@scb@WZ| z6$a)jIltCwGUGnQ&lB5V$y2TiD_bH_X%Tr7k%yjlhx%q09EDaz1k|TsR3$&gWh#Eo zPuYOf7Cx}e)=A$;e>zxf5sTEzIbi0keOfBvx@lRl!E0$qTn~mfv`(cf8T}ePJ^Nf&;jYROtOC1@fH#8;#>>xI#rz<}`dR!@Vtu)VX9)@K|t@g!&{KzPzx_aAQT_h`}mxQUW8#Z8w4lNMDE*j1- zT`}+tFYDMNG5@H+9g-0xS8?e@u}xjK2kzw%8dy=+ZSEbfK%`^>?zd!83| z|I`BT-|z9RAKAFw`!hy8c$wCaMo~EZ=PT4#TJPM%pLb~(?(4V+^I1CPHFDPqn17Z$ zwx(>rh?CykN{kN5Z=kM+7ZRpW7Y8K<#=VzA1#?}$zYoJo>=;iOeeis-c&xCQ?n|?@ zN+rFqkRBgOU2&4qxO}W|HhHyJNP8KXbt>Ifo0RCSmPZNxkSnM7u}m!9HeaD`_{ldW zW~Ch3R%LOXXZfP$RpTGr|43Ng zfq|s!ytH`PiT$#SoXQ6?7SoTZJo(4({+XP|fkb6eoyD!Tebcf!AlaF8>HN2SGrK(o zfTkdYpA`6u{QUdhx}y!Qc>REJ-z4?_uR96C@&;+|n^5J110o}Hb?}GqzW2(wcg6pK zhmoT6dWPO-Jth}3yx_s=uvotQa>tNupK|J{Fa@HkWxhT>M?X_P?3UXBJ7vyrh0cJA zuRg0q&dC)v*~{44<^@cFB@=&iX<}kxBlhU2eYh4H?v%GvzXV1chI0D#OCBG$@2+fU z&}+dS`^5QHT+z8>4O{f26y?-?<@qn&eas4aj2`@|d9IIgRkrfa+~l-09O=25S)mhe z^Fo5|pF8qi@!)azzu2Q_%yQok zll#wK_^)aG|H^X+ETi>Z9j})+X$}ndO$Y$E&&WexcZ6LHd(pbK3^+kpL7pSFL?FM| zhVAzQ{nOQy56DM|Ysc6G1Iv#4Vsgy;-{%{{D@h$s{=Ojo*XlcQ{TR=$%t0vKp_gi$ z&F~sdN7vWwCS`pfGiZUg$CSKgYBkc(=$v$&xcTR%@xOUBEhWIt+P$bagym9uUvw^v zl~KFney~ErAzLP#zMDaTxqU9iu_%9}oO#q&^=}osfB&dpOv{rVV&#p=*1Ie)ayw1f zfOGj@gk9I~zcW~+PN5Wrr*U$9?CAl&ot5Ah1KXLwra`n&r{8DS2+RX`T$0$Wd$;$y z;6H!iUklZLp5%Y+i~r9&hn5d1gaWf_$MHINoctqg+0lALJ#b~*!+HmSZ_Ka7P1kfGbU*FDl`9erh!j)?d?amy{ji*09T`o4r;BE zM-%HxGZfJT=m>kDywRomc=C(#K$f;W5JHmJbf}*SC+AyrGzqTNNSBy3c;RO{Bwi|) z+s>dDo%Sla8zV1a(y$XU8?-3tvx?3cY-P&J+U;)2?>dGo;G~Tk9vbu79YDL!PY3m!q-=pQ z;q5J0sZ|ny=)O5)NXpFkGEh+%tsf{ndU9b12&0>>!IR({-?Lh_0P01=14S5YhDMST z!__w^psF~yirD1_db}e`y_&a~a>gSM7wr?;9Ao=-2y7!x{W;IAf%@)?7@_*naM^N8 z4t-|NC1KP&+XN(xH8jGJ-zXbx zg(eh|$0j3Ga^tWBLv>h^n}&oRFkro;xSH~Ip^Erx1eAO~>eqO$KX7ZWr~TQy0MpPY z{xHIP&2TPwo;KVbm_X7vOn&(mP+J6E*XF*j>oo*~Y-Vpi_P}?j1jHN0>YWWWVD@C? zITdzz3Aht^mVRs=0?d~k5VBg;2@nm5=5JTU6j%?p$CWzvd=grEBcg%oPLnSKCMU*m z!s(m$CEVw#1yQP+P~5=M1|YhOW?Ew|Stfh`bOz;T!tgkCt1Zcyxngmp!1#8%1c(Zkdn~x?RKu%5m*da?QS=V0N|i(&4f5mPhO#6 zm2$7#Ws&uDn1*+V(i5tcLPlp9lcYR~fb`-63dw9mr@Gx^Ub|o|+w78b$$kUo@yE~F zBE2S6wX5jG0hwE6UO>fqKC?fdiBLg5^0AG@T-JDWwtTY!n*B^LIivi^0Dw$bhj#S;u@w9#^>w53b!;T zWqwAOn+Ev60DPFRS({$Q(@oL;2%uZTbXBHh;E25`Kh>yhyIIIQYsv2E|KmM*Z^?J&cV-87tmfIKtBF#C9oz3raypPrBFY zbW6AqPlveE;94nI#QHCYX2n_wr0wUN`FWE0rk$%VvyyJ1c~a18B`i|X&GRwE2G+f_ z{-vE$ewciZyx%@#c!))$S6Y8|aKA~$`PS^v?Ry%lOBc@h{w0q0w~5H9bsikFo@-k` z_ZdD10?YO?4(}WPNZU6ArM(QAp03+11^@4O_%FE|&bRLY)7I;1Xv6sXSRHg$Jp^Vb z%qn5k~OI zwm;-|M-4nLTghvpwEK(_g_P@iU1VuZzezIv`}x=c=uKMS<#UyN0X{|*aI+8lQn!q1 zeQUuu5-9Qb_PW0u0A?+V?nXlZY}xZkZiz2-961Ge^;?ns2KSkiBbrHgR>#RNGs=4K zKzz$gGc>}EN+=; zHfbx&fNi^5_05mav_)&c*d^L8Jh?GnR_8n(00J95DY209zmK4&3KsxG6fC}dRc525 zG~>X%)2waZzdH^kd70aT73Cf}j&?!NsH7Emx+LS|XiQvwe&roVQFN3hPM3>6{K81y z8i6hW5vk%~l>H}D{47$l6ioaZGn98jlevW%Ccu(V^!5+WWjRAhOt%Aa1K8G;-rHgAs61E;zVlwT6To6E7htXOZ0GW;qc zvOi0kD_bxOx*Imn)*XB#$zp@gNO(2hfAd$qlfTLk2`_IF!xyvknqP3wUvMTHMy{u9 z879ahLI8H(NBO?e;GlpyX{ZeEK%mm_A6p)LhE=$^9wQSCeqQyL~sMTDBI&0Pbu zv-V>XKK{m+2>+HKY`K1nN{mkbx&%_Q!*&^y9C&K2AOD=ble>NG2)wx$_cHPtth5{ZVK>mgkG-lN(Gk%8HQP%AqV67@!{v1;&+wAjIBt zXT8I5p)W4LZxuVX)JV@=B_(b|-fAX2Kn;IP8JYe*Ugb-u+7MU**jT)k1~AyXlWRga zpCDo#$nK_o5;}PQ>1Htf)yg=mheLPjO!ahhQ%uT*OIYsmO82G+GUCOs3) zaQUDkyVpP-B;dV6bGYNqi9u^5`+`;VD#5XSE_DT*F~*LyfzKoNT>+9--<-*FXe0}+@}LKQ1WETiuqX7_|=Nq zk^}+YK+s#!sS?gMd-%vMVseaWFjp@ZZRXbTyiVSCmqKW+qAqQX>`{hv?J*%E zs}E$Ng&%!+=DagGkUugvJkYiWpjtGk7cGx?Nu|ATI?=U$(5T}N3Xq!G@$UOK5C>dA z-JCNI`~(zYMU;jVHnj^V?b6GPY8l61V9*(I!KU-tMBAtjx>wqHu&A34bP_~lk{Z>t zBXlcQyeTJRyU#DkMEdQKZcCG300Jpt$Hs3Pd4Lxa6y`U?1d&pp9p_KIXA$|ey;}er2qGHEK=rjkw$OUCXI6spt2=?nzesce*6o2-+IOzjQF;cL53@6cb_L--OGtIGvZ3>r2p1)YVt zj<5j9u|j@Ir)6vsLJQMln~5TL@S;MS+<&ChQBggA%p2EJNoqUzq;|^iYKzc6BM8nb zZ?}56o5Pc)4<~|BRtw{jKLtD%VN<*B%Ps310uUFkwuh1%ZP>7Y(i(2LgmQ zf*^7ktQijD{+q#>_Ydd=-hvDRHhAiA2Ca8jQuSaR*(rR?(?lMG)VXj4gL!cfLpqCC zwR?;d6ictntseHE{c(!JKb!t@&hq5bic_g1rqgui(u0183oOL;D2{qyJisXYF6W)Q zOnY^&yD0rdWkaUK2f!nVla)o3z@m3W7s}+0*zBO+@x)UykpopMNX*P}@5K(oWs@53 zbcy<1D(h%ocaorsgv3J%wPSaur}*(lD&JG$7rND9;w*Af*NCS-?9e;o_kw1&hiRxi z&au^2<#r($FcpR?^a6nkJEUHnTlPpWNF#QiCdNDpv{1 ziL_%h1a9(ET!J_i)g{hQBReml% zI%#TRHer|#D-Y57ebk9j-COtxjnWFHV+J{~-fAniLX4@7>?W8Xj;3eKN!5JJ$sUDI zw8p=v;kL8Xaoa~`#L{5&Qn<9IUa4;UarV4f6#(ZhFk`(G=m*&~DYID`*Z38{-q#5G z%{adHc1xtTIgH#yV;ofTIesW#sJz!CRN)a zn{Ha@VE>XH@xu4xZt&Bj)Td%XenrwqET}fUmjZU)k-m1 z0EGHnj|bbL6$LsAEmK-WE=Xq5Fv#!LeDC;0#H5*qHl+fs=*R>0omFeDUykSRfSrStM1H$imvFbXMc(Fk>%8Bg zpqg>oclEd)7sBz#s7<(+LRs||#$jbn2(WXQ;L?fEyW>8y7Q5eRmrbfY@9kzY<BUW<47m($GvylV%CXpU^D&N?UIGeojp&Y)w%N^5Kh!UU2lGVVLBLV9qC6FdPhm z6eY{p_(gD7qYvnKNF1&#`&Wa)SH0`YwFK%iITa@m@_udozL)p$y6ZP22*sfvs+1KA zCg=|0r3~p0nJ8A{l1+n2!LGyRKhCll%mm#R4q2d2ut%;He_4er<}7yKhHV{Gh1(*n zv}u~VsSWFmlz}b7P^%67EGjB&KCaW;E&g?C@p`kvJw`GI{U2P9uta@I9C z!be{HBUOb4O5cAIEF_h%_{ahyQo~uQl46m-01Ofx^ZMrtRzZCB%hWa=<_ZoSNa%(N zzvDgc?rUJ*jJ+++i$s9z2F2&&%ftOSdK(tk+SX7BtY151yHIX7XDbFguvet$QJO9* z(<`u1MEqqeisfGj?6C@7iO+eLznKU5g@!g%*UfRD#^nzz{FwZFZH1P}Rrh070mj^K zyz5nXOpr=Oya3cZ8q4jS!suWi85Y5Tv+dq4a~K*dw704VPgBmeNR3sR)fdL-RLXVS zH!$Xe(+90AH{249IUeETqWNQM4Q*cfk3-ObLk|OM&Qz^Q+m>Lnby=Y`J#Vn$j`8HU zQcw(96N$oW`_D}3y|tw7Sk`?@H={n7!3!*<-oJX1b-+`EN18I*#zTK(Kqp_fw1DYm zkNWs1%XczG-5{MwE8Mgrt8O93Jm5vn#p}2y@00Jxi(`2lNhT5F6=?MN97! z+fX(G>+Nw>B8hD={ELUpfb2eQxu@NB26BofWvZXO5I(s8I$hUuAYlU413_SG2 zIOxDfMC2Etj#j->>tKFt2iX6r)6wNYz1a}9A3*E$&D~TTNmZT!s9LC%ShLjZPQr}< zUc)RJ7S+%+Wku7lQV#!Tv8m^|;i?ya)@0|QFYDPk0yE4a zl-;9tx3$LD!*kBhJUS&Wqs@x5Jt`oz8$)A^dD=O$$F>E7biysF;Qmna!uuy!dNS|{a$eh+H94*-^1)agR}L_ z=9_A{{0-u4uRlu?Js-3G_pFU9n&tunA&DeaV5KmIdh6%>+RKT{PhdW!K{91gi;(G=vafmYcnkR3&$a6W5@=j>|$zqfo&6^ zi_w?$VduA-<*B`08K=5vHLOlBx#&(Tg9vX3@>;bzE~9Fm@!gOF`b|(K?fqS1m(Vs# zXpUM{I*P|D-}Rz>Q)*ChLo6$w7Y^KXNmoAGsRfk7&<{B8)MIiZG#Rg`(%~*-6FCMN zxuy;;(dzSOv0fv+yVFr{zv!-Hksp5HLQTw?b0s{4_FOr2PD-o|woI+xC1=tf2W}BFLem|y0fKj*?h!lnPRUtDm|mua;}|m+bwr-=hkBpk ztZAx6>qBs+hM$4Vh*G=hnNN8pGHOxBcj28>a=o)_=`**(X;Kers0*^`Qy*?vl#h8& z%?QfgrrDhj7OGGDRrjFyeUiWH3bSF3xtX6ZZP3SIgSNL9FT+pohH$@_c5Bf;1W{2k$Hg)q%#@5zMXJ3AK|kTU<7wI&XQn zNl}`0E8*H-pnIY4Yc>-_bF)%Xi7xz-f=?gnlP?F~Nv%A4U5s^}z0pim6>QG8cI}H3 zXkm5UX8Gmol2)FMwi(4e_#<$K)46H|df{@qUyfPz3wf^?FB8d)7ztAmLa@d2W;599 zTw&6c=dCRUo$Ke)FLcMgS-9}S$N&0X812#G=44x5QD;a11YMZd_MwW!k2)?Z^K1|Q zd6-{fYI#z6+02HY!VD>8{2YGjyxq~#LzaRXan|4Oz0}$cS;{duh)*@_JC|7_ z3;loWy=PQYS^NL3D55xs1w`oY0CFce286)r1S&wj!^-aaNYYXDG8#Q$Aw>4XVbZa-? zw9d(@`{_6(he4L{tKhpL>9xlmr#`&agA^|IQF2XJkDIFXS-PuUYlFUXdPtH5DWcsb zoptZ~h64Ke&+BAW(&JmplC)%)CruN}N~ZLYWe$}>ed29c{`kedY>YK<{@tP*>6-!@ zX6oZtNO-H1)jC2{cPzR#_VeQAk2iDK@dW4r*Bi<;J;8aQu>VSyj?bPIx65vBVSM&l zmh;hZ<_F~mla^&CNZzE7uFLhs)nDw?8f594(E237##F*57Os`p&jWF}u)elyPpEw! zvmWl_w{_#St6SJj!+~?b7SD&PH#mDPaN3UrMQ}k1n<=cGNq3s7bP8H0@>h%6?A(^q z9#v$Da!|p+ee*nOTeAhMt6P#mx3&bHlwN?Q>QlpnGu#9?mvxJ6yFW#UF7?Gi&tFqOg~FZ-EkV%dN% zEomk5Qb+!UoQi)YL^)kSnlm)%$<^tp4JsR&)+_|3Tw zg#XSr{rU6}N#I5=S@*vGuX+4$JzAJP$PAwf*5sl!=#B5sTy^<*f{W5Fs8>>jp1(Uh z=!msoGoxW-or(CXpsf-j$n^DkU=r0yDNeZizr-)9b$7x&e?24Qqk#Cu318j~L^h6p z>R&F`^0%B6_uiM^*oTlq!V`-;-dtfA6SVtN?)Z;a9$0zuqXZ;fr^x92`7zs@>?3`z zDm&(iK3qwd=>A1my?-D33bnt8d;~**_N|o04Ne1jPaBzR+2hU4B%_p=1}|-&fZa*D zRrzaf+)u!be`|GDP-&(m2cqi2n($t6leoOzj^p-lf;K&I9V_LO~ z|N9I7^WFWQG7kF!ND7(QJLr-&_RppDU(@qh=7F22x158OOWK&jh^{z6`$V{dvi)^- z8p})GvHxh(fDp4PU$4rt05GTh-8g@}Vu}9-Fp1YT+JYJLVyWBF!z|NQrego&c(UL8?Hmc|8AE z9sc*Pf5P6*_1~lME4=n#A-3*T@7JJp!T;dm{_ijR-*NcYg8hHYj4Ho7i-3&DViCT* z=I19sOEeb9Jx*ZDPUxXdD2Iw4r4ci{3z7$mP^$M214_wWU9*_+ybN@`3<%2Z*R7)U zq8H!Z?je<=!Zv)QCSC|{`TJAn>s6W-^ojJWz)BfkjM_a5il!$9YgSwa3x=}xUE7)1$?4ovy6c$cFjpUmgr>b1q-@z9; z1sU#FhfvUk28Q(u<@O5zwe=8B99*O;FsLYl*ypHPkBm2=szHtFAwCj*%dAlanDC;y z*MPIMoQ2n@OF)XFT_hp(<|IWLi;ln}M7ktj6*zm7c)9{zFk z!>bBvDFk0>TTx^P$MfjT18Yt+IJNo??{2p3tzbQuM!}j({W(P0Kn(XtueoDU0jU&U zq|Dk^i@`clnQr>;^l(R1kx{KcxzOl)aMZ$}^_a^$V+66FZfpr^;iB};zNES@*{am9 zopB;O!rTS{%S{wV&`bx&DgAy2Hlx)ZZjphIZSGJP5C~S7CE~A|B9j94fTqifqzXTF zru8YHuPl&Jd(k-ee#ecuoQ&%l$^J8R{-DAsdb&~upvx~alh8u2^5WeJT z?6g>oDX2h=SoNmNTU`nO4Hde}iNLtoOBbXYc#Z-}ijVr8-Z3ByqzomTPmb2Q?pm0p zUg!vg08|;$k%@g^?Ew#T_Ml;nbD60~uBz$P4%1p{!pt@)i`ma!jAXz169OLZlvDLq zH_+xC8&f0c$nDG7xK9*5AshwNo3eW1ZV^zjMU9C;b3lS}7}znG-3vwju%4G&{|3RW zgeiuBB0ml|Og_3iTXcSwF)2j9gwdGHac0Ge?8I?9(U zENF0RZ|%)#zOZ_c`O!tPh{mT(QHSBiZfS@amsB+z%(2?EwSM|RE*5W~D%;nz39Ik+ znrZzcz7auM@h1=9fx5$zOR^Hqi&j!k1H25`qs0a@k8N-L@r6N{OL$jkBTYt9uz8rPk zba-F0SY^*P>NO<=oYzs<7B*9JM=UoHSy^6JqS&S$@PUK6uiXVFx$Q-0OuS|JslsR$ zCGIwmJ34SnL7_@x(H|%~6Y0jwV)ZS~7I0$!akTsQUjx1AYzCAf;Cg;P|gYc_L_SRbqlD2j$jetI9f2kiko! zYZ1QJsVZBWG4`PL>ln-{^LQcot#-QS#u6-$qS@tj2^+-PeM@gU?skv3=i*+trSh51dfTt%(4237+Rxm=>*RxNsI_CZ_UTY~=Fy-!^P($5a9 zh53@4FVL0sfZQUkq^zPy1=|m-m!lzYX zbNum^eV_OMQYUgNEcSZ5)TR1AS{Soaz28{D=J`h~NrWmFN)gMA>ogv?vG4M`s@sUj zY=uunG%h(n*Kv%KRRR$`&zSY^Y4X3Pn15#q%8mDP2dM|wX`Of4c|msT{GE^Pw7eVI zJ;4Pd{ztwA&^qn}@1KXhnSE!abpaNn*sLk%<@acP%)gxm8c>B`B|RF?5$LerVYFy{ zfJ&>W^wy*O6bRE5iI>}yhXtK-_VV;NrZG8U{~5X0)9+>m{_rgbkj77jrP8P^GntuSJ9 z{8(E?N*xWGid{Tqo+XMo3_k3PhT8|Wpfgo z|Lcqdh6$k4Aw}FJ46*s6zB*v*Ad{lA%HEmnpV5$-G;+CL(K#TaD>6in=pyTPwzyym&nPSi(@hU) zt(bTj-}m7hWRIH7Q>J@_g1NguM0M%=3F9^pY4Q7RPFOFzS4^o2pxS=~uu%*=>&8sS zsk!?}!2+9Sk){L_phjsP-GYyVoS@$51yuo72DhHJe39YU*`41yFK;Jy?h*=Gy}A=f zuk)YOlRuLK$5ULyIpV`y%rQFq!c3TJZHFpK;vw+id%DbO9C)0!$K3zQQzd zHJ*#TfXe(q+p9~_11F|bmbPRxiZ)7Ttd{r9}U(*k?zC}K3 zAsg5+ybX|BeZy2q4FW2w5Lqc4aeipaMk(&!EC8HN!JvM30=iJQ$WHjAbfc^5X|ENt zo~`?${E~8K7TFr&I(NtXrTPKb1_IPnuu`dj=#!6H#9tH2KeeYf^yX|sJj=;j%@REm z7JOLbGu5QoJ*1LVs&jlWPR@w!@AD5&$M-$Y2NdI?Ky$~)(@L8UBCi^J$xgnM#j+)f z?OL8OxtSy_qm$d-`@F&KMEG#G)#$VB+~s&{&+`3K)c%Rgd*u($Zc(D0s!2t;5Ys9{M(*zMF7ZT)IVJ!kY?s=@s z^Y+z^K9*{bQ?SA`7~_!+yY<=EyEPt!0cXcKbIZ;T1G!DpQ(ZY4P^`ds#LwACS^Ci}suKPH_WGLaaKNYA+<3G&zTts9ZWmTnpcr`@)(C*3l7rQbZ`%D|>;52Ji+8(=aR+!CkQ=yY zSY=nV4~N3QA&W+F+iS+Q!ha+iuBzLeCjl;Xk?_^g_N?F$!nSJ&y4d*AQtftRK*My; z$`z*_RK+e;WVPvGICh(qIO_9sgdQtW8phgHyXF=tSNx<30R?UO$eqBOZ!Dvqj8%<+ z*M}oXV3X^!j7=lmhH#Pru$KJWP(zT=+>!YS>cHzJ6J!fx!@9Ov%=tUHB_aYA#sQ#% z=QWBsoQnb|jP8rTvo9QQpm_s~=VaHZKGO=RiPR{>lUlTxhHJcREti`Sm&qGu{M`yY(XS1P`6 zFjuMU>8d14_EQt}Mxo>-tGYdc} z){IsWfFABnyu*;D6Ekql6_Yqr7t^I~pybq8;`R=bLsj1NF0z-|9?JyK*!vs(kEMg3 zmH6P=O#EDGgbb@Y2}31|yE03*;3!cU&f&I)dxhMa9jk5_8h=f{3zrulZ$~9;@EEYT`Xj!RcKf*AYjwn|EDYu(I}kT zdZdK{`Q%MT+jI#VV_J8J<+B!`Uqa?P(t8idU*{BrB%JxZZx40`GHMY&KbAI2wV=Wu z{{zd|*ax!Cw-GJ52jYWJ_d?!w#$EE=rwq0~?cGPH)^AOB{YzkVsB}Bkf!>wX9iYMw z3R7EV_7(6v&YF$Hir(z)Q|wZnQ4}8PEhop*jA@5ifk zuu=s1aVgk#n@58_Fyjy&W4e_{CXUSn`zAbaN9)Ld#88&9-nLO4XhJOqPqf8D-w=a> z4wo{+64BiBRxArctGkp*m>Ff4!cgzIv8o^d^Lv_i{0_FkZN}3-Hor69pT5I?EwM?r z%FdEPC^+vlZHKqWGx{b=g{^hpL^AJ-W_^JYKUCmH(72ReLMR}tc2WodhgnNci2MsT zwDvD7WCQC*%2K_A@T>mbc;m(3Hpat(RnK=BbR+ihgCws+cm!U~2=vbS`th1zP5gac zRLJ%LL^yDim6?~bg&1i!3b{>RF9(JI-Q-%}Gq4m;f)IYoQ`!gG>BJO84zU9gdvV3km4-VYmo=mClE+%_2VoTeI;f z$s<^9%tPqvyM$RIo#zevT=K?4V$c%M=+?_%VwuOl1*D0A`5S>h07JdzW_=S#n*}{0 zu=A966mI6IpNc-4l>`!YEQ!TvX>_zcpUD(RT~B z?Z9@_VIkNNU?B;?6NLJn0??DYeM=I>;#a%YGY^_sBx??kE(-!U7m69@+?*vVo;O}= zmB8RLUX`)h2*JiCvdQAB@)*tl>x=isUdBA8sjZh9X8oKnm4RyZr+-!sZJnCuW`*_e zvy;8SvN7BQUEebJKz=)ZuE`YVyKbhIS|>66k$q`YAEB|sYQN;-VcH2E43 z?U(+o#QR8x(OUjPW~2v1S8n~e@Ai5va>JXPI%|I;ev*X*Gubo#6wtp;GzX~F9h4%& zqYcV#j(EIxzy7!*(S=e=%kIcJVVPvB3r}-}{U8ffV7SZ3aQZm^3 ziu$jgOJqxL_iGs&Qx^#-@i_$r|8r)#I!0%}x_;IY(0`2%s3;6MYo64UsUTf4k&zO7eCL<${#?XKMvwo~`hBE_Z~~l74gYfpfAu#%!6OUEz?C z^MD9HE(7{GPqB!|SCo;_sYYi&p&_#w=8MPtZm~`d1yH3fXdr1;G!Drs`)sEsr5`1Lp;NPXk^Fa3*!Ud)7J;gv|q z^o&eAma@>_TI97h%@lk_t!mVr9{zbTO)E#Y*t!HiaI+OvBFF+-)VM1rg2>05>0j<< zV#H8;{((|g>`c+C?mEAV&%x-4pRPq)XxCMLD$pMg9Nr^OVnX9g+1q|IMXGMk<)5dI z@~Ha!;+PiIke*x0-DWNu5LvGi3F)6B_VT9tj$J_^q3ic9^4%0x&%1Mm5zCD3H_Y>= z$^=as%4jirCL3u${s{PdPfzdaY)01lhK#&>p=XPe37K)lgMF+w2-mn&J2O1= z)OE22i&||%HPkE8kkPfvl!j>?XJh_$w4q588Kko7CiO;F6xUvmn=W@h1WoX`25US0zN{FhCzz_Xoo+f%=Fj?QS&5|iO(bC; z_~}6rQ#N|mos(SbcJ~oOewrm3sh*nfnc&R`klXnfQ2#Twgc}1amOhgj^>}}8&SHtxW@Fk)Pj|@I zEh5mh5Nxv-(^F^Aq=Va%GPpV|OeEoLoS$Y~d#dpx<-kV(OJm_Ii|AVp}h!|}4o#xg8@KJZ{ z{mA`UX|3U%3~EEuXJdMXA%=VBiVX9IUJ^qMd#Yc`kw5n~FRFKavme{|+%=$nL2?Du z=9ri4LXfcI__p7D+{;xFZ5}Pn7eW_OJyFNvN@o0V)QoJtk(roY2!jl{tD_OOHd3`;`2cSUq>jc?i7{$)$fjE?=>O5U(e0q(|jGD*z@Bs9sE4; zkXc63Ssk?=7arPjj-)#XAvq7c6L`r->Tw##Ew*h~ z87(+eeP5sob|+IlKeG%YTD^w%74tw`iE`k*yucz#O$=YUdQE(1Dz0LE zfUk<8eXr?DQGJjMA+IPCpJmbNw?L;va4d1DglTdNIp5BUqLZn6Dz6~K!^t?K42Rg7 zq9s9zOXFLYj96m<%tpbvrDbzP3)gXStA&qA>{D4>OywTo%R6Xsr^-EgnzzQ+>W-dBi+B z59RZ5HBc3k=r(?tRv;dg{eyGQ<41Ez2{-F~<+#dZD(o7|4b_An)r#M@Vjv^stUim6 z^(5vtr`6V9!r!mjd4GC+M$UsQuMWX$EBUy~F&LxMqnYM55|?*ej<`_#yzwlezYa;^JlA}pWJBO?N%!g| z?c62ikZ~{Lre*{e)^Ib*Yn+|?_Z8RYah%cTk3zmE3}t$UsYm9yQkd8-@^U)i&B(Mb+Y|gpN10S*8Vz1q? zUC;L%N0cmH*r)2sE6Hf)Jn3M%Dmc6~V%?8ZVS>dg@$@em3fyaQ!j-PoOrG#~U^l|v ztgH9Q=DIWbp2D!UlEpQ-t)EBhCnMU7M4(I_pC;c%h0NEkv^Fhens^7O>O^E;8_&1M zbQ;dL2nj;UUOShSSag=}0oHHSGtyqrrHc{X`y}#wo#Z97CpaRxi48odIgu<`bwV}l ziE9&adA(*P%V&q&HQ?RD-mkA0d&f2^Aju(>A8nzr(X72@pFlTSXIs*mI%J*Zxd^?N518TVOxkY(ha=dW*b#|oLY9v1nGmFQ96m-rD;(Dxt{#0Vz3 z-bnXU;)I_|Z^jtpN40N~kM}N+XS(LUHOe;+DD3(Mu`HPHrH##q~t?Nf!% zi3qannZsPnN1><0lxiQ5qI8&v%QqNgE!++K>$s=V(!5-vpumPbXDXKsRw*r%@kh(! zK#SeujGxvDPB6-f7*>yxjL063^%ScP(R1`?xzF>Yh}1Fd_}W1*XD2*J#F$o#QAs}g zikCyTjj>UvRxKTfWB6;oE}_Wbsv#4Wa}e80E~*;Gixffs{wkf_E{ycYiMX6O$(!l; zTE>u^$9aRs*auCL#`I|}FR4nef%nEDCV{T&v8-vnYxI*hSolD&x=cN&i24_wW0G0v zk(vxPBO{%t1HOinWE0npys|eMB9KM6?Ck7qySfPGK3FR%m7qb#XJTfW*UoN8FKwPl7U`e5 z-=(}-mx~qd@Ki1zbxv9K99+-6XWoW(Hh>mWHKr$sH)YlSHSSfTvtGbg{iRH#KN76h z;%m!0`O53{k8OFZ84cy|f2|h@`BgM?!Kdru9g&63>UCEw`8zm- zr0Qfr;?xt7H0EvbmV=|_UMK>mGV-HSl}-G$u%OYy)1l70%WF0#^}Cd+3TrIy&+Pqp zV{q4Jm91&BUhxH6^ag{A`9TQlhkVtov&&;_eU9U6xd>K|b6j!|7w*ZyYdtp`-sakF z%IrxSV>Kb4K>~IJtZU2TA1zJ@pG=EzTb-h`)sS8t!LW?$K&IRs#nE*X`LdASk7wS# zEWpn(YT8K~kxWP(GixPpwm;^ZG>0G-p-rj2OKA;1?%WR(jz=w4UvM(66dKUFWgIpy zb52ag*x;#2OPJw_sXB>tr*jGUyK|j=zW@93rtV{{|MT*cB02iDqL8rfm+EQm5;)VI2nnVpf zz`#<#!j1v%e@&CTd??)U}K|K@(7xk zw)o5QkN)O9-u@)bh_v40pwl%pO0do0ZP{3Ee#5O}iM`ujB}D(}%^7c@CH4o^@4GT={BkJHxcAI1uv5TK2?Nde`}xS1kF9F6iO0RDo1rl+mxIE@17 zfxy-MMh0n!3?YJ+{QsRAC~(T2FrlS?I{LvnFbT9fXBP#8v+MLDdlP9s)6r*d4#*%X zES1%nY2MJ={gUmIM+a&C$jC(inK7(BOrj-tJK4V9O3?a(JU*E>N zXd%`W(nr*?wPB5SUXf81M(xMn-tm8G%AN7@;RJM{>Y{O>Niwd}pgYgWhpjz7jr>(Y zZWyp_3T=b8t1Y?-Kxy}KbxE1}%lA3FbQo0i9lG=i%WIFH#e}7~*}u8M1eZFb>pg$$ zS@-f4!oH|E@vKVGV1(#VZwTUUQ`9eisQrz8-*NEjH6*Ro0|KjbY4!t+%;{y?KI;wWz^057VCi4_n~x*})Agk=J);#d>rdNUlcqm72v3tL(l%-}8@FHU)?36Y-(-dlPsJz(!Q= zQX?ytaFoL`?ImSECDWU&wkMedM3PBP9AZwD%_pQJA+KU;TmkFggw*DE#sV-Vt@2&+ z`G%w{`RehYmDfrP;j&Y=KGNVrALUiQMJK_h^<*x#9C$1nR?b?(-05FLaT(0d3cl~$nOYLsyOK9nyY6YR_THhnK%Hi}%BDb^-RAcBX$~-o57@Nv_9fZH zah{+{*EC68g6^_gFkh97_Db&4rH(RTbF2)avvL~8680Y zKu8orMq@eDRk0rsDvDetAL^p~Wypi3yQ_*I3!0@V1h$-^(lr1oW zGJ?oB&CS80jKqxik?bghRY!IGF@ZG>1IkWrh8Dtyyeo&B#ou*qseP_!@ zywZ6fP_t##~gSX*%{q=#}dwJt9~x)X_0P68tZLlX+}+oI&BQRGp#df+UZ0yg(Z zgP~%Ie&XGt?{ukdJ@E=BVP?*kx;o0H27#)W9syi|ZM^zBop2jv)sBXMmf%>Znv8Q8 zgHc>BbyapYyr!{jJ`+|qZwV7k>}S==c!+j~#0~pS@&{PS)VsIWG;}A5)laAj=;Z(G z)FdvE=7;LJ@yKOh9xHt`g#26Us?&s1$Vi2?9A&-#m~O^Hrqry)3}`sz7Rf5#RTKt< z#<9^_BLaFoa;|SzXMX-n&EC2E%jsI}5f~Llb(~i%H0?O*;HIG;-p_SHRqz`p(_Jz9 z!2uxk-Wh)wtuv}31gV**t2_wN2<;k}*PgPg3sF;w`Q-5^Ce)VZSjQi8a8n>&<^;-+ za%MR{IVIgy+9ATFKu5bLMAx6HI=@{1_qX!L3j1q6dt~B^Uk@oZ0(2HRS{ug3iFPEU zd4c`ZVJl2$swEuy3nw~o*Yb1+TXt9|moBL5(t!=ASj(3!vEIzwph&Fi*HW#(TPM}O zs+Gx|rnBUGC#pb%dU#fJ#<(qX#gAdirv$^hxI6QYmHxUd>9MYSnM}(j%e>aVe?k3% z&2t0McN;q|<*=VylpUw?(Uz8tuK!r}|NLv(3;H{muc6*&mW9pr!vwW?=K;HL@Z)bZ z2mk1oqx(DlO_IeeS{u^=y4N6-;?KTFOG)@#zWO+b4mX7EQ-6I|;H8`fSExRD@M|tU z_`m+t#~HrbrnT1|y$t$GlRt*NrTHu)!XTi-!cUU^niu+;p77Cibr;|+`}%e?%|!^@ z$=Tc*f9gYQjf>XZq4#w(?}zu@qedo3x%nXl;{B)3ejUGnf4$xAAlrI&j+ZXvk^qg~ zHNb9HZ=^?nylK)6sS%oEc(UCD{8SH~ph2@e?BL<3{egCRp!&=2cz>X;?+X#9I9?HG zKG@~`D@MbTXb!Z&{y=w$okY_f=>01|97gjaEjI$8bu5H8pZ1XKU*Rg~`s@y zLGX2bOkSJ$Z}0do(LC?|6+d-<-v76j^q)i9zTd=cfDFv#rZxHH_pj(EYNxe4^S+zf zKLYq2(vYD!;y=L^(T4B;yYc+b+u`k^ubxXI*Llo$z6}6f6_(|NU||Om9*)ju2GNrpDrY~=NuE&_LHwb}xsoW2J%8J(&JWnrU4lFX z62y@=A}72`+9FeJ6uCTt0K^(}%wXC80dn54Ib>6QWxVO0k(36v?&5w)ddIcrBdY4 zH@NLP_Jep;@dnb60i=*GrDwK`UZ`m3&IO zm@(;{#~qoy9c93f_sNx?$B=rWdv#JiF%4Ea_=8@h^uiW0%qm`zPz(B10npSpQ+A8P> zS+dfhMqj3%t1_a0J5+3J59Djr==`b-CLY;4xkC}a`j{D`#7`XrBta+8uj6Iq;y)J) zx9R&sk9dfA0pQOzybR(*OwE-+GrAb|xJbi-i5zL5qDeC4JsF>`WAE_ei_%61f9er_ zLCI;Ay|Fz+A25QSQoY+Q+j9eQB!$*S%f~woh3e&h;Nn7(hchD31$sjp`1+Rs+Ub4r zj`3npk4nNFz&2yA8_yp!050e!B-&*Nz_nhL5&D#93;5FmDZWb&?1zfOjXlto3cWvZ zLHO^B3rqEoDn`~HhbUF{ME3PGr}Nk0Sz*VaSH)aQSf6ZksAB+Y*b3k| zAGsITFlAK${^N~-_IQPHLx)&KD18?$gkef*iOS==b_9 z7ArJ+Y%f0b+1$?Rtt@X>FwwEV6TB33=v+K^aXODqg2jF4*efK=cVr@z=SRr5nete0>^5DrQ;l{^I&2_WtneBDFum(oz(MqxiN$k91UW16$(b zgB{7K?Rwg(s#)XNvGZ#xZ`QiNTJi=k`5_z4%xeq4X;vmW)CNnT6~hSr64oZ558Ln2H^ z0vVQw*|y@*&O~dKot3t&rjKLMiytkH)JV5%W$-%12`8_1Z+D5H>QZLe2k-Cg5%+f4 z-^w9T9G(wK-Xwswt#MWPTsF|##(`Ua0%{gRKhtf|YB*h+?A*gt7-f3Hd8pFe-#?|M z4fKj0?y8ZxA=v>`qa436X<;pO)V8ftVSO^-aC>;D!o;F@Z$^&i2=*DO;RguVS?Z?~ z_LxF^@)ok~+oNqBTb}D~wx_X$@tz4>I5%{;{0Kpm+9_B9?3_hBC-vqRfp@q>O`m0= z*}`nj;&?EnPJ(5}f3+ih$tu>raADM?YHu=I@KvG75a_M4JjvP1m2kN{_lt;LvC$kf zS#Z2y@39C{d^AG*huG4QL@2_niEgOB!M48@YVb3Y!^;FZ$Ajw@RE+S$7S%)Ji1iRJ zG1)DiU1yzAS>;iti^PVt6tg7adC&De`a@fJJ5kyX#eM@C-Z!Hv3b0*`47`Vhr-=Ry z>=iX`ut2L+o>3Ja+zA(g$)Yv75ECKTCkL0cuf>?>`#qr*#)JvJlfg9vi+=k!P*22v zA*Bp<3yL~C0y?ap`Z)m(@)zX7IAk-kMOoRPn~AZq+WyTts6Hrh}EUEfvk_S zOd3dT?D@t1A#;M=-43~RRw%&&I5!4y_%>+eP=UJqX*DRa^luIV8TZ#XG0@mlyomq_ z3j>AH`k1^>@;>g~s>q-s1Z;?BfM^@V3^gYo&DJ^Ugc-Rke(=UHP>aXv`I-CsobQ+3 zYUc50a|pfZBf&DtePet%jRBg~05%D3TQ=i$hWe@4fS)S;XGq@zK(Z#tgey^<61$UODV%|;uoiUm9FH6uI+3c z4>I#O)YYLBz-_*{Jmi3H;5p7xuq9kG?jYvZK3cmLRL-8w*=88%AtZuczCh2&bfw*F zb6HOdVF_AeuX;Y0wG4bh>qe@}<+qP6>J}SizZrAug>Hpy0wMaAN!-92F3;hngu9MW zxitVX3I|&^Yx_I|X^{0eCXLOjpN>d5Mj#dKDMdS7KM%(P7Qfol-?si42L2f`2D)A- zRUxknvl%{rOf%~I@KIhD-(LP@Fd~S?&m25lEeL`}EPC3?l1BU+=xF#nja-d@8A#Fi zzP_Z1Ly-(R-Zof)aLjfRnNk;M`FIPC{-VzUd0pI<6F zw1nwY+xA;gCPL}=x=|R=)a25+W9a+CYpfd%vT<2O4_z=%ys7!vTR#VflGrI*Gg)VcHXhf@en@@V z2(Kt|aRBJZsKlpH^B+_etFN}LYbUq}PxP*7uf?E&CR{8Q{Yn3C+4K)V`>A#S4t4-k z?Q)PTwiIHm@taL&t9~*Z*1JBrgruY}1>xO}5BwOh9;-C_UYOcAlc7@ zZO^@BAwJv7?3GK6VJih0#(02nGc2asaDcw%0L|j(m=O?10P6B!?MC*K{+!-2V2k3q zmX55lK1V=Utb=%&`$GWU`~|?ur~z!anhqm>~7)=_jzo7tfTMXBd^< zY=E^!Bm)(RdzsSEHObTfE)zXDp`BT*f;VQ+uXwz7P%(Igj;of9zrIw z&J2H;BRs1MXEgLzD%LWn!hRl9BC<_=Ov!H$k6WJc$=}jc0XQ%Q<9gIYNR#J$(Y=MK zxUnkIyqnaeQ5ZS8$*Xe8r$UCZz_HOIb!OEBNJ5zNEF!_3c`URQpG=)%o^3vj(CB`~+(DTP@#Tr`y zwSSL%e@?)e!+6v&wN79EttnKMq=#p>Z*oVpdzX@?Q}(>ih@~}#+jl^&cr=Pm8!7b8&8?{om!t8J2EC_J>4zqHqBXJii+?BAq0^@kEiGcY{SHNk-o_b z30n+gNKx?-HQkhYWMcln`k8etC4qh z00zYyKi*^nUAXQVdO&=0oEpUIZa8ECnf^))GLi0j5~oNept}db*-9%%6%KmTj~q|OY!$OTKF2iGwvy&!<8b~FcPvSV}=tVZ%by2aXt zJ7WrpOc1z*%}HiBAZ<^@6sST)P&*NB>hbv%K~)5oV(xA|qZPwA@=9AM_LSl}aM2SS zUh9o?mq`Ww3_pTue3uN`Okf_BZr_+2s2lSRlqGD_K$jAN!F~vTZ4;$jI!Y+MPI?L@ zbt0XJmEnWSnRe7!11&hG(7|d^SrO;$II^RlKTg>;f5bqx^WI=CPI=8s{)@h2q^0Q@@*1Hf(a@JTW;QTSKUn^5Y`XeY;d*&J#-z;j~n}u3v(F z-kk|hE$8ygB~0u29_`KG!AWXdI!@A!UU|*WZ3_g{IHvSZcWbP=>&$jX9Wxh~Cqh%} z%J2Nve2kxSDTN|O#!C(Kh)>#x1>{%Ht+iCi?IEE7A#t~z%}mL$(gX84^EqZYbPVj< z7cjAv$Uah`c3za=y+tim#^+aMNCO3f`DhcTy`jd$MJWh)_7=g za7wto1utq4IFJ3rBw$c#eI7A+7(D>XT+E2fmgu$ZM|v)%!6;{)sEXm+kyyojFzEV$ z-QnH!95*AEONtRp8+&t3zMwZua&`qi?3htA`T9C)7pi*Fd27TI@6%=mi*%I9<#7@1F58IHJ^GZGR&a>w$w!H$Ux~FT?;lT%bI>*nsq@j!I4Wk~^ zOlA=zEIZ={>%E5j)-o|{9@2AzMX=JsQ5d0i244x=u(bzv-fnFPK5PEv;AGsfG3v&Q zg^@Af{79k0TC{Q5XG;4&Nm#Fy$|lXjOA6YskVvn#Lcm_E^VCJ{a^JOR6>D;Fg{N_? zSAc=(BQ3p^YMrZ*^jZos!wxYOJ8wyzJ7QagbL;Kzu>nFNS4$f?!3A%_`i-8Ywfp=G z?w^GG8W@AKBxOJ9JlZnIocw%TWVU9-{@YoxBdHp&;Je5xWBwd6vM=JYW>C9zpMB=m z?x!k(fGonwg=o*CVwp2_f@7n)JN|YrC`gpa!l3#yJw@Mpw9F1R9fmE&YDfvmWcSx=gOUi_-A`_5HTUt`(w!?Z720B5W zf``+72qeD{xS8M$6?$3Hc~ZZcT>agQUV5~MpF_WOTm-{R((QI36Ur(m3RZ)j9N`kl zZKLxNu10w1&gSGrxQIPMbMcbT?IUVGfb<8syKTQTW3ygyNg9-!?M27t>y0&4MT4`n zLA|HYVs6Iv3u~cezno;drx_1Pqh1OKfcWS)!v+wMTC-A_s?eW$fD4m>s-(FsK88Q-ec;ko+)e+Y3Q+t z5gsn71B@V%8Y0VfvCDcNLv8QJS@Iz+F{Qu8ed>OJh2VUzqi&`yC!XpvO`@ATC!${} ze)Q9VE%>QoWB0z-*?94~-*lQlos9KK{g1GUAakMC`o_>8-DD|v$Nb!Fy8&@>^IT+r zbQ`^V=m}Vcd@mPO9?-~30G&q;M;}9KcK!T9E!g#GJriRCO`+a8GtXnT)g5W|z&(xAMO9PZK3qcpR&GUIQw^zUn zhA%miwm|{~0djlo^baeOa9(4gn=7PfE`eoZ;uACCkK(F#f}cH?-JET zJK&LY?%SvG@fgOtg6DF_3xdw({pNA&_oi@#;1eDX8LeKng{0iHh~R?G?lwVFAdFU^ zx7^T4biMhh|3u~)0Y<;8@VH1kmd_LjO>dGwi$A*v>dzyzF?9ZJ!Uz(LrV} z`kPhVq(gexr%H}m595r1XShsuON!uhVDe1A(n5IP(C z{{F!(**)sfJ)}**5VG*hqb+jsz`THBa%1gu!tv1!(xSXV_Nhz2R7TiQqzZ1?rCTQ; z;9YFi&a>yO`;A-G18dr)7bcZrLuFkbe9cJTz&Zeu$05ehIosmj=(XK{JPw=%9z64F zySItcOL?6Bnp*lkJSbk$DbH)B#wGQ#B%ZLDxv`!jm(1nIoxBjDs7Z;))~2W!Vc(mB z>|ao|YhAL>G$g3^U8?31ywCE5>HPhdbV#sL9mC8w@Xp4~bM-{`s0JD(ALvJVfQ7x@ z+^JG7Cz0Q%I-uC1S~gJ_xyeHIDN0MF%VQJ>Np=@`t5d*Ox8?dd(uERLlBjxRjuIF& zE}7@CHDz7bYtQ8cp0jfXeExO4^jtHFZ@uA-H;wvPlZMI^=K9rg82FVEYi4#Mq4Xk8 zD=)hjZg$4EEnf1Mm1(WAMX-@a(#fBdQn}cS-J>Wex12^W zh%rHO%$z|tQE=~Tq0|{WWyzbLi1gbgIYe1Wmy;@z@#$x{FnHk2k&l%{_oQsgy?B;AZyyeDPwGQ*s`e3iT^6+#@ADh2BQ#T~ zjH3m()PUWU;))=oy-l(fOzx;iT_J+x5X z!9Hj2z1F!l=lXwmKb~*k#pTSLnPZG+Jmdb|_ufU#E?aWHMpqj++dtj#=`o*tvHAJ2 zPt53?_RNxC;+?^Adn7JJ z3@IE7?S?_t`(vS5AQ1uGT$I4mJ#neX@tTtA_7 z!RLDW?Gc%^`BXmwxMRi56pg2?-$ez zGs1U24^=|l@JJ9t4m43SAKQo#;=+n#op05uIaSI}6aooeEk9Cmktg(U4!vb+`M;p9 zS<@UgF%qE!e4f529|trfoO?;Bqg+#{)36AkWg7W|NRK{(b~bJ zr?fZ;1|4nt=g^+y99x&%A>_D$viq}UqnTh6g)r56OZwD3|9)lAkD>u|z6`hZ}E;%+Ksfik6Bwk4S5 zk}|vACUcyhP0V2{fjzQR;oR^!@8~Mhx#cZM>_bD>s*Q|C#Q@!;gcku;b5S=OLH~5# z%SB9E6n8dIL2a})RRB4Lus~`?l*!b1AVwK3eC6wN_yiRnCs*dAlH(rpAb-*d-dLBh zQ*ZXX1HDt)B~hu(^0uhLiUvJ96Ehv+=}39uRtrc73DM*=OPE?L_JIMaT!iC_tJg9` zzjr!f{ek!*)fWBrCVjgiIIUf&7o`?P3XipyXY$z> zdaCX474nO$CaTh{Thko$)g@Uvnm@;)KzSYMXZ~XXZ&Johz zirjFjKTi415NYq>QYAfN?&j@HY15VrV%&fB4&VD^RbqZvE$HRwZSqxIw-Dh*mZ2k| zBlB2_pM6u!Gf^;zKIYhY!l-o`?ZnYOUs?IOX4OB*HC#gVPj-u1x}Bna=hf>ek(H~_ zd^yjzqQ+)ig-!|vJ1?EeUb4D98PTesDOh*!;EU-P_NnUeD|!BqA8&A7(G1=i-LThp zsC`{lRXD#VgHq84K?t86F1L@mSLE8XBPv3(C-+m{xQkM9uD*X0j zYe3-l@Flv>#58XW(Rjy+ZgL`5dr|A0s}eKiM>uir+ryY@uhtZ`N3*4MI?DRDx9L&WFm8(-IlaxR=H(l&%@CjEN0 zzMYz*8R3#_FkYqMvn7PS{LOjzN$Zb0d`_<%O*JIhP`0}+|L;e0-cUMa>4 zawfXb`LBJ3!J7qlS}V+kwS3cA#W_)4rua|v*`GX?vNFFc9|nt}V*KnG55f>bwLb3%tIlcpwO>honI7m@ z;Z)W;JsE7}j;|%L#BgONVz$vBXwLO6pz+s&ORW0D$J}CeR_2?PkxYINM{?)h105{+ zPi;f;^1xQVtjCuf;G=V%!7@(5yNu|_WRbZ`v%LL_V_hV!|CZs1j%=gGBik<{w;siD zVx5Nwt0pM=im~mAa z|J11z*vzJenXeZXl+BaAk@)qVhfP~1RFu&hc3BQxi#BZR0b3|#sijtt#qS^Za7vhi zXKj5%4Pxq(OcEMDw5nPEr?&Jn^S+Bp{M+&C-q6~VW%qBY8(|4`hxXAE+0YtAF zXP;V{OEVqyb+E?&=E+R8)6fYL{b40RCfHtI|3H?N@?h)7I;yU5Gq_fd+2lWe;_kF~ zk#_aEz|@^`Ub9RJ^;{j>q!?!WJ6MG-CEqA@F$iKY8rm^a}w+lq@ z+#3*fv^~Q!TAOU**qfSIbKuqO!KISX^~Ob3`|=iI!25RZ9TvvNZ@v$gRz6jjw+>1L zlY*Q`-D(ymCCVG_f!_#$-(t~rX=fGRNtt+9YS$`#H?%fzt|F1=dqgkkR_&*XC-IBL zja{*e4?s;Q1_P_j)B`9=-f^7YBee%d7&Sx8a_3)i{awhBL8`fAGb|J7($SSZW(Pez5na?}+?0#JTnyd-t0?UqG z_vFstAsq+h%MU*;cg;kjCcwqLIE*T5a~#sY_tU2TuEdKQr>QO1Zz(`3yd<9@R0_61 z2&;3A8rcL-m;d|9#a&r#d+b4sg`u-9!?WJO*;7U zYhq*cic-@dEc(+th7COAZ);MHALHY0a~b6@JL0|88qvro4^OT+M_`#WbSFCQYtg$o zu_6_Isk}Omcf|B8xnV8;(~u?f{qFV25Yc``+ec1;6kDmD^0ju-kZ10qv#8jC?jFS`dLbwP+ zdB#^Y*hrotweuY2gtM8(x4qORY`2z?2KJH=#Za0HO?ed2B?X6OX;Lvj3&+PUx{g+~ zNF~;2;tSYlt`li4xpUn&>`yfTWmTq87TT~|Xh&k6`&8ZtP>!4*ejXno9^#F!n3QRY z`sjC=E`0P8kKV!U=DiA&IHlIRiEI)>S+O?O%$5ee=RJRI27e1Fmtl2JEbd=XYxZKv z|E=`s`DScISqgqb8x?^8Dt6Keyyds=MB>xy50Z`@&fq@Wfe!RX>|62FXH zk>+HKNFMpgw#o2@+&r%_O)Qo|Nl7SUyuWv$(2C98u3PSX;_0Fi&t~PpdM1+2x$Tn_ zD^0kXe8VZZv86vteZ)Jo*C*RsizM*X5JCp-&_uH7&qJB?mF#1si=qyd$u|9qSQCA&VSA=nxK>Kgm40mv&y5Ja-~x7*VTJ3 z$^t*2H&wGzqnrL3l&aX6ze$``G9}*{87z!0Hb2nCw%yYY^Ls+m&V5&9G?h(pM=F+& zJxB7Z2ZOY~$6MY$m=klnGNfRVkjrXM;MMp;woRKZ!$E};Jz6nsJ%)%W@pGD28{J*i zwC_vipqApRoO;A~TfJV~OSrf-=%GezcNq;G!zIJA;b5vLX+@`HyJwIflm~$U`Lx$% z`pvAW1aB2+#y8N7?K(H*$-DUEjgWDW3HZDD^1#*ar5zO;(Z2JQ;^uo8>G8GvzS1;H z@jbD$l`ru4-`5+-cXQF!#HH$o5@#%zh#ODIw%7`yNJA3O6&H0m$jOeJ{18`?+H0x* zS~AGpm}i^kN%@t1<#E0mp&fA{KtgnTmXHUEkaZfDEmQ3_3+Qd^73Y1Pyk+T&_tHHZES{spl4#Mm4 zLCt>c&^LnTr5CGsb_h+_w)eeVL*p*s`H>-u77HqCswle!V=3OxB>eh!ZIjBtHo*jcoomM+1)%BZ2Mp`(<- zpyl#qggZgyM0daYAG ztgH0iSnSPFeKF1LVYbF`so%pH8=K&;fx&WZ=iC`z-}msUqQP zYi$_5d*&jen3CS!ZO9iI+A}|Rd9E#NZ1-Y?v2^wjm?~Q-aEeW)FE_~ZjO2bB>Ty}n z_m6Q6hINw&?pG1qO-$K}skA~2oD+G|lqZSI#x)1~fl|t~jcS@aR=e-jfs2g$&m;7n zXVXJB9bRc{UT|9|jD~uus1O0a=z82f@-j#(8@=6^+@{s#^6|uYicDbjtMXRWB?ai- z$`57}1AEJ{iE{U~A6AT)W|_+Q(|GQAeCRFfzj&f2byCKW3eU~ia`t&$_1s3NJ}%SV zWj|~%ptilDIXPF`fN4J6Ld^%fD{yxYek81I&o;R1JS2bo`5^*H1)P0Ff7WTHHdokP z<+}}yRoXPH_)sw`8YCyt{L*TO#|wk)j~vD=UsehJB;L>qTue zFC#xyY%pnWI@Ie=OP51+7nbz+&i-3B`r1zQ5Jtpb5j;kW7^5OYMp06#7KpNR`9+Vl zyCi&9?^K(TzpHmw3Du}IPT%%6R2zZTSUo(;$B}1mETQkrCWpB&^_2adiWWu5Ek^9G zL@Kz*U0tAdxI>gZTH&v+nz-zEuk7Wt&ZDDrwjc@VFj$_RclAi+g=s0!4&>s+G>*(l zO1uk{w!z8LhJP*S|Dp#z_B!`aLynCNY&}x^OfL;I45}qRkAx*ZVGjkr9&0MXM_wiQ z%*A!Speyc23nlcxMYQZ$j~0Zh(1V96-E))ws0Svgb*Q-~bUdB@%W(bcy&gXq1OUgX zyLwa7N6MeAU&q%WEg@#LnEH6|RAd!)&A5CtKy%Z2{u|UpU2p@_#OADVb>7J0;1 z`;^RYbp-G zLhVrC$=<(MIV%;t`R1|`(u~b@u;(A9vB5^yjvQTFG*b~ThwkXD16^A#aLe21kx3x| zcZNde&TZ6$V=k$vtrHwLDcFowBq^fyOMz8tgOZ5Vr;jY6c+f$#0Vkxdf#T@f^AN8V zCnu+ch12G&slrjJ-q!3Yf31}IPDhW@Y z1ZJ(6*v-!}i-=Ij?)|QQ=h2MgCoYsfr6bM4FwNjJ2MgpU+5iU47$XAn%Hayf%L}>1 zO2xn_^-F-jJT5;J>P=?QN|T%RT3e<1>rnjbi=DVY-KyAKrIr??RrouqM~7RpwGg0e zr=@1`QVtZ>SaAxo=*oAgGN);|*!#b3slURgrUallU7La!Xntk~ZaFrygbUUH*{FNh zUbtfHlT+e9nJZamzHUU5)5UWya($)3bzC3fs_Y=78@8jwWmpuG)P7D2Ih5Y+x zGa{a~eSg!wM&!2MDyWeBZ^+?Y-7UKI2Y@<4|4{4?Ovzfu+?<9>_;aCEPa1?qyh&vv zi)F0hn96D(W^`zb-EypcoL(Nlo26U){C?-=zWV`CgHEjC03~pR-|%+=8-PpU9~N(g zinX6QC-e>Q*b5$?f!gTf$EzzC7U%Y3RR{Z3euh(6+r?3LIIB1ev(C5~_qS^PZ{M{U zj|$wwW2sPv#q(hAf$Lfaa(H!`XQ5Nno+f}k>vswLu2!~6k{^PE3H4rS(spgR!-%ok zqp+E)rz<=jb5P5&8yQHKbTbf{Pn|rZUo$#Tka1vgdr#1(Vn(xTl|%8`@Cmw6lDTF# zrAGz@97RCTy9C;x#<5e_V)#bjxpRKMkf-{pzVimm|8FDS81OBzeUZT8_HQiX-4e0$ zu;i6Wh6z5o-t&Ce5z&^NOqp`HfiB3@C_mGYlcezo?Ir{i8X^_K=4Lqt!UvTnc$>SJ zJG=hYz5CY!SSIs2wXlxV6M*1zf4$o?9L8VwT|O7UzJ;yrEiGrM`dhR>DyT!Z2pn&D zaa8-BSUzx&JaRUer;UhxP$Q4>yK@vUaV7!@Chci&TD7B0=8wG=J3Rcdxo39-0&Xix z^?kQ2-cHwW;5Jy!9%pM8gJm+3QIK#1e026C{hInh=o?Ahv!-AG0aF0mMZdTI?+YLi zJJfNgwcxk#v1yCSf5hi`H;3a0vJez`O&j!oL;c?ZkT#X&MHT4hn`ttZ$8FR4mc0#* z9&XjlX>+N(ckb_F>Yp`;oRZ|zubF{@$Ctt$rlp!v=`yY>Y1ujb=Vkb?5>ZiMw+RsB z(s`VI@oRY%H6|M0?k7fb6|9y=Ae#HOxH2(Uo{nyi|$rD-vaw-tu z(4k*=>fW^@Ug@6_hh84zo=ZkRb;dD})L1)E>8vceSkZS_=qTAX2AqfQ02ld4WrzA# zlcY#(Lu6m42=Py5fvFa%uEvL!Ox&;o&y58Oz>1^+8~g`xcc#}uC3b}J;Vhzy7QhDq zuvOKky+cAXDy!(-v0u@UU1wi!wSkD5UG4 zs{06Jzs^2?gdVY&NmkPT{F+!!1+u=%%F6NA;9U|Fw~M#&9l$=~W?=S(jLgwj(cc3T5H__;M#I@WnXqQd@Ss!t2xgf8YlJ+(K-H|yb}6{?2u zUcxjx6ptM%kT_<^tM}+3bxS1hQfE9_x>nR2WOx3|Hnc8ri ziJ|>srG+?fENLeF4(2bQVc#+ehqx4-JaI3SPH~KfMv^J8Bzz zW$(}~I!ku}1VP};_DtOEj+nK0UfrTcbC2@I4NHw~$)FBV43Bdhz9iT#0n_k_XdpiiUc6IAj zL3lOPp+VTaczwRF%_&2<;E(_XmfTU~xgMybR?wC3R=y7CHa2+w&uFE5?x zh#mSUh))nSEcLBRWdoZ(Tj0pTT_Up$41E2QRFnii-l_#%y1~Nc3jo{0q~>*oLOPL8 zGh7x&Te!8+FM(dSuYXNcw)g5U{_6Xqd%kNOs_>}8vVB^E@4ly!lG6R^mZ)Evr_PIX zfXz+~FjJ9aw1sSrj?cu-et4*K!2=1IOwWqmjIl`gy5xN%YzaPij)w=Ei^`Ib7ePHP zjjA+HA6W>@Y`Unw0MAtbzV3(3<{B%5jR@n2`|LMQ#o3OC-|yk3S?r!oL2sK6z_>hz zTdOu}n{j^{4+hu+>8#4|L+5=@r~U2d3Yjsq=GXf zPd(#8gUTJ$R`jA{Q&1#GUcserdU;Go&gRKSE4fa6_V+WYrGCA-8>@Sq;=4O}CH9e3 zP)D5b$OZ9rcF)C1#=QlxqncUMT8Yaqmb7$AK@D|M@TJ$7&%S$M)$5c17U79k39jSb zbd|rP=q*9A7U$?k9?$i;sGy&OqRLCgTuPQUekP<6jWOgQLOa`zq#*UFxlcB}IeEe;0+njm zPf)FyO(4sa^a3u#d~Ie=CsBdyS3BWXOsgPO}NzAtlYRdj5tNq)St&(aQZa_PB zPVGQm1k4rQ_iGXR#*!r;Gz3HL&6_c1REuw9s>cm|Iij0&aWI{_6{@^s=V4Rebi^uk zjEd>3DfSDVDevsVzxqi2t;1)#3ChTF3dga({p(*U#sU)PXX0JU?${i~b^YIe@$bj= z|If3SmYCZEX)1OgjjqfZdkxeMR;}kTF7pEVWwz3-n?V9A&1{UrpaKXIkpJNh+6p$1 zy^@L-)AZ#DUqQSlD!3%Kx7}cr{6Z~bdyR1j6ma3DiV@A_Pf@}n_khueEo8T#UYoyX z5-h$8RZBh!5#>TLfYpFbm64*=tr}d2lqz$jAROSbG|g;~Ugi+AAm_AE@!^T9NFHw<4EX76*JU=%Ht+A!iVvGV$g~4kcc5OE^ z&c?oO4;3-QskGZKvFGyq9}n)oc{sym1F20wziCD^RaQfGm&0pTetfyv5uKWlEx+~b z)h^1{$8_Pn|4q%rHv)_n;$eI{Y0-FB5{K2kr3Ce-Mk&+r3z7WlZF6Ps`KpBSae4Z) z8(GQiRi-Thp1`9U&d4Poch-fUc&y8`Se%)@*STZ4kqqAic}BzF2pV%Iltr|u zcGsyaGD_Y}H%YU@Gn*|NiQjg^6K&mB>rZqI{cc)$9Sx#oIUIaB9TANbwceN@3^Y6{ z$4iz_XFDWU?mNFm+##}+fas_Wkin!K(G=lsIG&)+TOl;Ut|=U&MC*&r5S{tK%L#mi z6H@#7+_wdi5gL;7fDS~AD|;o_dR`ZGEfPg%8UET~+`L*2AeDoaoqEqmaz+kd+$ zS~dZETwkew-M`Q9aB|ue?%4Gao;4;9CLA-sbGy2X~_=%*F<#}arpg! zh}MA%qPD&OG+!gFwsn;Zmm(dHXMa1%SGAxpViG$1)@+2Zlq1+#;TT@-A?`)35$!;i zq+47ILX151DI2_AQBEsYp`k_D>i{DAvQMFh2gc~Z^AM}V-)zVcyV;3n{kmF7ekiq~ zP_SeTD~YYV3*#+$BR^7LZ1OsxDs4Kzvxdf z+)HBu50GvN4ILSjI!(TtcE{Xs;lE)$s8r96g`vk#_P6>^R&t@r0j(XJAKyL9D3x_$RCBZGNB<)tThNqFPic=*Z$Q6X=3=vwfG@T7zQj7 za;1ZRJVC-29nOKZd(q?0CZKop7Y^wiDM#;1HA}TguESO*)-)~lcFu1s6x&xA%F{^P zePsyn1uysYKgF%by^(+3H*I;I1M@`!FT2KjoSe<(ox6D8DKbf=0Qzx7#q;n&($7T z$Zd7ynrpV<@pc2bOC)~ZO7_plF!G@9L1&?v)m-c>%J-qA3=ec9h}n-F>r`U+OTf0?x)Pt+IJ7`$%0(&GSSjSRE$)T~ryl zXj^v$6o)nbuF<|DF7qJB*iO9HE(KKHr%0*!rM5#tT@SJ08q+&oK;$j%P!ei2a`#n^ zQ{Z{ek(u@wi%6bX9h8JPNFvr+3`0U49aow%FdHfO5^GKxKk(h7fUm_&9q~a^UZCMZ zIV>Yl5#iJ~?qL&0ypyw+V}o;*svH}2IhX~H{MrSDY2FB@BTZWW(g8<@X9|$aK7fO-@9J=i=WI>21U7(Gao$vIF{MMvx550 zDu3@1?WwA(-~Fh#lV1h7MBF@c=EY|Y(u8HRZ*7-ZRHd1P{pKRO=jg1vv1dZ7z9PL6 z77;1M_;iJs$q-FMAi4V~|L~_9v$|x0CV1b|E1l;bG(QS;&!r}^*|1=#Ro?$`XVkq9 z<IM!C8t2Web43Vuv50{gL|CF*T6~XZ%L4*vA;P&_dt62sLKbEdY!5`+FVeDuI z4n92ez<3p+nS8cAG&gOWu>8y}7jPfj!7z>)kj3SoxR~%;y1P5^;G$QAq7_ zv`83g6GbX?kiF8uZFPkk6X}=qt{NQQqe{~)-a}EBbna9OInRh8C9d61%$UD+?;*4c zn<*u@l-wl^Ca98THa586=>2Zp`M9*r!{EEBsJZKxNePxLCf=vsA@O~)Kj|QG69nBx z%5J8KGRk?9BwrYHgA=j0y4sd9y7>;wqVCNw$kH`^f%R&p8aw7ExtD)k@X=}^ed$vW zIUqQ@E?-MExpk==9J$(oXv@o$F)IW5Y``B<#uKIQj2}HyrNoI@qVPojtTd#2*tmRf z)#KM&WHqtA1sp#bXt%d;BHw!rR2(u|2ua*CXa?froqw$(2lB7CI@T^Yc1Ubb+2jX@ zsvJ^s@Xm7|Ozf@j zP4lUrj)2Xdt>AwRET|lLQ4a-N-P+iQRP$gP9?4kEJySb)a`t1ij1&uX2-x0PGq!^> zs4l4F#Hh3KO6@0X5wm@DzxNF$L3kMDe!kh9{~oiw*VGOzF3`q%8909P7!;_&FfY?s zLWonD zF)R=#Ml(H}afdzPCzp{U(L8LWMZx$dG(m)! z7DYj*x&NtnJ50<){4d^SK@u&s+QUwyj#SkIrz9kG%2v^7u@ZkWgJV?$7KRja~xv-476|THN-BjqXAG zP>m&Co}XnRpm?(dHFo|KNuS1=XCedEaM6_+gXqdbgab!co76 z0_?O&;fFb$OmlG=CQI?iA77~wX(YI~zMxaJ!sLM4z!PeP#*k8@y|Htu2Uc~O1-xHe z)+F;hfp77mIOmNGFwO2zw*?^-dt5%^6ggPB_9&<+`1d*UgjUUNX+eL)BCa-(y%c>J z4E0sNPY-W3*?977LDQ%ps(~+i&*8M%#Q!vZrL!`7GVBhI!fReA5^*6?(S@dJ=G0n?(ySTvIjl8o zWFKTO(3(4hPP7Bc_>1f(KQ|`W=!#T zNg3{}pZ#_YxxiJQ?;Ltf@VngME(f%d-^2BZ>7ktra>kU_$GEtqYm#%$aem2_@m>9l zE`zfz1qhm|E6;JhAmI_OSVQ>Sv&yHQULrb!y)X4zj%B2LTw?ICnhNw~e6S0&#|O>F z6#>Fx-y-{xizo`8P$B{UH8YacQ7wptd4m+@NN^#wL4il1*RIVyM0cLW%nN$Dx+0m4 zXFKNXas+nicXPu=9x%F|SJQoOI|7LGDx1%%#ESmPYT59GQxC3XM2u8E!^Mwo>@9?C z!0adEiKR=HRogATW$&*E%mqvlu*9*=Cu7|MiavM7UZvUF!#-&LcawSxMw6=1Jz^i3 zM>hf7k4pgLI86O4j9Se>J5P~pYzW)H20hX0jW#XyQp>USmdex2`AQp*JwjJH^W~9M z0As|+FX|63GMaQ-;5Z^4DmK^>E z*;RV(nbJWLV%$R4qv$PF+LTK(g?_zx0>>NSd(--$D!;3iu2dQYNW97}b%)Ec6^Xj2 zu=I{H%3!6Su3NIaUJP-o>N#3Nc|GJC7#(Qygax z`3#RdtcXFw3ZPRwTD)mwGl)o&zEVMF&svjbzPB##7FNJ#rkxt@+uGjl0yA}Sem0^! zqML^XC+{ATssh+()zbawG&V6Y_DNqE%aON{`e*|toSMuC+N>@8v(FbG3%_zLUkje6 zFX+sR@o!Uc&_8~iMcuk1i(jKv)Mmhov0{IRC~U~nyfmtKj+&82n$&5*;XE)#}j%}Z@??@D(1BqrIA%`CWYq7ZsP@>3ppZ(S`K zXY5vi31nbf%`EFePu-gyH^_WIQ0J^1WjPS=4sO%jl`{g7=nu0c#?5*K`#t17PwI>E5oh8qrU<-^KM}z@ zutRM@SCwvAy|YTLN>Mwe)LicOLC*Sf#+3o3vjeQB5)h6i|`bdQ@kZ%-C+&;A~&v$^~ zQIEG-r%gBd$E0B?MA`WSTry^*sgL_};So2#@Oud13)8CIE71Wg;rm6*9)iVt_2KYD zl#hoXQbKEdY{x6v*{7P`wc|`@3bO>5Si0nQrd5{LqNw)wg`v3XW6KxTXh(42LpVS3 zYW}7k`anKs!sUEhJ>B&jw0xB5_(gdaSPRMPifep}UHnTZZ3;2r^XFes(blP}qyqt! zy7ule+$?Q;oy$2M6((0c!eAj{%6V~Nh_<>bF>g#*1pgpGpJmef)EUvGc}7FbFSNI8 z-Mt=*W00od=-<34j+2?l3Ib->E@_(iT)6XB5A;`@WicFmF+|Kj_Djn&euRlCeaC6GOWikNuvL@%r4^ z#f-s*`JTp=R%|o`GXMPj!K>Ce5-;;d+Vlc%{kFc^>7{0x-`w}p>jU(*xH1##w;Ji_ zoOx;d&7RZA;FQf*1~`R$bJzT5&W|h_Q{)QTp9T6i?AmJAAuX zIyK>*wZ(DN2Fky`qfZ;w;qsb}8o7JS^NY=TZKB!MY^2Vtys!5k-kQrAuPqR~DUxqC z7^su5YUR?w&rj1|ZhL}v{g{wWszcguJW+WiH@0?nB17_vUJut9T&k>h`KW2=%einpakp@QUq`8W60K9XhZE|!ECw%SI~DGGePLXtDC1O@f5W!_+DxvA zBa(Mx^Xx%|wha!tCPfZDTUX) zI`K>Q)*P|KH_oPZVr7~xFg_SK{mjC@%m*aHL*`DI`Vg-!J(TQp4w497aBl1SXhh1P zF(;bKEsI^Tfd$ol*qfECi(4QXZn>+&E7L|pD&27Gk?|vCn(Vp_CS$gJ=g-uMRvXp) zhGzJ;y(?cj-L@9!43A!Lu7U8n!EuEIPP-E%RD-v2pUc>JbPfvj&&5x2LWOo}cO97- z&1&IJb2Bq7s#b&NYVN@09f4%Ji2bgmg31O{kVhQ2-oYKlYgsc znrW7&N6JdeB%s-o4vwosN-xOo2-l*N8+U&Hp+K_;`0KXy%g@OCn7SNlY(}b0By;-` z9gj*D-vvLj9O;>c7hsr}-NCRLn`L=kEOk?>mzSkPl_|P_u`*Xp!)|IiHOim7Hdmt; zlbz-T8a$)XmIrMawf&h(iU)1EJB1oH)@c)Eszg}?e~OmUPEo0wQMpGI9KyiWC%LH(Y`xjv%v% zf4qQlK7+&w*pBS1ajs?acVaFjX{CZ)m(JHu(bn&gKM7`#wIj$b{W zNcN?Z4RW54-O*R}-=~h2<>=Y$@V1xG?bLDiDDRh|+9YA!W{GGf`g7Veq651kS?xLv zcGPW%{eiXfyQ#+HChoNY=T|5}yv^WaX+gz)?c(vhNIf^=7T8@^28!@x{yyDlnnF~* zDLW#WH(44=tia{#_RBclfA{5uuunpPNfQcqq?EW_P!@7)mRM9Zjxm8|c7Hc_MIewD>HUV-_5AB`?P z8K{j|5O5N;KUHpb5BDrY*lw-BMkQCkd2W|GRa~df!%R%PxGD7K>{;A@B{gm%wUES-a2JwX^ zoK=CnS%?o3aoMHi5^kW~SWo?avCcP>)=xY4>VdyTozEClMG5wD{l*iKR~gveKr8P{ zp6fl)(3qXomU$9cD4(74g835t+k106*J-{7POAzsst+Zc3M3jPMi zS>*Wyu_zGup6-=%D-RXTqfyU&F&LHY3I8r=VlwCu?ptnVvj^-9YZ8%_nUB@W;UwoN zu1vv|dwXWLi&>3Fq+jE3Fg6nW-W=!Pe&g=08Bj1d`E5VSMO~s}e$p#0skm1fkk%PL zf?;Ocp-Q=e+^@|+y&}?nyA@hJWscoX;unx>uuF+$@GweZ^-E1mJ}6K_a3!LV>na^a zG(6g{fp1|u_1<6~wE)axhR7*-doM;tm%8-V$fS`DQoe{`35<9=lZKVhmIZ%d1yf~L z1)*E4G?`hEg&Q}hL;Ag+6>)tgzyHUE_Wc5~W#F+$-A=jY0bu)fnbk0k8MUEg;w9?y z*tGANswj-l-rja&Ssp|@Mw>c$-RK211M{Onw(fJznr*o{&9!BAxoc(xe3*s|i3F^N zsG-`j;uv|W&Pzr9fv*tWtHH{6MDw*X**-7Od7Ccr`lC4o!5qItn)YlPraNN=JKBstH|v=kE_N6q@^(+af}DY+l*&lU4Q%C z{j#fK&lV1L(A|+$Um8+n>fdRdh7Mevzm|MDWWJ&8JqIt04%#PxnB>?oh_DoHWhCmn zPk`WswMVq*Y=QCg%A!a1tf^Z@{B^E{uZL}#=d;}6qbh>!2(dYHllx6oO5bx#q+r^O z5{)N`*~WW*e~wi^y&PeJ^FMDfS27cKaxVivBV`wuiZGDRkgjoBLR8oee0o3b0V%`y zXPW@H8^rW8pqb1dCHhF`jKXQWZ9D)wE(blfaf-N=5?K);X&&HEwRwI!ES?{J@ zboO0;d}E+CH8!^}Dk+c>a3@cF(EXvu$n)sWKOZnIU89E@4thR*)}7DMKkk*Q(LEcc z({?6AJtcTHf%#X$tyE__Z1&JtWS0IdyN8I_l>I)$2g331m<>X-l1G8~zfKIdV6^mW zdTSoWU%q{&cTr^*+G(@ZAz?jKj=y)4m9nEAVz?fvw9}hoWKvBZ#c4aFXL$d9l99iU z2h;V8>1`<6v(QcA>c)u4Xf&_)mZqdS=<|H1IA2AwYxC1ZiDwNw3DTlmA|)aiM7t975uy(+hrKhV>I z`)Xlqa;DzD|FH;}mLUPl#pA|;D(vJpHk&T9Yn-8F&L<^%%mrJz9B!1roBks(o*$8- z1ak}5#m(*9rbysjv9*jf5Dqn;^};K0GR_!^`4YC3L6qgRO659|MO3u=#ik=en6dsI#Vw`1n)7TV28# z%4oADQTu1O2NnCq&06=B1sYdET1;Ed&V1B9Fshw!C<(mAH7C`9$x0(i7kJWiFvdf4 zv<6-8WqR{g+&do_FY|4Kfs|v6icN0Y7!S;|H^hO$aZIJV(K^L$&+LasM_c9Hd_@1M z*_U>UW7I`t;oEbuBZsTNu16C_;&zc8(6wsh>$JTqh5AjpeAIkOC4TvZpwc%rDU|%T zsL4=m|I;j8m1#hN<4I@7W{)#)mV)cX{42liy36YmDeR}cKfA*u;SH^GVa~K}Jax5V z(cBDG(a}0mt#f8?rqAlTF1KM?X(*D{c_bMO%BXsFZf4aL2<+|}BQ%ydMBL`PQgCya zQyDBDeieA^w9&+}+K7K!IHc zKW59in%XXxRgZ3TxRK8)(egmPaN^583i1v=(J}W4BsYl<8NGy$JQ)(1H~6&|9P`cK zv4mFd7O2yy70H+9OmU9~1j3jY?W<-GVO5M_A2F)pGSolz$S zJ+WFm_S^1+FVp*K4v*YtOR+^{`}P1P>S(qudBk#CS6;3g3hwUtkTsiCG} zu8Ba5K=Rs5T1XG;{>RY*AMcd4)5MGBFcpfPgYLBpJc72cv!_f`+Gsr4%4&_pYhb^ie*Gu5SCpYX^8{5E`zQV2K8?&g-^kB>Iqt?cn5OzK^v7WfNW&#|%OBVrl`=-pbo}cd z^f8JZK}X5C0_y;p*yzh#9B_mXNAmL_-DvvcF;CDDDx2&3K!rnhVAp=_sEbHH2hiZ> zq5j;ve--KeTX@3$|8yrnCvUD^oP5ORtpCL?*-cp<|B-v+h>NJ0p6nG6aQy)ta+>a) zRI<2v#NPp_Q`XH-QBhIcnn|ojTn#*mDdks-imS8DqeuP;*X>VEq-yPasJ)Hgt~JvL zGWFtR&Hqf-y*54B8p&(&I3)9Bsf~ub`IFU~pX^^p(0K6gSiL3TYnrd&a!eRnqf1|c zrj#c&*WdL2{{{mtZ(&rDYv9qaLOyJAq|X_wruQrYdZ5SRdvi=GM7_!^jCH~z4mm}MYT8D=qw*0AUVVZ6)|$F^ z_3*JUwsWh}9QFN4sxp9AR^xxkN4y?z_{ui^FZqblfKx=Oj;-b%1!g#0rJ)Z>+yn=m zb~EimA-jlR^VJwlHCu;tGmm7WCg#D2JO&P{_WEX&ErB_=b14hyWng?W(zC9^3I+?=l-4ocaC)6%e6Y z-A0}2A8Q8QB9_Q8GAD6x<#yE$VSH$=$0X09s}}Ho+TeU`A*x3{>R#!?&WidFNNr&3 z#u(BWDhzZJh2TPMz_!&4idioMhK(O14)T3#EXS%)=FqAHTcG0b|GdKH{ist(V*6Y< z>tRq0X|(d8fZ_>oAYKi!YV_fwADD`v5)cq;UH1TCm%J*m>+pe@WhN{p#_DOnZs&0M zk~sj@yASgz7)^k7xQv#sqFn?*m^T3VdQ0i&S-qgCqD-Y6;B2pI%x#;zI5w^;v0M)U zAtuKGHpT__d`4q-#NfLq^quN0-&}l5W#1FK;e{{YcI~aKtPJmaegG!nr5l4L%9SgG z3ua)?cBz$DzifDeG*V;asGXyhR&<_iKR|qY>LH$TIS=rFpS`XPRtdI&lX`Ri)3BKu zA9&qv(EzPSXd&Pye}aG-*~8RI=>DPPNFQF=0JLg;j8nCEkcAXDwso4-t-rTV9)u~O zJfMNq(|+|de7+k)CztjDpTI}&0u<4NF>`;j4u)JCb!@HyMY41`tKA5zjeOMZ)DOx* zsl9L9N4A4BPs)4-%9B{s2>6K!s!iva_EPhXuY=$>+vX4!KIJdL1f7)Uh*%LaEd>O0 ztPoaX>dt_?n(@KuXu1-h?;Zsf!&TM4Z0_;!w1o(#N0#%AKa0vstoqRvC){>T5IzOJ z=C$>!*$~H_QjNH7M{pJwv1HRdbK;-x=l>Wm^-@>vx=1Zqh8{n$gHZ*hBO^cpcu9Qw z)h3;o0}8}U+~=yS3 z*zy05z4r`ja$DO)6-5+8R76x1uwX+#L3+1=B3&t=sWhoUI)oyiVg>2an{+~GNq|s9 zq&KMnLIOx9Kxj#*VUNtc*8J958`e3$&UJq5UpnU%^S*h@7|(M*cL6L8b6{dH)1;+g zVD4z1)V zkl$VmtoUC#UqS-*E_i?@3F^_pJ;1s$zqJHe4Cdqt4rT7X9I*iL=Yaypz{xuD zymu%|3aKw-x@WqbQ06<0{94t+vYxTgGt9Azu0`TI!&|v2xK7U_K9zJ2BfD8N4&w$L z`vQS(WI0bebGifTU8HGRaq)4<2OEm!71=f?(zxtPT0wdV2fs;W9Ls0^eE9eAy6d|5 zg#FC|nipCrv;A`ur{%mdjbL*T6hQJU-?Nbg*yT6G3ES%MevX6N6r2?Z@by6OhZ~E% z`lBnjFqngV=z}_oC@`eVHtGi!7l+z&e-YBWFe#OjcgAjewEP&3>d|#I?)v`F1Sd-!v;eae6w9fcr)Vt@W%F17 z7113?NsBsC0G=98#gQOm>g7};EN_gM;}|Tjr?`y4W2w=yVk8Lg+o_&gZ>Y=u1SXW8 zu7d$SY-Oqg&p7b3uH@Bd1=UFOn+BLHRG?*u{?ge)a{1mK9j{HmdzCme85Yrvs?QsM`8gZM z&;++y7H?rX-jhh${Br>z3riRypQkaw)~D+SCE%l(nmiCd`Mc=h$hA7kLvmJhXB^^) zlN)(+Bt+lXeZ&fckq%*~V3Bv-ipqIg8VYd+P4p;Wo2lYAfQ&^CXtY53l@!wOw*kI9 zGm69Ii{na%0RA>@w}hJ`NQGMnQO6|hulqQCGuJI?*6U}9ot3SA91ZoS$ajtM4Ew)% z8&9V8HRNJk=O_?+(ii8b&wlPR$@hA94E(M4*dsF9 z`=++*u$1M(fJp+k-f8|dct&c5l~-T!RQZ}W&&dUyQjbIKla`g6c`VDj?d6JoHHGS= zVP@go$_8EyXOMfs8E@Abvs>LASfTKUv4T4H30JZDyy}*o;*J8SBjS5s_ypoks&0`V zMkl$I7Mjxw$Sk1;tpCIh|9yeseg2PBQHRX^T0o;e`u;k-+6>%1Hv8;9b4dU3FE9^8 z`SpccCj=nY1}KYD@-T@fm_0j3k|sT4Jb}~xtX?4ZzmrVCH+(d=u&A3;mX?*>(&(ub zup|$VQLZ(J$O7;B^G1R!J3laf{_Gl2p!`++ff;^$?b>l>>7I=_kEYu}i|L4*F zZYLW-*pFJbvQFDrms+ozKPD3JR%kXPxM&l%@i2XA-5ZV?R7-W1<$suH>ln1wi-15sE7s1d&a zR)4TTk%g))iBKW94YDb$;E;0>srHZzU>DODrY<)N_k9P-u}`jTl2JgG(*-wc4DTMwb0dydKNP_G@7uEj(2}?#T*T?T>E@hpNDJ_ zO@zvls0PpqQ{d=RC{1dyZ#Am$6oviy5O`6b2mhGh3zF=MRRl{rZA5*7v5<1K{6K}6 zZ6B{z;P4w^^Covc(mVU^IrKJx<myewgI|g9UuzEIpbF(T9+ z*tU4}e9$htjCOlsOzY#iq@7<9T>?ABgo7b@iQbf2ohlzA3`iHeFuLXUyvBGDk`e3z zwE^B2ci$a$tl6NLmzHIwQ5QxK2B7RQWO?N)XT0;yzsE1-!dERtXfoih4%ti_bpg7e z1$AgUlivSyhps(K3*}Ss6hIb*9`Vdbi*VSx1}tR;=J6P~`{E0olzVDw@@h-hkLv!evcs7=nW?L*2kV6k#&btgYB|F|A1r3C&+J~c+?Hc(^ypr? z$bT-e|J?shaQMY(4s0(%a>y&=Q6_(^Z<#f4^cI@KHpaphjd^-3_12cg4dn-YI5?CE zATkY`U78|)T^-?BViqf=1XCoAB}}iiifdQ=aD0Vm4viF?g8wLmIH94whTpln(py)W z?C1PPJ>x}f)XZWI0I8YXkdHLJO)95~YpBqig)|ZcEusAkJiHd~Ot}NIId;fAJzX2>-b|8M$!N@xk z7XoIdeSq{21^#2N*j&99Ye(j*Bd)~#zGVS-ogbX$K9a>GJ0k>|>~Za8n~Lq}40B`Z zH6agg`>J8t+e_Ks%=ar@Z^&EgW^4@P2`V5Ws%l}sCwTdvW`~0amo{Rg*=xMlay-g; zT0PH)xmSK;mL(4gjY$2m6=kA^(lz#|VF@J_eBts%(%VDXg-(;8brIcluri{UIht@F z2qZ(kvBCmS9P~2IvR8=9!dyjM3YO4SvS(#KPYN0qZ82{T<7Px6C2^J4Cn)k*oG zImTDgC9I1eb7}ziEKJHa6vU^crInuiji{i6g0vCIiFck;$@QwfJ~6ka*~K$|ICZ7h zmnb0DyL>49^>C9a?Je76#7vdJvQ@wJ={#C7v~>KPG7UJ1^~v@XDg{>x{>Za`01}*~ zjq(kmR~DK9sPqaNKaa+cXrG85?+M^|+=nZbPt#sAOarJToDKlJV}TbJw8Mv1hOO^#*U?dd_`Q&h`!D6+(aclT9*rq~&;1PoBX^ z;hXDpq%j(YWtH?B)O8f!a zWqD1?WInOHo*jhi#AN==C+u?X_AtCv=jG!u(_cN6b!(TO{955U(B7)Og)5xZ?& z84}a=;SG}S=fSZ)q)ikqYRS++P~-qn=G7l(`noCTbo2Di=XNBWKC1xnXI4O)l!EuB zin=xn1kAp_nElQFcWGKIl-o?yhC9Rnbw{)Uf@~tNzOL5E)l1HHt>l-s?#}QKyja5X zIOaAU<56&}-JDOvG`|gm!xx_O-MrPu#SDT7g^w!gLgq@`{l9%^k{E-J zAgRPnzzs&7A8Xnk3qu%clmh`Gr9}kVmll-}Vp>W^$wAaogNx0pU&kU036A4U6Ah{2 zGe^dO+KtlG=Fp7edFBk=4p`jh~q|X zdGcMQZ)Fz%W~V#XA_CK|dFOJ08<-tT;*UERDUl752^ZWMxqX+G1RIzwf;~qYEIL> z3O0vigDVCMfGGxxY?rWoKd{azo{|EZw$vVk2`)%-qTx5W+ZB{*<89jbL z`T`X0QDwqFC5ly(hR5no+LZIYie!0blK_BmT!e1IXq|4BRwDq~QajRL7ucU1biwu* zA4R7%Bi6vU*1gRX&$_P%{tcVE2}Fuz9{$@_w6^%*<+JbH_ilt4-ZU0x`qN`lZNs=G z?JO}cUt;<`OAZumF17C>+(5i1<~62QOpjJ^+7P%g_L1iBRjRL}M%Kh3!;m_K`YKaESoBGk<(IlS@D zOinu*FzHIUjzgmORCM$zJdtt9O<1X`_p$oo%A0Z{HND#QK;*M%|9-^@zJ|na(r`*a zXD3l3aJ2CVV?Xh1Veib|0|$1jvahZ%T;o`w7?|_F1XN;9kK_ zeuhXvd;EICzKJY)WqzNWaT0{hye`Mt!6)@@%LFJtBOc%0HRjykXC`8Rc(%7qI zaXZM6%}6|+<8!a3(d$f?f{Ff0??CKvDa)$KveJC3u9H|tea5S&0782jkw$)?Yh1pr z3x_naoWME@d_#IMx;pWgpb-8^ir>vULfI+U!*OP6ym!$WQjUcl=Wg+IX8Qx$P zku_ZE-?Ppo;qV$8|Ik&T{N9J-Azq6IHa)MYJM`1?%O*7Pjf|Xt|8Sd2L2_Y_z(>eQ zCQrLpewGobg11i$l8G)W9ZHa)$}3h{ml~4+={zVVsjNZwn(*kH^s)vCnrd6yFPd6-hZQPdXo8<~R<)UWX{^ zGk)k1)-OqkH=T%1caH7^HbUK$#=snfiZdHlQ{al=q zAiKe&-3(0w9#`%Hud3W?n6Fl?bT%kgirED@E6#};C(&Y|g&5B1z5*l>1`%lL@zfK@ z=&^KGyi5**mw}Cgs{F}}Q3ni^r+d%|6&v#zZP1k|A;VAh0tUp)B=;fz1V7APZStM^ZJ`G>=kIti$H%A6!I$&q7FH;3|pJ;THMnK%Vvhh1hE zZZ3};!c;^qjkD<$_p0F0m%?IhXYuq~nn^+lFA(=VjC9|uU!`JnHTk_GfLSevxPVT) zlR>*4Su0AsOh3i^$f@5ocJ=1Xg&3TEnR6PhvS7mW?9SEh*omusQ#NgdVLsxnvu338 zUxqL`4a{c&P1nDxHCCKHjkKnRd>|jU!BxKcY?Wc275n!vKU%MDo?-?{NM9H|6X_tF zi3=?9HRhVUPq%c5=cAc5_}yh*{*JZ>PY-@JLYIJ6;_>b$i{u6Jt>a^_20%plNg%p9 z2DT@CCS?=cGisS+|9BZ(gSV773=(Q{j>mI~adOCf;_r47G363yS2Dz+@nk7N0eJ`uhuu) zZ)@vbv3{P%#mwwL97!fsh!gDYl3VA+DQvxxdOqdk!xPJaE@5}@=$BQUD9xlQ^~ARM z2f-8`J1+bW-kqB!1uwrB##rWlZ~kBHQOC78PTT-GyGz604AQ!f>%o1TRIcl?-TE9s zFI)Tx9bMgL^zas|MxXDg2m2r#yO9rk80tFAA5>xQp5TySb;SvLEImeuy6wx+Up`10 z`F7g)+2(x=l|XbX$rSaP?kvC#nLPLwATQgG&0ks>Zg{p8CV*{6R6sDUh34i^Y;l8J zdJ+}HCogb2c%<3rbKCfS)Sb*HuMkgfWyiA3cs>xf+5GOZ7%)qd*ae>9MJYT&t^^s? zZkg6FbZt0|o&Cs>)DnGh!>HV`r%77p5$aQDzT=CCca@-W_hvCf*+|%(@@2J2z^~IU9sDx_s1A#8u z-=Ac~RUA3CaD;i6lEv#Uh8r#Wrf9-z$W0|C=h1P?-2iiqQvlfKtb;#@-*H!@H;ylu z$7YqzTbz8b+q?@{u1e5nVnK8F1a|$F<43_9W6yE@-Mi92Fs1le5TD7n^NR|weFiX2 zVOO}ovg-#%z(I^8Q0&Ni|1;b8_rEDCfZmru*=%P%80b{4o&^VCsdr6@+_eioMDMWo ze7Lde2j1QV2N~&8>ps47>vL3rcP|)Z9O4aO-0cHN4)jQXv$zkCU9b8j-mj4g33o%s z0R%5|srmW&gC|7&n0UPMGvUI-oze(2-Y|l}A>2;EA@3mRL+-lbv0;l&#ph`nJ8U+7 zCr&28!u}tIMtt0To7-k4z(A>OS{#@`3@9E->R?YijMNw zT>8^f+g`)J>%F5fYKcwBI#1pFi+I)R0D#%q>5RZK5Bj(Oh)~IUm+5_jVEMw*zPfs@ zkSGuT@wYvE-vXcSKYr|+%v$VkTi_fyYJZ zM>F|_b82X80E;?D-;Ri$Rp4X(^ABTf>g0&*j^`Pes_=-Zf>X%Y+Iu6x=A4u>8u*oB z3kwSe#ij^bw?XqB#nsl(kZ)YEUOIBiRIUIl@gyeOj9|GlmVi%XO=np7Zf}tHrPrT1 z22B7c*nLEKSFQ~SBBzlWg)5jGX6S0eVz9hiD>9@-<@zEzTGyk(5PsF?xh~}vg403?^bq>kIgW;LwuS{n zuIcG%Ky`m`pvlQEF20S~Tv9Q# z=`|DY0}Qp)*bmFXhe>CVL2(ct=l<#slw>UB1`RS7M6Ly7AV z7xv$~pT>Xmu-Aq_`62fr*p5%+R{=m9JPK^^KR-c$!W0FgCN#jR_>XRj3ZJ)r)}A0M zC@~kTBUzf}zMOdkRFTbB|G@<+yL`=9h9bM45ogn#`J6sW;_)?n0b2S3@Y2!gOh_pg zYs^fXD?~I$XYRdn&xxrk=Z{`a;zbYD9@a z#V}oG6!2~+{wO2} zpa7k75vUD3vSs=G$l)Dt^Is!+A&nLy;E|pf+m^$7OvY1&1K7AVLcpN28HnxuMuzXX z*c1Yn&3$!D%liW(e5v#JJR$opCZ&!@>?2CRv&e>p*L(W~5UGl9`N?@2nXvhwSw;WBeJEW7d3Le#dp}?S0-f15XysEwdax zXJ;*bqTXog@){{ZzAw>j@lWlG1Gy1ZdIht_CMlDP&7C&oa_F!_?m0IY2wJ-ob$b}r zIF+-K(2%tBJDxLHj9D72GrP@=c)xz9Q#?Y=DjNTmPW&HN?#Et+{w?>jH)D?Ny2~Cs zAqO_mx6oKk(;YUjzcyCCqu>+gZ+_m@qW!8DeYYPHORd}WFz3N1?mj!UE6GD+m0tT) z+CBKPYwdFfd?N72&E1@^6_OSJcS6t$_xa3DFXms@<_#zZj`%&_H4aH(1n^mmR#B%# zcm1sgk101d^qm&I26HZToPL!da%{KaV~^jHW6CNjDiPTlY`b0Bz5AE0C{Qo2Et~!q z?h4&a{tj0__WyBv|M^A(Hh+>EZS&iIA)bJ1m+`wU>H4lWXpPR~k(Zy?^@l5;r?WO-;Ci`jRzrOa408v&LW_Fnms1?(5@HtzEYB&g~ddrdroXZ0yosrPYp9oYR5^N zUb$j4;gON>Qhm_drE~e#z8%*W@U=fi_`V9`V>92n88Z+D&c>)OhwZpy(>sihsJ~D)KCT={a?lC-Z+q*3?;zSPN56Jpf63m=y`n*QCRvc)t=;{Jipc(coV_R}4lf)(GR#w@h7I#=(}^tjd-bYZ9ELZCPT)e#OVUNwt-%$r*WngMpnd(8CNENzs!T!s=Df_WQ0~3q zYEbaU5?Asn^(h?ELua4gxM+&uyp&)L48MZCDf+IjD;|8A5p>U$0<4p;-s~Q~y<2BF zSj#ny0EOn{sW7JnV5H@Ek>CUS&Z%K<2LMepdWd_z4iNYSj5TQc(cFBG!W~Bai4#%X z`Om#AUFa!72i*G1qm0HLx5;liN>6-Co%F_Mo@q6=evJgo{B1pVlx7xa&o!LDus$P-;B)USC0jP{?MM7Q)oh=TiYAr#k=X z1+bXMdgk_Ti?%p(Fk5>DXdp`vWvmkz1R6j=Hj&$M*)f7I^Xhd`Kx(OX7Z(oRzQ(%^ zR2p`_-s_etEVhDtqCr#hrtlBpCMa8d1cy7tb3f-Xn?Z0QOa8>u#>CkpkqHU&3{#`^ z!6;xI#C}~w&qufh*x{f6@29K^G$hFBE=xWU>0N^6k`6MT02u{53$*&?)LEiF%aQ(c zElzHLP8A$rEez=GU=alT(K0vrj`5CSP;$A$E9(d9dA93Ymz(&ZB}r~8UIhd;F3Lyo zAwBUdz}Ps{&Y5$AkpO7WBajzS*PnJ)uD%E*&$wvi=1SH9t;1Wp{z5+Cc@QB5+(T#d zs(dmAUhm~}2W^rw5l813fspqQFDBDx)On-4h_!ML7TeD6h}i)fgw&}|$&Yv5nz0_tbC1;#C>@F{5`^34I!53jKMk8%=7t^&T1E{$drgdI_$|wg7G#W$sRK4Hw zd#`jzP$2~`_@sem#E>`!D)X{erCS*~-PT{=^2n4nb7XG)^9L5+Xwec}CaONL?8jZtTM! z0}9VF?;PafSEEF)&2sqJ`6{yCx+(52@9KO+XVTEQW%p4j5GUwBG2Kyrs*YaFO1auX z0ac{*X|7%eix&v|9&R}7x3@KiNhHBpOI(}+Iw|VsE)aaT?It_Wq*7z~ zAq(rL>UaJfius$A@aNEXTNNl{3CEOR!)r9}2|chjeYz%5;jw%Yn@IU2bi|GAjm*fK zpteVTeZY@zbSr+An{vu_G54^v+x$w|p8lEjmfn3QqXdm6kj(Np*pEM+vC?jp-fQD} zh_}>xRO&Bwwti{SrCuo`MXLznc4^btQlbf`Zvv>lp&lg2HZ23^&nRMe4oHV)%C2gn z`AgHLmb!;l)#U}K5V+eC_fJV&s~{Rgg~7ZN#9QDc-@3?q9y6kOV6#?;jZpl2aw2dw z%_Z_1il0sC0u}`sL=o?ipF9`CQixgP%RA$sdl;^9t@NgA4sVpla$o&=l2EO>MJM0@ z-d07R6l$3})V-LPHwv7E3$H)D&Q$dF2%Wm$TReTswd1~S$;l>C`EoXFY8;prlDJ>@ z7n*-+C1o${%eqMJ8shjO$E? zr--uMOZbM8e7~us$~B8@EAROVsv4q&1%Ce%s<+HV3E`t^yD+R$^YMdpCFoHp6kJV3 zpjRcS^c>*7BX2PlgZZM^gjc*Y5A-j}Rom+66k;im;q6Md0w5HfN$z=0>CF|gi1$^k zfCSwEf>&{`fM$7#!1iX^_iRAL6T@bg+2aQuwz_2s^J~!N&;%~ zcx9B3$;djHg5f)qqmbgFD}E#GQN*LwcX#`ic5E2w-H0-;xzt1Z{^CmSs+al+=#LZX zVox(Q2rE(9U~eS@qecI3J=*fPK&h;^t^g-E4zW#f!3Nyg&5>V0kv_YSrn6jRPBvfratDD2Q z^$Atk9t^s^Fam7paiA7pG|pp|(c8#w3PQ(+>SvVCs#nh)htkZOb#k&V?RXEEAM={~ z6HL5rwO&?@2{EnJ$=j?o_VIWU^y))-^grk_gSpIv8KLOgul^jSlU@e(m$dBje~_xS zZw>9;xHog!&?2PA@&sTXJZ}u$ZzRDQv)>IC{;{Hi4s5D=pnvbGKf1*E*&z6>(>rL5QzPMSt zu%k>|!S5gt5KD`Q(C5SvrfEP7R`~gIKv$t>wLf1vpsTG3xOIM?Hh%4*t*D!$&}2|( z_EywNODMUp_y{DbJY8?O9^wNCzjHrX)Z2+Kj>nKDVwv?a1>pyY|ooo{E**rlf$6)7bJCt~=EaN(A zp48uKe#$@f4<3k(%gmF%h#+-eHk#l8kP+AE&R#W{0z0#{8MUzhHaRpuLZ<$UL5V|j zE1;L?wi5`ehXAei6~df+fjH=~EfW&GEq&A`8mLYQWkOB@CCGWCjqb9H8oD!0<$k+8 zvy>aLx0)zss$>=eQK;IYoWe>~Jjv1$++$ScTxN9+$X2bv{*eu;-CASX3m|^Gk{3~e z(%+`Z=Dr&BjEpXAe(?E}=?sG@;BymJ69|+XuOFMBB7T2EzR&^sY<{rTIGW$k<2iA1 zQrBh{1S9=Eb34;+$Q%8;DN?|c&WZvkUYGgWQu8!HEZ)@G4^V7cbD=}?_M0HV^*M0s zekWeu4(w$2L18_`JU)7VD2i*uiN`DJ_Ba&26olZ^8YDhSP%+8TNVXmH?m=Yuq#do` zY1C>P*lvyznxv@DiC5oto;s-O7Mg}b&D%;UIMIgSHBmn+Q6B$IM22h8$i$5f`HZt&d(y$6nDH-Rx2ZLy#A;v@yX)DZid&*WjN{soqQacaI3*VwK)bogL zrKihS#8INVpi7PbJakg&jWS7L7{v=w8RdZ^GXqQM0Piibt+?s~I$8a7VcFs7-y<*S z-nRC>;1#oPS+87b+7Qgl*-E_|*qgAZlm~o2vQ4m4TgszP+v#K>RL4=FAHJcS4L;bM zFtjGabc{`#o4BuLkFKe}_ zmG4r;74p@<$i#wn##+(yB}Rr1Nb-y=pbBNj{P|df_sO@5o%kRB>VL&EH1ZkV0<54% zhou`lgc5rh_xJ+WSuUO5r7BUQhWTY$Y0SN4wN*3PsrHdP-#GhDd0)@ zPT+AH)XYQLl}Eo2&?XWxHO~n-81%Ph$L?MFtj5g1e+quGqS|@zm4G3?6PXWwa+Eo! z?h$Z>Ei%GTB!IH$T;O1*%JOE-|)ouAT)wo!~& z?J4bK*^F3*aARHJLHJ)kd}m&K_gX8|$ea!)QDd56LI!x zd&Ar1ZqT;Er){Bp^7XUr2KXi~a?fMH79Dr_qJ@`yAf!urD$m&KJ?giQNpHM~!|0P4 za!E&Jt0VPFU)8qk_1@X!W;Tl;q)+wqnIP9r*qnUF##f3E_~W6XqdcG;#jcKSt!=u1 zD%~F&>hrppt0Hb{ebs!w3GxddetBfBlaEY%5` z7lxYoCMzSI;y1m@cdXaG^7w^+kP@}g86p?O;f?%D<$SAXo{>ZooQWc`j*MCTCh6pd zy;GVO`yCrCGFkc@`7vIcq$Hl}^R-8yX*$C#xOko%5DCU^#^}ra_!pVMlYnPx+&r}2 zo384lZDZ_m`gkOu;f}gI)N1xznR-d*&^DvwmE)94XL|2?`{!`Xzuqfri0@{78##%j%CT3k+{qkGzQdF8XHoebT57TKTO%k#A-uafljDGCPmK~SW3jR11z6&0yOd!14I_ye&aXmNJd_?R#V`CBKz{6rg zMnVr`U21UUEJQAcZBfo!3RCMm>;4(`EY1JB5oVs0Xkesr^`_VM)?K_OBbKksr+^SB zpp7@n41Ng?rT9{I%;9&E2;0>=eyU_&?sz~`ueKr6-mxr*z5F_tCo_=XIs7Mpt@0Tf zFV<&wk?aJ2wnXYV}f!VRSJU>22-CV0NPQ_9|Zu-Ky0M8r)-cuoydob$+3Q_jKZ##vm9zg|BMrCKM z1@gSb`ql>$pR&pK+;U#5-P+Q8Wm2=5YS~#MM}Ga7+yfVUx3Whg4P-%=(&~E5AA+Qt z6wBsCV}T8I$+S=WN*hJ?KL5~|uWnQ3r7J&Pw;Ns!Q#j6hds@KxDT3IE-B^2}|j zUF8waBNd3sy#~l`TPs!tOl%M~*MgOoNvaC!>zh8HB(e>tv{2)eKKWsh?~~dvYSCnZ zd^d_jo{8dGT#QEW|1O%5t|EE)&4<9^2rRzsL6~5YE3dDcyi3PqJJq9S*($w)#0DZ7Xa9FpkDa#xK9+ zTIi)!ufG#PP(B2=pDowf*D6NAw%yh^H5KP6=(`t+uX;5qw$Z7Cr(ka@%w`xC&&?2z zPM!6;;o_CKjKQ9mT@UX*WWnUA*gYfCRv!nmnD*xR>U!W_cf8mdx+=WMbkoT(*7)SL->_&S8Y^q;i=onOD z@Y8?SqZIN3#Wu$A^R9yJ_Y&YB$J|u(j_!CTK{ck}3ixG~xS+^S&ckmuv%>N$4cjMr$ zN5KG`qlmv=Vn>(&TtvV3U=n%m75WY1u17J`<#lsqN`SvgmETsYQ%w-a!9qjArlgEHEe~~rMZ2g zaAhz!YGcUs#F1SWBwl&z&^t!Y9<6lm;qt$VC5xDCnnIoTTp z=5=P%{tC!pitTF?ZCm ziW2~M)*-l?mVEzy7Cq|CnK9Y&P5fe}v{^%{77#$`7g-dXmGyiAQhJQnuvNy9Roh$J zkN0>Z&PqD|viW#XDX0m2cLil_Bgqt$tJ%d?U0Qji6W4L$3nLc`eYbru72rpgjz{`T zWCWH|20mG|c`uEH`G(=_xp;wbn;AoAa18kpkhR??z>1nac#u?e@53K4{=w1%lb!^R z$*1fR?F}IBN_teYbGV`{XAWzE#gzj1+mc}o3JHMCd;Dli!)B{u#b5p5bOpEpPnt>; zzp4gJ$_j@b-8wC|DOtMI1ZwrSPsOJ4Y`iuXlPL*P zQHReACQ7-4{;IF^)gpuTeJFQ23< zps}xScx7s(g&sb){-#j^YKxuZmU5ll4+e1`WwusEUEQLE(^PM+3uGBkg#j(T^tM&! z?|Ezs`P()pyH-=$(-mW7e^#<;@AYdI4m z=uSjV#;ndf5)KNqwe~36XG3oBUORI({8*HDUjz`;4uwTMP%j*Re7nYAA7NecO00zo zLazW~UnzO<&Zr5J}PtX^5fNp98xG>^Y{Co zn#(Cx4y7QlFr_{&39TE*%L6>vE*hZik+QdM;*%6*5d^>u%dd8$s>ud!$c)x+*N1zS zHXW!C#Tgl~a9p3_KJH+!Zx|wM(U)Is>K=S4uZ~4!ilE|-WiIeO(@aBqof;CYi_zc<4^8JAlWO(2*yZsMb z^e~uJJu3blXrYKPfIwn?)tZM>9Z1;DOP)7!?0_IG{YKNJjUtDUKaXpW571T1lIPIS zq4U6zPQ={e%r~D!F`|)3RgK;BbjH1XLzR=ii61_ESS-8TIXe8HbR>(`#EW^Y+5mLRaMHA42%IJ`XGZiznS-YTSAMB5^IU3l-=~uvOc_Yy)fiEt zu(?&B%!lbDMKjUE-Nm*aEWTo550n83n^|vft67SAVc?2+(m?UaRbbzsLL!Lc+LLJ? zJ~az~S$i?IJNv1Vx?9Bq);Az#_wGdNMt>1-XoGSHn5T+%TkTw;I6P;*=bVl{P)SaU zP}?JO1*$mNru&iBjWE%@Jq%YZ3P27vfZ)qwbFteBTk{y60Z&|OjJe>4a0grRe-a9(wfIsjZcqu>GE&t_APgHdP>DJzf zw+YuUZdJYe{U)2=A0tI`%B1%lzaF!e>r#pIRYL>Qfd>|?>ksQbpSCH?Q*Qty@iN@@RrCbEj(%8Ii-hN7!IKlV^N zFFo*(mnBVY5(pL1e40Pw6Mc5R2uF<=jDR=$JEX*bg>d*c{`Y-V^l*w4t%WAa{3yF@db= z>c$dQ>+^g6{E)b44ZF;iqxD^p<{_UA^W~)`bc;*xE&YB~KJ^5diMOG?VW7~!?IHK* zx-G`#0Rz3dV$|cfZy|f0j1uQ1u%}WV}}2xAqDepo&S! zXE(}$6-^jl!f@kpHqin~;_YevDm(;bYhi5~m41Gg2#^StrA^dG-AJ1crvxe- zQ3Js7CJkux5?Z#3T(~E8AtW?#G#GA=$iT@Hd3Ixsx`jL|y^E6lw|R5rEsM-WlM>?M zZUDe?5%V3Otm}B5mzID}!EjEBf1}6~-mia8ORIGDu<3vFz5nF@87eRw3!8?g2PB@m z#-6?vZcgBihcxmBvo$P99N2kP1pm}fkKq`{iq1esqVJW_%k7apM0CT$BX*}h@(4zb)~cf{_~>!=XG%)>`#%Jicj!) z1m>gDoI;Y4_kp|_8Vn4K)q~DHq~O%u$~F$nj!pic%wfiVePH(qGStIR=A^Fz(J70E zrk_k`Sa4&W5$tY|o7Dc8ddYbbl~hIg8g`|NuTLZ-)Nzm`bw`%RQ%+REOJYG}463?Y zjqW6L^p+w?zvyJ+@8s=JK=nVjHYu;v2w?2-zJlQe%NJj8TbGU*5pix7U67&5#Q|Izo~lBkQL^=aA}I!nLU-5Qe=(T@^j zJj`bM8YLZD@fee9QARq{&y$cMi-1Ex_7bBWcT{g$b-E>&^^J~DSZoUS#NV#LJPd<4 zu8p?}lMiCSpftv@OVvpDOMBxvHaT}p_znHr2HN0WH5!`E*?y5t447!qH)GghYMVSR zv9km6u`6Gt|LB;V443znEB*7)LTuCkD-N{co%zpIv$Xgc1>L$c4>@j;bsxQW!M(AD zgZhR79fvu?_#sE&Z=#&`$b8OclBacE_0e?!7otv>Ro6trOb^#%{m`(>zx9;{&`@hk zT6%hP?&8Bj7oyrV-MsLX8JCD(wY8Gk4V`H=<5k;6*7BqwUuFC9f+DBe*GFLz}06*m8bOzzxd^DkTTk%D=&<7EDgV=J!rf=(JDRr zJLsH2-yS$r9_-NiRl0^9gA`yTmUlwH^TW#@{X-su#NT$U#U}aBqGiFp{xl5`*$Ukk zt&4n<=gQnjil&{dA~*QC!F*Ue`j5eP12hFy-Y}V{9u*$`#Hs3!-OY04kUI$;@n0hN zeZxSUJZ_^KWv6Y`ky0t3y9`c7d7CEQXAgXVPtLi4EB4KHrJxf-6+O;F+EOMK6};@Y z4vIC!m9kAFv@-e{m>S-q=EZET6D{at|24$nAfTeGZSkIefz28PH<2F*Fbl3ZIX)4r zC@z$RK7RZe8xHEOa3F1MPFKaXEQjG_Oj=ut3~C6m!E)QoZqm|bl&P^-yRdfKYyGiJdWA&^+O}T!CZY6N1)zNe7?r*UUAk zy8n>_^D9r;=1xXhd|o^p8PV0%HCGV86s$b*lX1L5iRt};cOci1#ba~P&saBQ?_*YL zQh$*;*b1)PeYX;(ozZDN(=C}0Gblw#4qP7?p2CpSZu-r-8249^i6+-EZkF9?VD1;0 zuqJwS?e|@D?6YZ-u+QcbKxTS2yEJZNzA#Lfkjb$1`06)!qP&G zP}3Hbit-lC2iu`O=;w0AhO29qI#qbSDYyzO&Y@k1g6zt2^U)>&P1SjlZu4DeV-?s9 zqq2dxqTsP64jb0VRs-vOt!IXHZCFtl109*p&Q4pr^EpJd`_}O!#eg%He_$RK`=G0m zsBQ7Ev1+D`f#c%9!zcvVg)FrxvXLc^jeVBFm6?#TFjo0S2)n7@mB-oH`GK&tgZSxi z211T!-D~=6qL0+m`O?YDTyY7KPRVOOpT`_ZHvv(AhU+HhL1)+a?UqSXoaCo8)z}AZ zw<%~j?L&tag>C!JIj+1cJQV8tB4Bd;?PrTt@XQKo_hNd_V~Kku#ODy9%5Uc( zk(Zuy+0KGY<34-vXo~G!*z_^c$vnq=Lb>~e7AavV=@lGaEXDi ze2c;MZ;K}EH)*wAS(<$ox|m>T+?i9sU%uSx-}JaYIhg-E$LZ?q5D~j=pTJs?Cjm1d zm-&mbUeL|et2E6J`v?jPKEB z`QG>aF^|vv2YmX`<1}7gbIx_H>v}F%@+ER#y=782;=-bw-_;`Kt9UVSIpMrP935dd z%q}p5vHRg1{#@O&3w*lT4uXhGMU$+%t1u*_8By7 zz9$>SW?M7MQo-$~&gKhcuC`-JICJxR@R{C<;pUd7M`tfd8094IS)JF>l>NxWYp0xt zVGj{pl3pB=zd)g!O@5~4Ipxp49-gxYQmUPz>aI;j&no znchnWDPV8#KFmkz*86FduP&^ttm04=?|7dM#2x2%rqBqe@EGgUS`3%7R81i`ZL1DB z#dc>D-7GP7!nI6WOK_*Gq}EM*D6?C6PFaQZtOqu8`a)UV3oeI#(^23I-e=^uxlR=* z3=b@bH|Smw+I-j!1}yJyKZE0294dR-0z5R5j?9;cze0yDj8xx?6f!DpD%)@PDaVG? z#x`4%qQvy3dqoxq)K+6}blW!~lThSgqz2_@WBvjnX!fJIQj6`BeSw8uPJwB;+O0PM zV;8=t9ZZIZh;-8L?eFxLO8Y6zG_^3h`|ZmU^LhSj=dqyvP0nwAqr7fbrXPG7ULBX5 zGjaLy_(jg`maEi~AAo(Whe<`&msF? z)Z7O4>Ef_@LoZ6cWqn=1{M5dHrRrk}wIlO93zs@C@g;84D2u-Je7A8j{4PUszU&9L zc3$Q_se9lSwy$8X#r7<9PZ}+KOLgg}6H?n^GRp;1bB9vLOVtj;-k6^K>-J17y!yQr z-=*!XGph%5>Ks@R^?^nq4}K~QZYvz(Zg<Ln$7HvghhXiG34wtS`7WFQc_q~7#UUVOjHM*;?uOrvXVV*m;D*qk}V`R z2ScrNT~J7Ex}M(}gdQ2M|VSY!)$U=W39*W)D|zNAB9};Aq)*^8_i<_8OKO zx$jm^do;?-Pj{P+|Hhu7GaFapwEMNr4hjsOrr?>gOf2sN1ok|~?;6*%@-*i71a86} zoaYa7-+$&HXC!CvKXrWns|<<|1fSRKRdipHAzs4*R)~aN+|>+ervTwl1t>`7tYxwB zu1~xJgK&TqSN3?E{IB2ouU%+i*bZESsehQg>tC}|-iUw|;d_2E^6m`PAOSsGWNKjJ z_1$2&KY#n>9mU+F*9@kl1CajcXw9;=btxz8LVItW@lJrYQ5E1x@Puk4ug*Ih_P?` zMK+tVg~2{gPrrNQg_QG3NJYn8W zVnvtKgxe-h{u5yLk8i{M0jrjN09VKFPQ%Y;ZsC!=%W@3g_bW5uhmS}9O$_NYU~b9R zVX_|i1vxqWN=jL)dusr&BbJXhynYBuN=ojXW$`^fF3kR~weWvD>1S=RokG+af+E$6 zO7Zpm#-AW%Co~|9QZ9+E!YwObDWBFD*^H} z@55(7R3X;cQ@|*dVHt-mj-tu2AXx*@pdVcdoP2Z4xXxcYF!DK@+4Z-A_1ZY@kMmVK z$dM6{NWt$p;atz^>dI&7##8_gT++KHlG?9LfFmr<=kw!h$gvRg){mvbKIl5;mA$*4 z#|%qhzcPz2@8Gx>%=Ljs==K!}^tnu%d=_sK5 zX`SnK(*<(_Y#U$Rk+E%_G)~umb7ct#2d*u|L5u3I!^kx)Ui3pJqA!8;0TVYhWeWrf z`3t~Ex&59N(3#v+g2@fGqqTs^=ViP0W|ro7V5ZcG_9I$i)R1Q`Tu8H53*iSZEZ)p* z5fZ#P@)xMBqD^{*Kp`Fyqe9#&4O$yKsjI8xTBOp(6{*9CIH7t+B8L0Xabu= z_!WRq39gELy;O9eL1<+4R{}E_ar1a25(H1Xvy%@-NT`d%UI` zTuSyMj@m3~Uj5%B}0xk@8y;;Ej z=|(n3iGq@lf2yR7Z#WoWQ1LHMoEu03?u4tYmeQ7WiG3e#;)zHhb-KDIb+O)d`KQ>- zDXSz6(xP}^b;@G?m-)oR0eu_V;;PEVwl938#lM==^AR9rZwFMtr$vJe)5uH=)(br@ zWvWGLYPnr><9*%i79uW{3phY?W)-En8~Akd5RbHNgY5^& z<>h71Z)~{4BTLHB5=MfozUK>C405e0=SGy~81$yPML6U&-6Y0Uw`jUDZY>CBLgD?W zn}+}|i`mla;$B+`j+FR#CzdVHZYq1@b2K zucgpRxLOPtXC_`;d|^i7UY8?iS-6yx*i5Z2Pl|g{?{d{G5$1(>cq9TqtrO&{AZ|p* zx^kbt-yA#t;oQ+nzau(s7n)Z`_cjvURNn?R+z5k8#YID`er(8Fk5;%ys1(JMf$NDE z8WbD1FsNz66zxL>c60GMCu*cQ1kh8|lrQ6!bdi+T)ACsYE2 zxV9hO2RZ13b#_+WK|AI-Nff}{T;w@oiq#4Mlbg0>Axth}>hz0d-mWy`&tp4A%|rVM zTpciW{~7g1u2`_Tee)1(?LduHD!Vg-3+I3+2Y(*urns_sL&sHe0j&bHdHFg@1sK|J zfkv9IS)i$2G-+P;GxNP*)g2jP6d3Jw8(6@0cwH++leJ%WzKz%FhT%aoE8e6uCuwu6 zbP)qwN`$$@pLXiwG*Ge$bS%#UW+)RgsobYj9Sp7N_JaD7dyUkg1Z!ZyEgNhr`jV6~(noC$t6Q`(}Cr{gxR!pr_g zn>@gVtZDlMdaeBlSSf9^^F{yP@5=ot_@getYXJ9zb{He7CiesmwcPJ2jsD_bPr5EMoPh{&52tk8bT#Oln(*Ta~BXi?KE zDpFGY=JzB57K%kutIO?tmw%pEOkmzz!2tHcNF7Jni&_5Nf|G=^wa3v zUxTYK%qt1d06hER#fI}D6pi$9$W;@ENe53n0`2Ip2=W`ntLuxoWOS_$A$mU1?j;dO zU`CIQ)y#Z}oYz7u7MiR8<#wkWL}k~SKx6zTVAkjpL8fTiSd?5URWw;T#L3BdT4o%D zD-NUxl>oIXI>8%UnHaj)iBCA=)6;GC9T&ScRbNpCyR2cr`xWexpd-HefewRwwD-G~ zKOn=D;<%7By2E=)4Vun$9(!ZYOo<0fr}i~}W`-HK7Fy$bU9&M0R#myYHU2}S=r~eJ znR)l$0C-Wc8sBYUDjyNcs;(=}CiQT=F1|6-Z)SRkk&$w?y>18Jj{Rfu`VU#{XKiT? z4^85$Ex7vzgC%?R*$Z=Am^SDf=R3(TR}T$6(Hu;gTTVaz;svYbTQ3d6c+WF*5*u#^wNzTtmPk(Il7Z$v@K~%`A?MR z=pZClB`Z6-1#pML&Rfy*v$B!RYa}4z7Lm1={@aRsKRc|}qUlZY>4WU2qJpXHE$hp@ zQ+;{PX`2}*KUh`Rk`E`8?wzCuUgTeumDr{H)gT?#YKNW|3JFT@Pmza4v91Wl0|~0m z(yv)`zur>p$!2I>kb6l+BIY2 zygjO&9ZOzhio0v!^U~e+MG9M31S{-dO%)({wSIrm>oWhcs%oUCAy^hfsH-_kX!>TY zev*t{q@b#JwjNJRs;bJXNRVk3WjSfkk4|9M zcwYDVSr-|o-@t;Y#g(>rwrr0{Q(tz~2IoO#-D?pE00271He#6g8->ig9{1O9qPev-=~Sq(7ScqX|ZR>9;#^QGxx_l7E59(LXtTf0L9QnHpyx8{Af z7X}YcPlOYHw@*4dC~RzPv^is8NacQzEBUT%Y?5&uGaIB33szQ-wUqFqLUDkO(UT~h z;4bOV|}tHJ7}XvTojI+YD>5$7Yb}4bOTLk84h0_Kgzk?CYfhZ`%%f> ztaiH`rxsu1li!x1eXhgnq&-Ci)3h*%9`JE$;+En|oXrobf4fOB&e7KeCc{qvD%lR_ zEq3hn-5`b}3@K5F>b7Y&sS_dFb7SFZL@Grnc58!caV%C%(Fuq%0Pkw`sHAySC}}CU z+&+=+vaXT<-^)4YW=Tx5OI8UviZYcawKp;Ea^&{xi`4iYtLVW!MGXX)lR}jW08wOs zWRKs!Ul>vAIKI+rBr$Bgl=+tZ9T<5rWX!|2f27JISq^&F3b11KE(2&SX?$2dW3}=7 z_W`(a0&^mjg4PrabWS(-)D_v-n0+MfZ`Qn@XWMpPSN0U08aRCZ=LMUFSS*UaP4w-E z2;X}D>Sl$MnXhhBq?9%;OHBWqNsjw^;1k$;f4sIM2jDKkr*HAO? z3zTPDgQnjb1{!v#)bk_rPymT(mEtKtuk<95GB&0o81!4A6l-h7WroYsG|A1*${H#^ zSn9h8>f-Orc4TfM2X3!D#*LIO`2y zB^^y&BDj<^QImy#i*U7%TBFaf`dT>UG$ zlfDLsR}XLnO~87=G;3l~=~~K~S`_!?Nno-^ zz4JSe{`(EAIw+-x8#pD0(vfRK>xr*%#%OwWPhCJTHPfnfUBTiNb0;}z0gFP?I45Gm z`e$!G83i7JsC%Eyn(Vh+?9>tj50ey&aEnw*I&F7ORn>vhU=Am53EvIbWyct81SXz$ z5Wr4UJ}oBS6X)XNcAyWM+~--_TBmrw#+Q$rLl$%Qns?Zy%FgoepRufPMyckQ)br`% zXo%?=Xo8=$g^Hs>v0eSIQwYiYFe5f)r%()@Pp+!ZP7H&r{caoYRuMeXng#)AyB(AIB&+Zt=tpKG^YAlV4$8P?5;VC4-gD)`c>bsPy$oPH^t?d43#_G z*%({pU9WNKZ#{#|9PCD97g{kp^ajq#`Q+o(*;m--R-U>qi;*HJU_M_1BGL3i1_Z*UanUzMD5j z|H3n`rW}A;8|mp+CELAmx-nwa8-3ESx4?oe1!{fMY;mZ6!Cor&_ez`4kTHj;%w^fP zwEpK<_)yAn4L%>&uP}!SGtLF{!om1G-PSqG(!qK!9Mq*U-)nm3R+naj2}1DNCk-`M z{p#=nLFzug%AycT2Dm($cM7zn^8vNi3QaH=_kPu33+CT7CWz`sAME8o@YxM)xbNMf{SkEzNjYIJz`B+)HpX_ee>uh)ig zaK|~qibj??;TCABk~{GzrDkLjKiq&@+7g%>drVDKBX)Xyob@Y|m?R2+Gsfj@pLha> z;Li8l1^ACsLjEJ?lpLE83U-i4M|yPRra@F^3JfY~G=X`zOIYW0puy6H%+(dhu0c2K zKwajYFe4QVlMDVWZsqF|)yf+(MY!8TJalVofQeIk4VT;)&JdN*4T z<@vmy5RMRqvvH-_kP%HCaF}e6 zdQf-5g%C}*jx(w9(D$e))U;07)iQJrq2E~Ky(u6h7~FNe%3NueP8{@7W-`~i>!47m zAeA2-8ynR^a5I#;utKA#n@#2yxw@&!UAeF4K6k&PqJfNcRY*2z^P3Bh&o_JWnQsri zOwxuetbm5>9h@D}k*GDn%-YVY1b^KShzz;7dNJ6WO{F8`Kzb4t7QCq)iq7F(-z;!S zH&4b!fNDm`!@`uwY@4vhWw?@9jCqO${*!Pix1alSfwsV9$s&4@XHKYN6D#lJ>w8sv zT{4ikkBp-M+;b*{ciL@)26|GW0juJPGM9O`$Zn}o5-vZ>FY~+aOiR@kUiM2==vS&k=(k_d=zl zCWhro?-0ZYhmVpQ;e1*B2?R$aHnuGE@2_Y02Yg%g=GK9dHZk2NV+Xpsm0O*Lyl2($ z;fpis7#?L7IM&-douDWS<%K?(oNfy|IEU;sO z+LUXvLlHQrHa0f4ELK|aJ4cM!OaS=zq;)QQ`GUK(5miP9MJuFSx0;(DCN;(ZcZ~!d zt00VEkG;;5o=yGKc^}76Gb{JJ^2S@YZn@B49eucTG7(Ks(ix^^gzO-oYZeHcJ?)_m z`a{64|(!9yumhBZDR^rtH;z08=<@s_S?QO z{DnS)4>`JddI-3?UIu+@L2BJoM8f25fr=;mk0@t;^BBC)MD~*Cdq?pK=2H1Co&o{& zRJ+pZC**D$)56Poud9+9VsBz5q_rnb;r3Lkyvq%nSYKZcC3JIgAq?`%<7x685&a2U z<*xW*49NxPbnNTH*}C+RuyXJRjonx6O}`j>G2yH;`8s zNlhC?2S$)3UKX2Hr(d5^8m%Lhp(S5TZf{^$=VdtyQX&EcYEU+PlIJ>=f6)BKQ)N|f zGlIlgfjPHP8YO-BJzf2apXIJ?Z}WkDbkUMy{vBbDn;P``R}N0djfGZ5F0=ivnO$Zh zJv$@_{oTC3k1zG$La!sB5pE1+j%nS=oLk;(2dLWGY$>XsDLHN6BR_!i}GCybaxTfKmYIPQbYXfXfY9hMI8JT zY*UR~krW&qXz=Wx><@U#v2kO6Oa2RsFmyYRm<7+N!Gl^04l;Yb?I&1*fP}REw3Kb(H7%>8%4bOcwV?FBQvV z*(+6(SE9peJ~3vUUt-lV@1kZen@!$&xrP)V*W!$To^R2d$|&KeD^eiHTw z(SgOEVA1XK3$}Z~v$LP0fMiJ14K;xhGswuBhxy3?IBnoqo-(K@U8{O_3e+-N!5@b6>k(R5$P#C5+4d&#?OE zG`NefXz6!RuX)05!>$r)GC7mA8GWqC&NB!+M4sARHMVnp|BbU6_TdOD;hZ1@Z+biq zc87DiXC**P%}@Ut@quDO3}N_qDeXD_w&6#K=?=M^E8o6)5>AN8EZQGr@c5w`-Yip* zk&j?Sp4OE;J((`7CJoCGvC`4P9^!NOZ7b*_^2}$*H}l%*MCa|vD&z;9TFw%8{4o>z zwQ4h3>kH%jhZ#XKHkE;j!wduE&C?ulNuy3G!QJrcdAIy*{1ZJ}9R2pZr^&IF6a1on z-cn+Xhnh_2&-*#Ep{u8TRpb|AQ%cN6w{M?`6J{zr>mu7E=KZybVy(ugan7O@tI@|? zpC8|0qZe!5J;y0&`nRDSMBQ4pA)KTi9i3pizuv6x>|cr%=Ul%8mPv`D2*+2xLR=qD z3aMM_nf9GMBjv>Nj$rzTL}rMPf)MuMSVZfJ_(!w6+m-ZDVJ9Fw)TniM ze=+Zl$W3GY9LjQ;x2{^MbO)5XEZdOp&N&W}Lcw74`8#b>F<}2_&pKM6-Mi?g7~WzL z&S4*<+X5wB{0J1~-K8r=2kMlJ&$vS5TXT9i17EGNo}( z4LAqrr&GVT+**@J(dGL2P>ifq!-fd&9jkoHJjOVV%vuRM$<{yVcFc<@kdu0KdAH)_ zPbiLnhkY(r?_4gQ#_(LwYw3y8A=H$5M|m?kqu>ocJl)aXHvNiIhWH4a`1+#9Mhv$8 z=VyNci7)xEcD6%Vp~~4}6GjH;9G7>!6430g7`X7{(xz-fQ8;5 znOV6L&fpu`69)7#fMw+sGBj?#_1|}hn5uu%s{3s{gv0Yn<~hb+Nt75zl%|`QvisE2 z*JUOQB$itdoXE`yB`zUBsP1lqb3Y=W13+RCTt8s);NonbJwtmFq+{h z;9$8YtS}O}YsLB>S&(oK$T4*`1s?8BwBEUi9vlWKWb(^M2H!2Xs-!`7*F)qWFx*=# zW*~VRxb4mOXnBLYDheKcUx>lq8mch>(xz`Y8KMjqWF6j8&zdy*CE_(Ux(EEz)i%^B JylnmW{{b+0-2DIm literal 0 HcmV?d00001 diff --git a/docker/ci/config/windows-servercore-setup.ps1 b/docker/ci/config/windows-servercore-setup.ps1 new file mode 100644 index 0000000000..28c342509c --- /dev/null +++ b/docker/ci/config/windows-servercore-setup.ps1 @@ -0,0 +1,227 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# This script is specifically used on Windows Server Core based docker images + +# Set TLS to 1.2 so SSL/TLS can be enabled for downloading artifacts +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +# Install Scoop as Administrator User here +iex "& {$(irm get.scoop.sh)} -RunAsAdmin" + +# Disable "current" alias directory as it is not preserved after AMI creation +# Use static path in environment variable +scoop config no_junction true + +# Install git +scoop install git +git --version +# Path for git windows usr bin +$fileName = 'nohup.exe' +$fileDir = 'C:\\Users\\ContainerAdministrator\\scoop\\apps\\git' +$fileFound = (Get-ChildItem -Path $fileDir -Filter $fileName -Recurse | %{$_.FullName} | select -first 1) +$fileFound +$gitPathFound = $fileFound.replace("$fileName", '') +$gitPathFound +# Add to EnvVar +$userenv = [System.Environment]::GetEnvironmentVariable("Path", [System.EnvironmentVariableTarget]::User) +[System.Environment]::SetEnvironmentVariable("PATH", $userenv + ";$gitPathFound", [System.EnvironmentVariableTarget]::User) +# Make sure mem size are set to avoid "Out of memory, malloc failed" issues on Windows +git config --system core.packedGitLimit 128m +git config --system core.packedGitWindowSize 128m +git config --system core.longPaths true +git config --system pack.deltaCacheSize 128m +git config --system pack.packSizeLimit 128m +git config --system pack.windowMemory 128m +git config --system pack.window 0 +git config --system pack.threads 1 +git config --system core.compression 0 +git config --system protocol.version 1 +git config --system --list +# Rename system32 find.exe in case it gets conflicted with POSIX find +bash.exe -c "mv -v 'C:\\Windows\\System32\\find.exe' 'C:\\Windows\\System32\\find_windows.exe'" + +# Add some sleep due to a potential race condition +Start-Sleep -Seconds 5 + +# Setup Repos (This has to happen after git is installed or will error out) +scoop bucket add java +scoop bucket add versions +scoop bucket add extras +scoop bucket add github-gh https://github.com/cli/scoop-gh.git + +# Install mingw for k-NN specific requirements with renaming +# Try to lock on to 12.2.0-rt_v10-rev1 as the newer versions on scoop pointed to the ucrt version to replace legacy msvcrt +# https://github.com/opensearch-project/k-NN/issues/829#issuecomment-1499846457 +scoop install https://raw.githubusercontent.com/ScoopInstaller/Main/dad0cee42bb2c0be7acf9f341fba2a55e415e0f2/bucket/mingw.json +$libName = 'libgfortran-5.dll' +$libNameRequired = 'libgfortran-3.dll' +$libDir = 'C:\\Users\\ContainerAdministrator\\scoop\\apps\\mingw' +$libFound = (Get-ChildItem -Path $libDir -Filter $libName -Recurse | %{$_.FullName} | select -first 1) +$libFound +$libPathFound = $libFound.replace("$libName", '') +$libPathFound +mv -v "$libFound" "$libPathFound\\$libNameRequired" +# Add MINGW_BIN path to User Env Var for k-NN to retrieve libs +[System.Environment]::SetEnvironmentVariable("MINGW_BIN", "$libPathFound", [System.EnvironmentVariableTarget]::User) + +# Install zlib for k-NN compilation requirements +scoop install zlib +# Reg PEP +$zlibVersionInfo = (scoop info zlib | out-string -stream | Select-String 'Version.*:') +$zlibVersionNumber = ($zlibVersionInfo -split ':' | select -last 1) +$zlibVersionNumber = $zlibVersionNumber.Trim() +$zlibHome = "C:\\Users\\ContainerAdministrator\\scoop\\apps\\zlib\\$zlibVersionNumber" +$zlibRegFilePath = "$zlibHome\\register.reg" +$zlibRegFilePath +regedit /s $zlibRegFilePath + +# Install jdk +$jdkVersionList = "temurin8-jdk JAVA8_HOME", "temurin11-jdk JAVA11_HOME", "temurin17-jdk JAVA17_HOME", "temurin19-jdk JAVA19_HOME", "openjdk20 JAVA20_HOME", "openjdk14 JAVA14_HOME" +Foreach ($jdkVersion in $jdkVersionList) +{ + $jdkVersion + $jdkArray = $jdkVersion.Split(" ") + $jdkArray[0] + $jdkArray[1] + scoop install $jdkArray[0] + $JAVA_HOME_TEMP = [System.Environment]::GetEnvironmentVariable("JAVA_HOME", [System.EnvironmentVariableTarget]::User).replace("\", "/") + $JAVA_HOME_TEMP + [System.Environment]::SetEnvironmentVariable($jdkArray[1], "$JAVA_HOME_TEMP", [System.EnvironmentVariableTarget]::User) + java -version +} +# Need to reset to jdk11 for Jenkins Agent to start +scoop reset temurin11-jdk +$JAVA_HOME_TEMP = [System.Environment]::GetEnvironmentVariable("JAVA_HOME", [System.EnvironmentVariableTarget]::User).replace("\", "/") +$JAVA_HOME_TEMP +[System.Environment]::SetEnvironmentVariable('JAVA_HOME', "$JAVA_HOME_TEMP", [System.EnvironmentVariableTarget]::User) +java -version + +# Install python and lock onto 3.9.13 now +scoop install https://raw.githubusercontent.com/ScoopInstaller/Versions/13510e2f3fb92b696c203de5a008c0dc62a44220/bucket/python39.json +python --version +# Reg PEP +$versionInfo = (scoop info python39 | out-string -stream | Select-String 'Version.*:') +$versionInfo +$versionNumber = ($versionInfo -split ':' | select -last 1) +$versionNumber +$versionNumber = $versionNumber.Trim() +$versionNumber +$pythonHome = "C:\\Users\\ContainerAdministrator\\scoop\\apps\\python39\\$versionNumber" +$pythonHome +$pythonLibHome = "$pythonHome\\Lib" +$pythonLibHome +$regFilePath = "$pythonHome\\install-pep-514.reg" +$regFilePath +regedit /s $regFilePath +# Windows AMI does not preserve alias directory, copy all the files to an actual directory +New-Item -Path "$pythonHome\\Scripts_temp" -ItemType Directory +Copy-Item -Path "$pythonHome\\Scripts\\*" -Destination "$pythonHome\\Scripts_temp\\" +Remove-Item "$pythonHome\\Scripts" -Force -Recurse +Rename-Item "$pythonHome\\Scripts_temp" "$pythonHome\\Scripts" +# Same as above but different dir +New-Item -Path "$pythonLibHome\\site-packages_temp" -ItemType Directory +Copy-Item -Path "$pythonLibHome\\site-packages\\*" -Destination "$pythonLibHome\\site-packages_temp\\" +Remove-Item "$pythonLibHome\\site-packages" -Force -Recurse +Rename-Item "$pythonLibHome\\site-packages_temp" "$pythonLibHome\\site-packages" + +# Install maven +scoop install maven +mvn --version + +# Install jq 1.6 +scoop install https://raw.githubusercontent.com/ScoopInstaller/Main/96316b49652d5c9672960b39db141962ad4210b0/bucket/jq.json +jq --version + +# Install yq 4.34.2 +scoop install https://raw.githubusercontent.com/ScoopInstaller/Main/1e9c8b3dfa5ebe7a8c270daf7355aa1db9926e00/bucket/yq.json +yq --version + +# Install volta to replace nvm on Windows as Windows is not able to handle symlink after AMI creation +# While Volta is using a fixed location and switch binary version automatically for the Windows Agent +# As of now, volta is locked to version 1.0.8 due to issues installing yarn within docker container +# https://github.com/opensearch-project/opensearch-ci/issues/281#issuecomment-1654424423 +scoop install https://raw.githubusercontent.com/ScoopInstaller/Main/49d6f71e5bd7096d49b3286ad02d5d482726b467/bucket/volta.json +volta --version +$nodeVersionList = "10.24.1","14.19.1","14.20.0", "14.20.1", "14.21.3", "16.20.0", "18.16.0" +Foreach ($nodeVersion in $nodeVersionList) +{ + $nodeVersion + volta install "node@$nodeVersion" + node -v +} +$ref = "main" +$JSON_BASE = "https://raw.githubusercontent.com/opensearch-project/OpenSearch-Dashboards/$ref/package.json" +$yarnVersion = (curl.exe -s -o- $JSON_BASE | yq.exe -r '.engines.yarn') +$yarnVersion +volta install yarn@$yarnVersion +yarn --version +# Temporarily comment out cypress caching due to performance issues on Windows startup +# The opensearch-dashboards-functional-test repo can still run tests +# as cypress will be installed globally by `npm install` when running integtest.sh +# https://github.com/opensearch-project/opensearch-ci/issues/297 +# $cypressVersionList = "5.6.0", "9.5.4", "12.13.0" +# Foreach ($cypressVersion in $cypressVersionList) +# { +# $cypressVersion +# volta install "cypress@$cypressVersion" +# cypress --version +# } +$userenv2 = [System.Environment]::GetEnvironmentVariable("Path", [System.EnvironmentVariableTarget]::User) +$nodePathFixed = "C:\\Users\\ContainerAdministrator\\scoop\\persist\\volta\\appdata\\bin" +[System.Environment]::SetEnvironmentVariable("PATH", $userenv2 + ";$nodePathFixed", [System.EnvironmentVariableTarget]::User) + +# Install chromium (internally it is chrome.exe in app directory) +scoop install chromium +$chromiumName = 'chrome.exe' +$chromiumDir = 'C:\\Users\\ContainerAdministrator\\scoop\\apps\\chromium' +$chromiumFound = (Get-ChildItem -Path $chromiumDir -Filter $chromiumName -Recurse | %{$_.FullName} | select -first 1) +$chromiumFound +# Add BROWSER_PATH path to User Env Var for cypress test to retrieve chromium.exe path +[System.Environment]::SetEnvironmentVariable("BROWSER_PATH", "$chromiumFound", [System.EnvironmentVariableTarget]::User) + +# Install ruby24 +scoop install ruby24 +ruby --version + +# Install gh +scoop install gh +gh version + +# Install dev tools +# Lock to 3.23.3 +scoop install https://raw.githubusercontent.com/ScoopInstaller/Main/56eed69c3bf04110e306f77ad45cfc8c1c5bb9bc/bucket/cmake.json +cmake --version + +# Install zip +scoop install zip + +# Install docker +scoop install docker + +# Scoop cleanup +scoop cache rm * + +# Pip +wget https://bootstrap.pypa.io/get-pip.py -OutFile get-pip.py +python get-pip.py +pip --version +# Install pipenv +pip install pipenv +pipenv --version +# Install awscli +pip install awscli +aws --version +# Cleanup +pip cache remove * +Remove-Item 'get-pip.py' -Force + +# Refresh env vars +$env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User") + +# Enable Long Path +set-ItemProperty -Path HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem LongPathsEnabled -Type DWORD -Value 1 -Force diff --git a/docker/ci/dockerfiles/current/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile new file mode 100644 index 0000000000..77ac08095c --- /dev/null +++ b/docker/ci/dockerfiles/current/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# This is a docker image specifically for standardize the ci/cd environment +# for both developers and ci/cd tools in OpenSearch / OpenSearch-Dashboards +# Please read the README.md file for all the information before using this dockerfile + +FROM rockylinux:8 + +# Ensure localedef running correct with root permission +USER 0 + +# Add normal dependencies +RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo && \ + dnf config-manager --set-enabled powertools && \ + dnf install epel-release -y && dnf repolist && \ + dnf update -y && \ + dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip jq gh + +# Tools setup +COPY --chown=0:0 config/yq-setup.sh /tmp +RUN /tmp/yq-setup.sh + +# Create user group +RUN groupadd -g 1000 opensearch && \ + useradd -u 1000 -g 1000 -d /usr/share/opensearch opensearch && \ + mkdir -p /usr/share/opensearch && \ + chown -R 1000:1000 /usr/share/opensearch + +# Add Python dependencies +RUN dnf install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils + +# Add Dashboards dependencies +RUN dnf install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib + +# Add Notebook dependencies +RUN dnf install -y nss xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc fontconfig freetype && yum clean all + +# Add Yarn dependencies +RUN dnf groupinstall -y "Development Tools" && dnf clean all && rm -rf /var/cache/dnf/* + +# Setup Shared Memory +RUN chmod -R 777 /dev/shm + +# Install PKG builder dependencies with rvm +RUN curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - && \ + curl -sSL https://rvm.io/pkuczynski.asc | gpg2 --import - && \ + curl -sSL https://get.rvm.io | bash -s stable + +# Switch shell for rvm related commands +SHELL ["/bin/bash", "-lc"] + +# Install ruby / rpm / fpm related dependencies +RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && dnf install -y rpm-build createrepo && dnf clean all + +ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin +ENV RVM_HOME=/usr/local/rvm/bin +ENV GEM_HOME=/usr/share/opensearch/.gem +ENV GEM_PATH=$GEM_HOME +ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH + +# Install Python binary +RUN update-alternatives --set python /usr/bin/python3.9 && \ + update-alternatives --set python3 /usr/bin/python3.9 && \ + pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.22.12 + +# Preparation for awscliv2 +#RUN pip3 install git+https://github.com/aws/aws-cli.git@2.11.17 +#ENV AWS_CLI_FILE_ENCODING=UTF-8 + +# Change User +USER 1000 +WORKDIR /usr/share/opensearch + +# Install fpm for opensearch dashboards core +RUN gem install fpm -v 1.14.2 +ENV PATH=/usr/share/opensearch/.gem/gems/fpm-1.14.2/bin:$PATH + +# Hard code node version and yarn version for now +# nvm environment variables +ENV NVM_DIR /usr/share/opensearch/.nvm +ENV NODE_VERSION 10.24.1 +ARG NODE_VERSION_LIST="10.24.1 14.19.1 14.20.0 14.20.1 14.21.3 16.20.0 18.16.0" +# install nvm +# https://github.com/creationix/nvm#install-script +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash +# install node and npm +COPY --chown=1000:1000 config/yarn-version.sh /tmp +RUN source $NVM_DIR/nvm.sh && \ + for node_version in $NODE_VERSION_LIST; do nvm install $node_version; npm install -g yarn@`/tmp/yarn-version.sh main`; done +# add node and npm to path so the commands are available +ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules +ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH +# We use the version test to check if packages installed correctly +# And get added to the PATH +# This will fail the docker build if any of the packages not exist +RUN node -v +RUN npm -v +RUN yarn -v +RUN fpm -v diff --git a/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile index dd4574d6b5..89af112145 100644 --- a/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile @@ -79,9 +79,9 @@ RUN update-alternatives --set python /usr/bin/python3.9 && \ # Add k-NN Library dependencies # EL8 requires install config-manager and enable powertools to consume openblas-static -RUN dnf install epel-release -y && dnf repolist && \ - dnf install -y 'dnf-command(config-manager)' && \ +RUN dnf install -y 'dnf-command(config-manager)' && \ dnf config-manager --set-enabled powertools && \ + dnf install epel-release -y && dnf repolist && \ dnf install openblas-static lapack gcc-gfortran -y && dnf clean all RUN pip3 install cmake==3.23.3 diff --git a/docker/ci/dockerfiles/current/build.windows2019.servercore.x64.dockerfile b/docker/ci/dockerfiles/current/build.windows2019.servercore.x64.dockerfile new file mode 100644 index 0000000000..6b292ae4d2 --- /dev/null +++ b/docker/ci/dockerfiles/current/build.windows2019.servercore.x64.dockerfile @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# This is a docker image specifically for standardize the ci/cd environment +# for both developers and ci/cd tools in OpenSearch / OpenSearch-Dashboards +# Please read the README.md file for all the information before using this dockerfile + +# This image can only by built on a Windows2019 server or higher version + +ARG ServerCoreRepo=mcr.microsoft.com/windows/servercore + +FROM ${ServerCoreRepo}:ltsc2019 + +USER ContainerAdministrator + +COPY config/windows-servercore-setup.ps1 ./ + +RUN powershell ./windows-servercore-setup.ps1 + +RUN powershell -Command Remove-Item './windows-servercore-setup.ps1' -Force + +RUN bash.exe -c "curl https://ci.opensearch.org > /dev/null 2>&1 || echo add certificates" + +RUN bash.exe -c "curl https://artifacts.opensearch.org > /dev/null 2>&1 || echo add certificates" + +WORKDIR "C:/Users/ContainerAdministrator" + +CMD ["powershell.exe"] diff --git a/docker/ci/dockerfiles/current/release.centos7.clients.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/release.centos7.clients.x64.arm64.dockerfile index 55d270c3d6..7735d98693 100644 --- a/docker/ci/dockerfiles/current/release.centos7.clients.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/release.centos7.clients.x64.arm64.dockerfile @@ -51,8 +51,14 @@ RUN if [[ `uname -m` = 'aarch64' ]]; then mkdir -p aarch64-builds && cd aarch64- ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib # Tools setup -COPY --chown=0:0 config/jdk-setup.sh config/yq-setup.sh /tmp/ -RUN /tmp/jdk-setup.sh && /tmp/yq-setup.sh +COPY --chown=0:0 config/yq-setup.sh /tmp/ +RUN /tmp/yq-setup.sh + +# Install JDK +RUN curl -SL https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.15%2B10/OpenJDK11U-jdk_x64_linux_hotspot_11.0.15_10.tar.gz -o /opt/jdk11.tar.gz && \ + mkdir -p /opt/java/openjdk-11 && \ + tar -xzf /opt/jdk11.tar.gz --strip-components 1 -C /opt/java/openjdk-11/ && \ + rm /opt/jdk11.tar.gz # Create user group RUN groupadd -g 1000 opensearch && \ @@ -60,6 +66,10 @@ RUN groupadd -g 1000 opensearch && \ mkdir -p /usr/share/opensearch && \ chown -R 1000:1000 /usr/share/opensearch +# ENV JDK +ENV JAVA_HOME=/opt/java/openjdk-11 +ENV PATH=$PATH:$JAVA_HOME/bin + # Installing higher version of maven 3.8.x RUN export MAVEN_URL=`curl -s https://maven.apache.org/download.cgi | grep -Eo '["\047].*.bin.tar.gz["\047]' | tr -d '"'` && \ mkdir -p $MAVEN_DIR && (curl -s $MAVEN_URL | tar xzf - --strip-components=1 -C $MAVEN_DIR) && \ diff --git a/docker/ci/dockerfiles/current/build.centos7.opensearch-dashboards.x64.arm64.dockerfile b/docker/ci/dockerfiles/legacy/build.centos7.opensearch-dashboards.x64.arm64.dockerfile similarity index 100% rename from docker/ci/dockerfiles/current/build.centos7.opensearch-dashboards.x64.arm64.dockerfile rename to docker/ci/dockerfiles/legacy/build.centos7.opensearch-dashboards.x64.arm64.dockerfile diff --git a/docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/legacy/build.centos7.opensearch.x64.arm64.dockerfile similarity index 99% rename from docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile rename to docker/ci/dockerfiles/legacy/build.centos7.opensearch.x64.arm64.dockerfile index ec46156328..56bd2edeae 100644 --- a/docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/legacy/build.centos7.opensearch.x64.arm64.dockerfile @@ -89,7 +89,7 @@ RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \ # Add k-NN Library dependencies RUN yum install epel-release -y && yum repolist && yum install openblas-static lapack gcc-gfortran -y RUN pip3 install cmake==3.23.3 -# Upgrade gcc8 +# Upgrade gcc # The setup part is partially based on Austin Dewey's article: # https://austindewey.com/2019/03/26/enabling-software-collections-binaries-on-a-docker-image/ RUN yum install -y centos-release-scl && yum install -y devtoolset-7 && yum clean all && \ diff --git a/docker/ci/dockerfiles/current/test.centos7.performance-test.x64.arm64.dockerfile b/docker/ci/dockerfiles/legacy/test.centos7.performance-test.x64.arm64.dockerfile similarity index 100% rename from docker/ci/dockerfiles/current/test.centos7.performance-test.x64.arm64.dockerfile rename to docker/ci/dockerfiles/legacy/test.centos7.performance-test.x64.arm64.dockerfile diff --git a/jenkins/check-for-build.jenkinsfile b/jenkins/check-for-build.jenkinsfile index dfa2c762d7..078f5ff70d 100644 --- a/jenkins/check-for-build.jenkinsfile +++ b/jenkins/check-for-build.jenkinsfile @@ -14,16 +14,16 @@ pipeline { } triggers { parameterizedCron ''' + H 1 * * * %INPUT_MANIFEST=1.3.13/opensearch-1.3.13.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip H 1 * * * %INPUT_MANIFEST=2.9.1/opensearch-2.9.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip - H 1 * * * %INPUT_MANIFEST=2.10.0/opensearch-dashboards-2.10.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip - H 1 * * * %INPUT_MANIFEST=2.10.0/opensearch-2.10.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip - H 1 * * * %INPUT_MANIFEST=1.3.12/opensearch-1.3.12.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip + H 1 * * * %INPUT_MANIFEST=2.10.0/opensearch-dashboards-2.10.0.yml;TEST_MANIFEST=2.10.0/opensearch-dashboards-2.10.0-test.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip + H 1 * * * %INPUT_MANIFEST=2.10.0/opensearch-2.10.0.yml;TEST_MANIFEST=2.10.0/opensearch-2.10.0-test.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip H 1 * * * %INPUT_MANIFEST=2.8.1/opensearch-dashboards-2.8.1.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip H 1 * * * %INPUT_MANIFEST=2.8.1/opensearch-2.8.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip H 1 * * * %INPUT_MANIFEST=1.4.0/opensearch-1.4.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm zip H 1 * * * %INPUT_MANIFEST=1.4.0/opensearch-dashboards-1.4.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm zip H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TEST_MANIFEST=3.0.0/opensearch-3.0.0-test.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip - H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip + H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TEST_MANIFEST=opensearch-dashboards-3.0.0-test.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip ''' } parameters { diff --git a/jenkins/cross-cluster-replication/perf-test.jenkinsfile b/jenkins/cross-cluster-replication/perf-test.jenkinsfile index ebfbe66f43..c7e507db53 100644 --- a/jenkins/cross-cluster-replication/perf-test.jenkinsfile +++ b/jenkins/cross-cluster-replication/perf-test.jenkinsfile @@ -9,18 +9,11 @@ pipeline { timeout(time: 10, unit: 'HOURS') } environment { - AGENT_LABEL = 'Jenkins-Agent-AL2023-X64-M52xlarge-Docker-Host-Perf-Test' + AGENT_LABEL = 'Jenkins-Agent-AL2023-X64-M52xlarge-Docker-Host-Benchmark-Test' AGENT_IMAGE = 'opensearchstaging/ci-runner:ci-runner-centos7-performance-test-v3' BUNDLE_MANIFEST = 'bundle-manifest.yml' JOB_NAME = 'ccr-perf-test' } - triggers { - parameterizedCron ''' - H 0 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.5.0/latest/linux/x64/tar/dist/opensearch/manifest.yml; - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.5.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml; - - ''' - } parameters { string( name: 'BUNDLE_MANIFEST_URL', @@ -28,7 +21,6 @@ pipeline { trim: true ) } - stages { stage('validate-and-set-parameters') { agent { diff --git a/jenkins/docker/docker-build.jenkinsfile b/jenkins/docker/docker-build.jenkinsfile index d52e9d0de8..2b6c2f0afd 100644 --- a/jenkins/docker/docker-build.jenkinsfile +++ b/jenkins/docker/docker-build.jenkinsfile @@ -1,3 +1,18 @@ +def docker_images = [ + "linux": "opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.9.1-qemu5.0-awscli1.22-jdk11-v1", + "windows": "opensearchstaging/ci-runner:ci-runner-windows2019-servercore-opensearch-build-v1" +] + +def docker_args = [ + "linux": "-u root -v /var/run/docker.sock:/var/run/docker.sock", + "windows": "-u ContainerAdministrator -v //./pipe/docker_engine://./pipe/docker_engine" +] + +def agent_nodes = [ + "linux": "Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder", + "windows": "Jenkins-Agent-Windows2019-X64-C54xlarge-Docker-Builder", +] + pipeline { options { timeout(time: 5, unit: 'HOURS') @@ -22,12 +37,17 @@ pipeline { description: 'The script path of the docker build script, assuming you are already in root dir of DOCKER_BUILD_GIT_REPOSITORY.', trim: true ) + choice( + name: 'DOCKER_BUILD_OS', + description: 'Build docker image on which operating system.', + choices: ['linux', 'windows'], + ) } stages { stage('Parameters Check') { steps { script { - if(! DOCKER_BUILD_GIT_REPOSITORY.startsWith('https://github.com/opensearch-project/')) { + if (! DOCKER_BUILD_GIT_REPOSITORY.startsWith('https://github.com/opensearch-project/')) { currentBuild.result = 'ABORTED' error('The repository needs to be an opensearch-project repository') } @@ -37,9 +57,9 @@ pipeline { stage('docker-build') { agent { docker { - label 'Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder' - image 'opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.9.1-qemu5.0-awscli1.22-jdk11-v1' - args '-u root -v /var/run/docker.sock:/var/run/docker.sock' + label agent_nodes["${DOCKER_BUILD_OS}"] + image docker_images["${DOCKER_BUILD_OS}"] + args docker_args["${DOCKER_BUILD_OS}"] registryUrl 'https://public.ecr.aws/' alwaysPull true } diff --git a/jenkins/gradle/gradle-check.jenkinsfile b/jenkins/gradle/gradle-check.jenkinsfile index 2490a3d23f..b0218d0b44 100644 --- a/jenkins/gradle/gradle-check.jenkinsfile +++ b/jenkins/gradle/gradle-check.jenkinsfile @@ -40,7 +40,7 @@ pipeline { choice( name: 'AGENT_LABEL', description: 'Choose which jenkins agent to run gradle check on', - choices: ['Jenkins-Agent-Ubuntu2004-X64-M58xlarge-Single-Host', 'Jenkins-Agent-Windows2019-X64-C524xlarge-Single-Host', 'Jenkins-Agent-Ubuntu2004-X64-C524xlarge-Single-Host'], + choices: ['Jenkins-Agent-Ubuntu2004-X64-M58xlarge-Single-Host', 'Jenkins-Agent-Windows2019-X64-C524xlarge-Single-Host'], ) } triggers { @@ -58,12 +58,6 @@ pipeline { printPostContent: false ) parameterizedCron ''' - H 1 * * * %GIT_REFERENCE=main;AGENT_LABEL=Jenkins-Agent-Windows2019-X64-C524xlarge-Single-Host - H 4 * * * %GIT_REFERENCE=main;AGENT_LABEL=Jenkins-Agent-Windows2019-X64-C524xlarge-Single-Host - H 9 * * * %GIT_REFERENCE=1.x;AGENT_LABEL=Jenkins-Agent-Windows2019-X64-C524xlarge-Single-Host - H 12 * * * %GIT_REFERENCE=1.x;AGENT_LABEL=Jenkins-Agent-Windows2019-X64-C524xlarge-Single-Host - H 17 * * * %GIT_REFERENCE=2.x;AGENT_LABEL=Jenkins-Agent-Windows2019-X64-C524xlarge-Single-Host - H 20 * * * %GIT_REFERENCE=2.x;AGENT_LABEL=Jenkins-Agent-Windows2019-X64-C524xlarge-Single-Host H 3 * * * %GIT_REFERENCE=main;AGENT_LABEL=Jenkins-Agent-Ubuntu2004-X64-M58xlarge-Single-Host H 6 * * * %GIT_REFERENCE=1.x;AGENT_LABEL=Jenkins-Agent-Ubuntu2004-X64-M58xlarge-Single-Host H 9 * * * %GIT_REFERENCE=2.x;AGENT_LABEL=Jenkins-Agent-Ubuntu2004-X64-M58xlarge-Single-Host diff --git a/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy b/jenkins/legacy/TestRunNonSecurityPerfTestScript.groovy.legacy similarity index 100% rename from tests/jenkins/TestRunNonSecurityPerfTestScript.groovy rename to jenkins/legacy/TestRunNonSecurityPerfTestScript.groovy.legacy diff --git a/jenkins/opensearch/perf-test.jenkinsfile b/jenkins/legacy/perf-test.jenkinsfile similarity index 75% rename from jenkins/opensearch/perf-test.jenkinsfile rename to jenkins/legacy/perf-test.jenkinsfile index d4c443d754..149a5dee09 100644 --- a/jenkins/opensearch/perf-test.jenkinsfile +++ b/jenkins/legacy/perf-test.jenkinsfile @@ -7,36 +7,14 @@ pipeline { agent none options { timeout(time: 14, unit: 'DAYS') + buildDiscarder(logRotator(daysToKeepStr: '30')) } environment { - AGENT_LABEL = 'Jenkins-Agent-AL2023-X64-M52xlarge-Docker-Host-Perf-Test' + AGENT_LABEL = 'Jenkins-Agent-AL2023-X64-M52xlarge-Docker-Host-Benchmark-Test' AGENT_IMAGE = 'opensearchstaging/ci-runner:ci-runner-centos7-performance-test-v3' BUNDLE_MANIFEST = 'bundle-manifest.yml' JOB_NAME = 'perf-test' } - triggers { - parameterizedCron ''' - H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.11/8029/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 - H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.11/8029/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 - H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.11/8029/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 - H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.11/8029/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 - - H 8 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.7.0/7771/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 - H 8 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.7.0/7771/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 - H 8 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.7.0/7771/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 - H 8 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.7.0/7771/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 - - H 8 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 - H 8 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 - H 8 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 - H 8 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 - - H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 - H 4 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 - H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 - H 4 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 - ''' - } parameters { string( name: 'BUNDLE_MANIFEST_URL', diff --git a/jenkins/opensearch-dashboards/bwc-test.jenkinsfile b/jenkins/opensearch-dashboards/bwc-test.jenkinsfile index d68c64c96e..18166baa31 100644 --- a/jenkins/opensearch-dashboards/bwc-test.jenkinsfile +++ b/jenkins/opensearch-dashboards/bwc-test.jenkinsfile @@ -6,6 +6,7 @@ lib = library(identifier: 'jenkins@1.0.4', retriever: modernSCM([ pipeline { options { timeout(time: 4, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '30')) } agent none environment { diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile index 783eb9476a..4aaced393b 100644 --- a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile +++ b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile @@ -1,4 +1,4 @@ -lib = library(identifier: 'jenkins@5.4.1', retriever: modernSCM([ +lib = library(identifier: 'jenkins@5.6.0', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', ])) @@ -75,6 +75,11 @@ pipeline { description: 'To create a github issue for failing component or not.', defaultValue: true ) + booleanParam( + name: 'CONTINUE_ON_ERROR', + description: 'Continue building the distribution even if a one or more component fails', + defaultValue: true + ) } stages { stage('verify-parameters') { @@ -131,31 +136,6 @@ pipeline { } } } - stage('Initialize the job info yaml report') { - agent { - docker { - label AGENT_X64 - image dockerAgent.image - registryUrl 'https://public.ecr.aws/' - alwaysPull true - } - } - steps { - script { - buildInfoYaml( - componentName: COMPONENT_NAME, - inputManifest: "manifests/$INPUT_MANIFEST", - status: "NOT_STARTED", - stage: "INITIALIZE_STAGE" - ) - } - } - post { - always { - postCleanup() - } - } - } stage('build') { parallel { stage('build-and-test-linux-x64-tar') { @@ -186,7 +166,8 @@ pipeline { inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'x64', - distribution: 'tar' + distribution: 'tar', + continueOnError: params.CONTINUE_ON_ERROR ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String buildManifestUrlOpenSearch = [buildManifestObj.getArtifactRootUrl(JOB_NAME_OPENSEARCH, "latest"), "builds", "opensearch", "manifest.yml"].join("/") @@ -275,6 +256,7 @@ pipeline { platform: 'linux', architecture: 'x64', distribution: 'rpm', + continueOnError: params.CONTINUE_ON_ERROR, stashName: "build-archive-linux-x64-rpm-${JOB_NAME}-${BUILD_NUMBER}" ) } @@ -302,6 +284,7 @@ pipeline { platform: 'linux', architecture: 'x64', distribution: 'rpm', + continueOnError: params.CONTINUE_ON_ERROR, stashName: "build-archive-linux-x64-rpm-${JOB_NAME}-${BUILD_NUMBER}" ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) @@ -370,6 +353,7 @@ pipeline { platform: 'linux', architecture: 'x64', distribution: 'deb', + continueOnError: params.CONTINUE_ON_ERROR, stashName: "build-archive-linux-x64-deb-${JOB_NAME}-${BUILD_NUMBER}" ) } @@ -397,6 +381,7 @@ pipeline { platform: 'linux', architecture: 'x64', distribution: 'deb', + continueOnError: params.CONTINUE_ON_ERROR, stashName: "build-archive-linux-x64-deb-${JOB_NAME}-${BUILD_NUMBER}" ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) @@ -457,6 +442,7 @@ pipeline { platform: 'linux', architecture: 'arm64', distribution: 'tar', + continueOnError: params.CONTINUE_ON_ERROR, stashName: "build-archive-linux-arm64-tar-${JOB_NAME}-${BUILD_NUMBER}" ) } @@ -491,6 +477,7 @@ pipeline { platform: 'linux', architecture: 'arm64', distribution: 'tar', + continueOnError: params.CONTINUE_ON_ERROR, stashName: "build-archive-linux-arm64-tar-${JOB_NAME}-${BUILD_NUMBER}" ) @@ -581,6 +568,7 @@ pipeline { platform: 'linux', architecture: 'arm64', distribution: 'rpm', + continueOnError: params.CONTINUE_ON_ERROR, stashName: "build-archive-linux-arm64-rpm-${JOB_NAME}-${BUILD_NUMBER}" ) } @@ -614,6 +602,7 @@ pipeline { platform: 'linux', architecture: 'arm64', distribution: 'rpm', + continueOnError: params.CONTINUE_ON_ERROR, stashName: "build-archive-linux-arm64-rpm-${JOB_NAME}-${BUILD_NUMBER}" ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) @@ -681,6 +670,7 @@ pipeline { platform: 'linux', architecture: 'arm64', distribution: 'deb', + continueOnError: params.CONTINUE_ON_ERROR, stashName: "build-archive-linux-arm64-deb-${JOB_NAME}-${BUILD_NUMBER}" ) } @@ -708,6 +698,7 @@ pipeline { platform: 'linux', architecture: 'arm64', distribution: 'deb', + continueOnError: params.CONTINUE_ON_ERROR, stashName: "build-archive-linux-arm64-deb-${JOB_NAME}-${BUILD_NUMBER}" ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) @@ -760,7 +751,8 @@ pipeline { inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'windows', architecture: 'x64', - distribution: "zip" + distribution: "zip", + continueOnError: params.CONTINUE_ON_ERROR ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) @@ -851,18 +843,21 @@ pipeline { node(AGENT_X64) { checkout scm script { - buildInfoYaml( - componentName: COMPONENT_NAME, - status: currentBuild.result, - stage: "FINALIZE_STAGE" - ) - unstash "buildInfo_yml" - archiveArtifacts artifacts: 'buildInfo.yml' + def findFailedPlugins = buildMessage(search: 'Failed plugins') + if (findFailedPlugins != null) { + currentBuild.result = 'UNSTABLE' + } closeBuildSuccessGithubIssue( message: buildMessage(search: 'Successfully built'), - search: "Successfully built", + search: 'Successfully built', inputManifestPath: "manifests/$INPUT_MANIFEST" ) + if (params.CREATE_GITHUB_ISSUE) { + createBuildFailureGithubIssue( + message: buildMessage(search: 'Error building'), + inputManifestPath: "manifests/$INPUT_MANIFEST" + ) + } postCleanup() } } @@ -919,13 +914,6 @@ pipeline { manifest: "${INPUT_MANIFEST}" ) } - if (params.CREATE_GITHUB_ISSUE) { - createBuildFailureGithubIssue( - message: buildMessage(search: 'Error building'), - inputManifestPath: "manifests/$INPUT_MANIFEST" - ) - } - postCleanup() } } diff --git a/jenkins/opensearch-dashboards/integ-test.jenkinsfile b/jenkins/opensearch-dashboards/integ-test.jenkinsfile index 15f3ce2257..5b9e5456ad 100644 --- a/jenkins/opensearch-dashboards/integ-test.jenkinsfile +++ b/jenkins/opensearch-dashboards/integ-test.jenkinsfile @@ -1,6 +1,6 @@ -lib = library(identifier: 'jenkins@jenkins_report', retriever: modernSCM([ +lib = library(identifier: 'jenkins@5.6.0', retriever: modernSCM([ $class: 'GitSCMSource', - remote: 'https://github.com/zelinh/opensearch-build-libraries.git', + remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', ])) def docker_images = [ @@ -23,6 +23,7 @@ def agent_nodes = [ pipeline { options { timeout(time: 4, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '30')) } agent none environment { @@ -198,22 +199,22 @@ pipeline { switchUserNonRoot: "${switch_user_non_root}" ) String closeCommentMessage = "Closing the issue as the Integration Test passed for ${local_component}
Version: ${version}
Distribution: ${distribution}
Architecture: ${architecture}
Platform: ${platform}

Please check the logs: ${RUN_DISPLAY_URL}

*" -// closeGithubIssue( -// repoUrl: buildManifestObj.getRepo("${local_component}"), -// issueTitle: "[AUTOCUT] Integration Test failed for ${local_component}: ${version} ${distribution} distribution", -// closeComment: closeCommentMessage, -// label: "autocut,v${version},integ-test-failure" -// ) + closeGithubIssue( + repoUrl: buildManifestObj.getRepo("${local_component}"), + issueTitle: "[AUTOCUT] Integration Test failed for ${local_component}: ${version} ${distribution} distribution", + closeComment: closeCommentMessage, + label: "autocut,v${version},integ-test-failure" + ) } } catch (e) { echo "Error running integtest for component ${local_component}, creating Github issue" String issueBodyMessage = "The integration test failed at distribution level for component ${local_component}
Version: ${version}
Distribution: ${distribution}
Architecture: ${architecture}
Platform: ${platform}

Please check the logs: ${RUN_DISPLAY_URL}

* Steps to reproduce: See https://github.com/opensearch-project/opensearch-build/tree/main/src/test_workflow#integration-tests
* See all log files:
- [With security](https://ci.opensearch.org/ci/dbc/${JOB_NAME}/${version}/${buildId}/${platform}/${architecture}/${distribution}/test-results/${BUILD_NUMBER}/integ-test/${local_component}/with-security/${local_component}.yml) (if applicable)
- [Without security](https://ci.opensearch.org/ci/dbc/${JOB_NAME}/${version}/${buildId}/${platform}/${architecture}/${distribution}/test-results/${BUILD_NUMBER}/integ-test/${local_component}/without-security/${local_component}.yml) (if applicable)

" -// createGithubIssue( -// repoUrl: buildManifestObj.getRepo("${local_component}"), -// issueTitle: "[AUTOCUT] Integration Test failed for ${local_component}: ${version} ${distribution} distribution", -// issueBody: issueBodyMessage, -// label: "autocut,v${version},integ-test-failure" -// ) + createGithubIssue( + repoUrl: buildManifestObj.getRepo("${local_component}"), + issueTitle: "[AUTOCUT] Integration Test failed for ${local_component}: ${version} ${distribution} distribution", + issueBody: issueBodyMessage, + label: "autocut,v${version},integ-test-failure" + ) throw new Exception("Error running integtest for component ${local_component}", e) } finally { echo "Completed running integtest for component ${local_component}" @@ -234,14 +235,27 @@ pipeline { } post { always { - createTestRunManifest( - testManifest: "manifests/${TEST_MANIFEST}", - buildManifest: BUILD_MANIFEST_OPENSEARCH, - buildManifestDashboards: BUILD_MANIFEST, - testRunID: "${env.BUILD_NUMBER}", - testType: "integ-test", - ) - sh("cat test-run.yml") + script { + node(AGENT_LABEL) { + docker.withRegistry('https://public.ecr.aws/') { + docker.image(docker_images["$distribution"]).inside(docker_args["$distribution"]) { + unstash "integtest-opensearch-dashboards-$BUILD_NUMBER" + downloadBuildManifest( + url: BUILD_MANIFEST_URL_OPENSEARCH, + path: BUILD_MANIFEST_OPENSEARCH + ) + createUploadTestReportManifest( + testManifest: "manifests/${TEST_MANIFEST}", + buildManifest: BUILD_MANIFEST_OPENSEARCH, + dashboardsBuildManifest: BUILD_MANIFEST, + testRunID: "${env.BUILD_NUMBER}", + testType: "integ-test", + componentName: "${COMPONENT_NAME}", + ) + } + } + } + } postCleanup() } } diff --git a/jenkins/opensearch/benchmark-test.jenkinsfile b/jenkins/opensearch/benchmark-test.jenkinsfile index 855d7e050c..fac57c3589 100644 --- a/jenkins/opensearch/benchmark-test.jenkinsfile +++ b/jenkins/opensearch/benchmark-test.jenkinsfile @@ -1,4 +1,4 @@ -lib = library(identifier: 'jenkins@5.1.0', retriever: modernSCM([ +lib = library(identifier: 'jenkins@5.5.0', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', ])) @@ -7,6 +7,7 @@ pipeline { agent none options { timeout(time: 24, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '30')) } environment { AGENT_LABEL = 'Jenkins-Agent-AL2023-X64-M52xlarge-Docker-Host-Benchmark-Test' @@ -14,70 +15,74 @@ pipeline { JOB_NAME = 'benchmark-test' } triggers { - parameterizedCron ''' - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} + parameterizedCron( + ''' + H 0 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.7.0/7771/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 0 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.7.0/7771/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 0 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 0 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} + H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + + H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-0-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-2-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-4-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-6-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-0-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-2-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-4-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-6-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-0-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-2-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-4-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-6-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-0-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-2-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-4-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-6-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - + H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + ''' + + + ''' H 15 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-arm64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0"} H 15 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0"} H 15 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-arm64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0"} @@ -87,6 +92,28 @@ pipeline { H 15 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-arm64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0"} H 15 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-1-shard;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-0-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-2-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-4-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-6-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-1-shard;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-0-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-2-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-4-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-6-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-1-shard;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-0-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-2-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-4-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-6-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-1-shard;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-0-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-2-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-4-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-6-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 18 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-arm64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-min-distribution-zstd-no-dict-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd_no_dict","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} H 18 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-zstd-no-dict-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd_no_dict","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} H 18 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-arm64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-min-distribution-zstd-no-dict-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd_no_dict","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} @@ -96,6 +123,7 @@ pipeline { H 18 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-arm64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-single-node-min-distribution-zstd-no-dict-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd_no_dict","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} H 18 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-zstd-no-dict-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd_no_dict","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} ''' + ) } parameters { string( @@ -154,6 +182,11 @@ pipeline { description: 'Number of cluster ml nodes, empty value defaults to 0.', trim: true ) + string( + name: 'DATA_INSTANCE_TYPE', + description: 'EC2 instance type for data node, empty defaults to r5.xlarge.', + trim: true + ) string( name: 'DATA_NODE_STORAGE', description: 'Data node ebs block storage size, empty value defaults to 100Gb', @@ -194,6 +227,11 @@ pipeline { description: 'Enable opensearch-benchmark node-stats telemetry to capture system level metrics.', defaultValue: false ) + booleanParam( + name: 'CAPTURE_SEGMENT_REPLICATION_STAT', + description: 'Enable opensearch-benchmark segment-replication-stats telemetry to capture metrics such as replication lag.', + defaultValue: false + ) } stages { @@ -255,13 +293,15 @@ pipeline { clientNodeCount: CLIENT_NODE_COUNT, ingestNodeCount: INGEST_NODE_COUNT, mlNodeCount: ML_NODE_COUNT, + dataInstanceType: DATA_INSTANCE_TYPE, userTag: USER_TAGS.isEmpty() ? "security-enabled:true" : "${USER_TAGS},security-enabled:true", workloadParams: WORKLOAD_PARAMS, additionalConfig: ADDITIONAL_CONFIG, dataStorageSize: DATA_NODE_STORAGE, mlStorageSize: ML_NODE_STORAGE, jvmSysProps: JVM_SYS_PROPS, - captureNodeStat: CAPTURE_NODE_STAT + captureNodeStat: CAPTURE_NODE_STAT, + captureSegmentReplicationStat: CAPTURE_SEGMENT_REPLICATION_STAT ) lib.jenkins.Messages.new(this).add(JOB_NAME, lib.jenkins.Messages.new(this).get([JOB_NAME]) + @@ -300,13 +340,15 @@ pipeline { clientNodeCount: CLIENT_NODE_COUNT, ingestNodeCount: INGEST_NODE_COUNT, mlNodeCount: ML_NODE_COUNT, + dataInstanceType: DATA_INSTANCE_TYPE, userTag: USER_TAGS.isEmpty() ? "security-enabled:false" : "${USER_TAGS},security-enabled:false", workloadParams: WORKLOAD_PARAMS, additionalConfig: ADDITIONAL_CONFIG, dataStorageSize: DATA_NODE_STORAGE, mlStorageSize: ML_NODE_STORAGE, jvmSysProps: JVM_SYS_PROPS, - captureNodeStat: CAPTURE_NODE_STAT + captureNodeStat: CAPTURE_NODE_STAT, + captureSegmentReplicationStat: CAPTURE_SEGMENT_REPLICATION_STAT ) lib.jenkins.Messages.new(this).add(JOB_NAME, lib.jenkins.Messages.new(this).get([JOB_NAME]) + diff --git a/jenkins/opensearch/bwc-test.jenkinsfile b/jenkins/opensearch/bwc-test.jenkinsfile index eb9f6cab6c..23166124b7 100644 --- a/jenkins/opensearch/bwc-test.jenkinsfile +++ b/jenkins/opensearch/bwc-test.jenkinsfile @@ -6,6 +6,7 @@ lib = library(identifier: 'jenkins@1.0.4', retriever: modernSCM([ pipeline { options { timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '30')) } agent none environment { diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile index a31901d8da..6e45e6719e 100644 --- a/jenkins/opensearch/distribution-build.jenkinsfile +++ b/jenkins/opensearch/distribution-build.jenkinsfile @@ -1,4 +1,4 @@ -lib = library(identifier: 'jenkins@5.4.1', retriever: modernSCM([ +lib = library(identifier: 'jenkins@5.6.0', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', ])) @@ -68,6 +68,11 @@ pipeline { description: 'To create a github issue for failing component or not.', defaultValue: true ) + booleanParam( + name: 'CONTINUE_ON_ERROR', + description: 'Continue building the distribution even if a one or more component fails', + defaultValue: true + ) } stages { stage('verify-parameters') { @@ -105,31 +110,6 @@ pipeline { } } } - stage('Initialize the job info yaml report') { - agent { - docker { - label AGENT_X64 - image dockerAgent.image - registryUrl 'https://public.ecr.aws/' - alwaysPull true - } - } - steps { - script { - buildInfoYaml( - componentName: COMPONENT_NAME, - inputManifest: "manifests/$INPUT_MANIFEST", - status: "NOT_STARTED", - stage: "INITIALIZE_STAGE" - ) - } - } - post { - always { - postCleanup() - } - } - } stage('build') { parallel { stage('build-opensearch-snapshot-linux-x64-tar') { @@ -312,7 +292,8 @@ pipeline { inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'x64', - distribution: "tar" + distribution: "tar", + continueOnError: params.CONTINUE_ON_ERROR ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) @@ -395,6 +376,7 @@ pipeline { platform: 'linux', architecture: 'x64', distribution: 'rpm', + continueOnError: params.CONTINUE_ON_ERROR, stashName: "build-archive-linux-x64-rpm-${JOB_NAME}-${BUILD_NUMBER}" ) } @@ -422,6 +404,7 @@ pipeline { platform: 'linux', architecture: 'x64', distribution: 'rpm', + continueOnError: params.CONTINUE_ON_ERROR, stashName: "build-archive-linux-x64-rpm-${JOB_NAME}-${BUILD_NUMBER}" ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) @@ -486,6 +469,7 @@ pipeline { platform: 'linux', architecture: 'x64', distribution: 'deb', + continueOnError: params.CONTINUE_ON_ERROR, stashName: "build-archive-linux-x64-deb-${JOB_NAME}-${BUILD_NUMBER}" ) } @@ -513,6 +497,7 @@ pipeline { platform: 'linux', architecture: 'x64', distribution: 'deb', + continueOnError: params.CONTINUE_ON_ERROR, stashName: "build-archive-linux-x64-deb-${JOB_NAME}-${BUILD_NUMBER}" ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) @@ -564,7 +549,8 @@ pipeline { inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'arm64', - distribution: "tar" + distribution: "tar", + continueOnError: params.CONTINUE_ON_ERROR ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) @@ -646,6 +632,7 @@ pipeline { platform: 'linux', architecture: 'arm64', distribution: 'rpm', + continueOnError: params.CONTINUE_ON_ERROR, stashName: "build-archive-linux-arm64-rpm-${JOB_NAME}-${BUILD_NUMBER}" ) } @@ -673,6 +660,7 @@ pipeline { platform: 'linux', architecture: 'arm64', distribution: 'rpm', + continueOnError: params.CONTINUE_ON_ERROR, stashName: "build-archive-linux-arm64-rpm-${JOB_NAME}-${BUILD_NUMBER}" ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) @@ -737,6 +725,7 @@ pipeline { platform: 'linux', architecture: 'arm64', distribution: 'deb', + continueOnError: params.CONTINUE_ON_ERROR, stashName: "build-archive-linux-arm64-deb-${JOB_NAME}-${BUILD_NUMBER}" ) } @@ -764,6 +753,7 @@ pipeline { platform: 'linux', architecture: 'arm64', distribution: 'deb', + continueOnError: params.CONTINUE_ON_ERROR, stashName: "build-archive-linux-arm64-deb-${JOB_NAME}-${BUILD_NUMBER}" ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) @@ -814,7 +804,8 @@ pipeline { inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'windows', architecture: 'x64', - distribution: "zip" + distribution: "zip", + continueOnError: params.CONTINUE_ON_ERROR ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) @@ -903,18 +894,21 @@ pipeline { node(AGENT_X64) { checkout scm script { - buildInfoYaml( - componentName: COMPONENT_NAME, - status: currentBuild.result, - stage: "FINALIZE_STAGE" - ) - unstash "buildInfo_yml" - archiveArtifacts artifacts: 'buildInfo.yml' + def findFailedPlugins = buildMessage(search: 'Failed plugins') + if (findFailedPlugins != null) { + currentBuild.result = 'UNSTABLE' + } closeBuildSuccessGithubIssue( message: buildMessage(search: 'Successfully built'), - search: "Successfully built", + search: 'Successfully built', inputManifestPath: "manifests/$INPUT_MANIFEST" ) + if (params.CREATE_GITHUB_ISSUE) { + createBuildFailureGithubIssue( + message: buildMessage(search: 'Error building'), + inputManifestPath: "manifests/$INPUT_MANIFEST" + ) + } postCleanup() } } @@ -964,13 +958,6 @@ pipeline { manifest: "${INPUT_MANIFEST}" ) } - if (params.CREATE_GITHUB_ISSUE) { - createBuildFailureGithubIssue( - message: buildMessage(search: 'Error building'), - inputManifestPath: "manifests/$INPUT_MANIFEST" - ) - } - postCleanup() } } diff --git a/jenkins/opensearch/integ-test.jenkinsfile b/jenkins/opensearch/integ-test.jenkinsfile index 920a931c78..f81b7622bf 100644 --- a/jenkins/opensearch/integ-test.jenkinsfile +++ b/jenkins/opensearch/integ-test.jenkinsfile @@ -1,4 +1,4 @@ -lib = library(identifier: 'jenkins@5.4.1', retriever: modernSCM([ +lib = library(identifier: 'jenkins@5.6.0', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', ])) @@ -23,6 +23,7 @@ def agent_nodes = [ pipeline { options { timeout(time: 4, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '30')) } agent none environment { @@ -30,6 +31,12 @@ pipeline { BUILD_JOB_NAME = "distribution-build-opensearch" ARTIFACT_BUCKET_NAME = credentials('jenkins-artifact-bucket-name') } + triggers { + parameterizedCron ''' + H 3 * * * %TEST_MANIFEST=2.10.0/opensearch-2.10.0-concurrent-search-test.yml;BUILD_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/builds/opensearch/manifest.yml + H 3 * * * %TEST_MANIFEST=2.10.0/opensearch-2.10.0-concurrent-search-test.yml;BUILD_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/builds/opensearch/manifest.yml + ''' + } parameters { string( name: 'COMPONENT_NAME', @@ -198,13 +205,22 @@ pipeline { } post { always { - createTestRunManifest( - testManifest: "manifests/${TEST_MANIFEST}", - buildManifest: BUILD_MANIFEST, - testRunID: "${env.BUILD_NUMBER}", - testType: "integ-test", - ) - sh("cat test-run.yml") + script { + node(AGENT_LABEL) { + docker.withRegistry('https://public.ecr.aws/') { + docker.image(docker_images["$distribution"]).inside(docker_args["$distribution"]) { + unstash "integtest-opensearch-$BUILD_NUMBER" + createUploadTestReportManifest( + testManifest: "manifests/${TEST_MANIFEST}", + buildManifest: BUILD_MANIFEST, + testRunID: "${env.BUILD_NUMBER}", + testType: "integ-test", + componentName: "${COMPONENT_NAME}", + ) + } + } + } + } postCleanup() } } diff --git a/jenkins/opensearch/maven-publish-1.3.x.jenkinsfile b/jenkins/opensearch/maven-publish-1.3.x.jenkinsfile index f1305525e0..00066f9ea9 100644 --- a/jenkins/opensearch/maven-publish-1.3.x.jenkinsfile +++ b/jenkins/opensearch/maven-publish-1.3.x.jenkinsfile @@ -13,7 +13,7 @@ pipeline { } triggers { parameterizedCron ''' - H 1 * * * %INPUT_MANIFEST=1.3.11/opensearch-1.3.11.yml + H 1 * * * %INPUT_MANIFEST=1.3.12/opensearch-1.3.12.yml ''' } parameters { diff --git a/manifests/1.3.12/opensearch-1.3.12-test.yml b/manifests/1.3.12/opensearch-1.3.12-test.yml new file mode 100644 index 0000000000..21cd9ec744 --- /dev/null +++ b/manifests/1.3.12/opensearch-1.3.12-test.yml @@ -0,0 +1,69 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2 + args: -e JAVA_HOME=/opt/java/openjdk-11 +components: + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + + - name: anomaly-detection + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + + - name: asynchronous-search + integ-test: + test-configs: + - with-security + - without-security + + - name: alerting + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + plugins.destination.host.deny_list: [10.0.0.0/8, 127.0.0.1] + + - name: sql + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + script.context.field.max_compilations_rate: 1000/1m + + - name: k-NN + integ-test: + test-configs: + - with-security + - without-security + + - name: opensearch-reports + integ-test: + test-configs: + - without-security + + - name: opensearch-observability + integ-test: + test-configs: + - without-security + + - name: ml-commons + integ-test: + test-configs: + - with-security + - without-security diff --git a/manifests/1.3.12/opensearch-1.3.12.yml b/manifests/1.3.12/opensearch-1.3.12.yml index 97fecf1c1e..7136194047 100644 --- a/manifests/1.3.12/opensearch-1.3.12.yml +++ b/manifests/1.3.12/opensearch-1.3.12.yml @@ -10,7 +10,132 @@ ci: components: - name: OpenSearch repository: https://github.com/opensearch-project/OpenSearch.git - ref: '1.3' + ref: tags/1.3.12 checks: - gradle:publish - gradle:properties:version + - name: common-utils + repository: https://github.com/opensearch-project/common-utils.git + ref: tags/1.3.12.0 + checks: + - gradle:publish + - gradle:properties:version + platforms: + - linux + - windows + - name: job-scheduler + repository: https://github.com/opensearch-project/job-scheduler.git + ref: tags/1.3.12.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + - name: alerting + repository: https://github.com/opensearch-project/alerting.git + ref: tags/1.3.12.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: alerting + platforms: + - linux + - windows + - name: opensearch-reports + repository: https://github.com/opensearch-project/reporting.git + ref: tags/1.3.12.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + - name: cross-cluster-replication + repository: https://github.com/opensearch-project/cross-cluster-replication.git + ref: tags/1.3.12.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + - name: security + repository: https://github.com/opensearch-project/security.git + ref: tags/1.3.12.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + - name: ml-commons + repository: https://github.com/opensearch-project/ml-commons.git + ref: tags/1.3.12.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: opensearch-ml-plugin + platforms: + - linux + - windows + - name: asynchronous-search + repository: https://github.com/opensearch-project/asynchronous-search.git + ref: tags/1.3.12.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + - name: sql + repository: https://github.com/opensearch-project/sql.git + ref: tags/1.3.12.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: plugin + platforms: + - linux + - windows + - name: k-NN + repository: https://github.com/opensearch-project/k-NN.git + ref: tags/1.3.12.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + - name: opensearch-observability + repository: https://github.com/opensearch-project/observability.git + ref: tags/1.3.12.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + - name: anomaly-detection + repository: https://github.com/opensearch-project/anomaly-detection.git + ref: tags/1.3.12.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + - name: index-management + repository: https://github.com/opensearch-project/index-management.git + ref: tags/1.3.12.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + - name: performance-analyzer + repository: https://github.com/opensearch-project/performance-analyzer.git + ref: tags/1.3.12.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux diff --git a/manifests/1.3.12/opensearch-dashboards-1.3.12-test.yml b/manifests/1.3.12/opensearch-dashboards-1.3.12-test.yml new file mode 100644 index 0000000000..d3226fa041 --- /dev/null +++ b/manifests/1.3.12/opensearch-dashboards-1.3.12-test.yml @@ -0,0 +1,52 @@ +--- +schema-version: '1.0' +name: OpenSearch Dashboards +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v4 +components: + - name: OpenSearch-Dashboards + bwc-test: + test-configs: + - with-security + - without-security + - name: anomalyDetectionDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: ganttChartDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: observabilityDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: alertingDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: indexManagementDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: reportsDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: securityDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: queryWorkbenchDashboards + integ-test: + test-configs: + - with-security + - without-security diff --git a/manifests/1.3.12/opensearch-dashboards-1.3.12.yml b/manifests/1.3.12/opensearch-dashboards-1.3.12.yml new file mode 100644 index 0000000000..e46bd46610 --- /dev/null +++ b/manifests/1.3.12/opensearch-dashboards-1.3.12.yml @@ -0,0 +1,40 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch Dashboards + version: 1.3.12 +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v4 +components: + - name: OpenSearch-Dashboards + repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git + ref: tags/1.3.12 + - name: anomalyDetectionDashboards + repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin + ref: tags/1.3.12.0 + - name: functionalTestDashboards + repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git + ref: tags/1.3.12 + - name: ganttChartDashboards + repository: https://github.com/opensearch-project/dashboards-visualizations.git + working_directory: gantt-chart + ref: tags/1.3.12.0 + - name: observabilityDashboards + repository: https://github.com/opensearch-project/dashboards-observability.git + ref: tags/1.3.12.0 + - name: alertingDashboards + repository: https://github.com/opensearch-project/alerting-dashboards-plugin.git + ref: tags/1.3.12.0 + - name: indexManagementDashboards + repository: https://github.com/opensearch-project/index-management-dashboards-plugin + ref: tags/1.3.12.0 + - name: reportsDashboards + repository: https://github.com/opensearch-project/dashboards-reporting.git + ref: tags/1.3.12.0 + - name: securityDashboards + repository: https://github.com/opensearch-project/security-dashboards-plugin.git + ref: tags/1.3.12.0 + - name: queryWorkbenchDashboards + repository: https://github.com/opensearch-project/dashboards-query-workbench.git + ref: tags/1.3.12.0 diff --git a/manifests/1.3.13/opensearch-1.3.13.yml b/manifests/1.3.13/opensearch-1.3.13.yml new file mode 100644 index 0000000000..fbe60b2db0 --- /dev/null +++ b/manifests/1.3.13/opensearch-1.3.13.yml @@ -0,0 +1,16 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch + version: 1.3.13 +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 + args: -e JAVA_HOME=/opt/java/openjdk-11 +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: '1.3' + checks: + - gradle:publish + - gradle:properties:version diff --git a/manifests/2.10.0/opensearch-2.10.0-concurrent-search-test.yml b/manifests/2.10.0/opensearch-2.10.0-concurrent-search-test.yml new file mode 100644 index 0000000000..9c3bfeb230 --- /dev/null +++ b/manifests/2.10.0/opensearch-2.10.0-concurrent-search-test.yml @@ -0,0 +1,189 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 + args: -e JAVA_HOME=/opt/java/openjdk-17 +components: + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + opensearch.experimental.feature.concurrent_segment_search.enabled: true + search.concurrent_segment_search.enabled: true + search.concurrent.max_slice_count: 2 + bwc-test: + test-configs: + - with-security + + - name: anomaly-detection + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + opensearch.experimental.feature.concurrent_segment_search.enabled: true + search.concurrent_segment_search.enabled: true + search.concurrent.max_slice_count: 2 + bwc-test: + test-configs: + - with-security + + - name: asynchronous-search + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + opensearch.experimental.feature.concurrent_segment_search.enabled: true + search.concurrent_segment_search.enabled: true + search.concurrent.max_slice_count: 2 + + - name: alerting + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + plugins.destination.host.deny_list: [10.0.0.0/8, 127.0.0.1] + opensearch.experimental.feature.concurrent_segment_search.enabled: true + search.concurrent_segment_search.enabled: true + search.concurrent.max_slice_count: 2 + bwc-test: + test-configs: + - with-security + + - name: notifications + working-directory: notifications + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + opensearch.experimental.feature.concurrent_segment_search.enabled: true + search.concurrent_segment_search.enabled: true + search.concurrent.max_slice_count: 2 + bwc-test: + test-configs: + - with-security + + - name: sql + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + script.context.field.max_compilations_rate: 1000/1m + opensearch.experimental.feature.concurrent_segment_search.enabled: true + search.concurrent_segment_search.enabled: true + search.concurrent.max_slice_count: 2 + bwc-test: + test-configs: + - with-security + + - name: k-NN + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + opensearch.experimental.feature.concurrent_segment_search.enabled: true + search.concurrent_segment_search.enabled: true + search.concurrent.max_slice_count: 2 + + - name: neural-search + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + opensearch.experimental.feature.concurrent_segment_search.enabled: true + search.concurrent_segment_search.enabled: true + search.concurrent.max_slice_count: 2 + + - name: opensearch-reports + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + opensearch.experimental.feature.concurrent_segment_search.enabled: true + search.concurrent_segment_search.enabled: true + search.concurrent.max_slice_count: 2 + + - name: opensearch-observability + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + opensearch.experimental.feature.concurrent_segment_search.enabled: true + search.concurrent_segment_search.enabled: true + search.concurrent.max_slice_count: 2 + bwc-test: + test-configs: + - with-security + + - name: ml-commons + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + opensearch.experimental.feature.concurrent_segment_search.enabled: true + search.concurrent_segment_search.enabled: true + search.concurrent.max_slice_count: 2 + + - name: cross-cluster-replication + integ-test: + topology: + - cluster_name: leader + data_nodes: 2 + cluster_manager_nodes: 0 + - cluster_name: follower + data_nodes: 2 + cluster_manager_nodes: 0 + test-configs: + - with-security + - without-security + additional-cluster-configs: + opensearch.experimental.feature.concurrent_segment_search.enabled: true + search.concurrent_segment_search.enabled: true + search.concurrent.max_slice_count: 2 + + - name: security + integ-test: + test-configs: + - with-security + additional-cluster-configs: + opensearch.experimental.feature.concurrent_segment_search.enabled: true + search.concurrent_segment_search.enabled: true + search.concurrent.max_slice_count: 2 + + - name: geospatial + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + opensearch.experimental.feature.concurrent_segment_search.enabled: true + search.concurrent_segment_search.enabled: true + search.concurrent.max_slice_count: 2 + + - name: security-analytics + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + opensearch.experimental.feature.concurrent_segment_search.enabled: true + search.concurrent_segment_search.enabled: true + search.concurrent.max_slice_count: 2 diff --git a/manifests/2.10.0/opensearch-2.10.0-test.yml b/manifests/2.10.0/opensearch-2.10.0-test.yml new file mode 100644 index 0000000000..ee3d4331dd --- /dev/null +++ b/manifests/2.10.0/opensearch-2.10.0-test.yml @@ -0,0 +1,133 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 + args: -e JAVA_HOME=/opt/java/openjdk-17 +components: + - name: alerting + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + plugins.destination.host.deny_list: [10.0.0.0/8, 127.0.0.1] + bwc-test: + test-configs: + - with-security + + - name: anomaly-detection + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + + - name: asynchronous-search + integ-test: + test-configs: + - with-security + - without-security + + - name: cross-cluster-replication + integ-test: + topology: + - cluster_name: leader + data_nodes: 1 + cluster_manager_nodes: 0 + - cluster_name: follower + data_nodes: 1 + cluster_manager_nodes: 0 + test-configs: + - with-security + - without-security + + - name: geospatial + integ-test: + test-configs: + - with-security + - without-security + + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + bwc-test: + test-configs: + - with-security + + - name: k-NN + integ-test: + test-configs: + - with-security + - without-security + + - name: ml-commons + integ-test: + test-configs: + - with-security + - without-security + + - name: neural-search + integ-test: + test-configs: + - with-security + - without-security + + - name: notifications + working-directory: notifications + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + + - name: opensearch-observability + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + + - name: opensearch-reports + integ-test: + test-configs: + - with-security + - without-security + + - name: security + integ-test: + test-configs: + - with-security + + - name: security-analytics + integ-test: + test-configs: + - with-security + - without-security + + - name: sql + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + script.context.field.max_compilations_rate: 1000/1m + plugins.query.datasources.encryption.masterkey: 4fc8fee6a3fd7d6ca01772e5 + bwc-test: + test-configs: + - with-security diff --git a/manifests/2.10.0/opensearch-dashboards-2.10.0-test.yml b/manifests/2.10.0/opensearch-dashboards-2.10.0-test.yml new file mode 100644 index 0000000000..3a3ac1c35f --- /dev/null +++ b/manifests/2.10.0/opensearch-dashboards-2.10.0-test.yml @@ -0,0 +1,82 @@ +--- +schema-version: '1.0' +name: OpenSearch Dashboards +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v4 +components: + - name: OpenSearch-Dashboards + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + vis_builder.enabled: true + data_source.enabled: true + - name: alertingDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: anomalyDetectionDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: ganttChartDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: indexManagementDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: observabilityDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: queryWorkbenchDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: reportsDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: securityDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: notificationsDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: customImportMapDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: searchRelevanceDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: securityAnalyticsDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: mlCommonsDashboards + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + ml_commons_dashboards.enabled: true diff --git a/manifests/2.10.0/opensearch-dashboards-2.10.0.yml b/manifests/2.10.0/opensearch-dashboards-2.10.0.yml index 6ad6e5f2fd..2621168d75 100644 --- a/manifests/2.10.0/opensearch-dashboards-2.10.0.yml +++ b/manifests/2.10.0/opensearch-dashboards-2.10.0.yml @@ -10,3 +10,46 @@ components: - name: OpenSearch-Dashboards repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git ref: 2.x + - name: functionalTestDashboards + repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git + ref: 2.x + - name: observabilityDashboards + repository: https://github.com/opensearch-project/dashboards-observability.git + ref: 2.x + - name: reportsDashboards + repository: https://github.com/opensearch-project/dashboards-reporting.git + ref: 2.x + - name: ganttChartDashboards + repository: https://github.com/opensearch-project/dashboards-visualizations.git + working_directory: gantt-chart + ref: 2.x + - name: queryWorkbenchDashboards + repository: https://github.com/opensearch-project/dashboards-query-workbench.git + ref: 2.x + - name: customImportMapDashboards + repository: https://github.com/opensearch-project/dashboards-maps.git + ref: 2.x + - name: anomalyDetectionDashboards + repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin + ref: 2.x + - name: mlCommonsDashboards + repository: https://github.com/opensearch-project/ml-commons-dashboards.git + ref: 2.x + - name: indexManagementDashboards + repository: https://github.com/opensearch-project/index-management-dashboards-plugin.git + ref: 2.x + - name: notificationsDashboards + repository: https://github.com/opensearch-project/dashboards-notifications.git + ref: 2.x + - name: alertingDashboards + repository: https://github.com/opensearch-project/alerting-dashboards-plugin.git + ref: 2.x + - name: securityAnalyticsDashboards + repository: https://github.com/opensearch-project/security-analytics-dashboards-plugin.git + ref: 2.x + - name: securityDashboards + repository: https://github.com/opensearch-project/security-dashboards-plugin.git + ref: 2.x + - name: searchRelevanceDashboards + repository: https://github.com/opensearch-project/dashboards-search-relevance.git + ref: 2.x diff --git a/manifests/3.0.0/opensearch-3.0.0-test.yml b/manifests/3.0.0/opensearch-3.0.0-test.yml index c355a5af43..d0c36dba83 100644 --- a/manifests/3.0.0/opensearch-3.0.0-test.yml +++ b/manifests/3.0.0/opensearch-3.0.0-test.yml @@ -6,6 +6,34 @@ ci: name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 args: -e JAVA_HOME=/opt/java/openjdk-20 components: + - name: alerting + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + plugins.destination.host.deny_list: [10.0.0.0/8, 127.0.0.1] + bwc-test: + test-configs: + - with-security + + - name: anomaly-detection + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + + - name: asynchronous-search + integ-test: + test-configs: + - with-security + - without-security + - name: cross-cluster-replication integ-test: topology: @@ -18,3 +46,88 @@ components: test-configs: - with-security - without-security + + - name: geospatial + integ-test: + test-configs: + - with-security + - without-security + + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + bwc-test: + test-configs: + - with-security + + - name: k-NN + integ-test: + test-configs: + - with-security + - without-security + + - name: ml-commons + integ-test: + test-configs: + - with-security + - without-security + + - name: neural-search + integ-test: + test-configs: + - with-security + - without-security + + - name: notifications + working-directory: notifications + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + + - name: opensearch-observability + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + + - name: opensearch-reports + integ-test: + test-configs: + - with-security + - without-security + + - name: security + integ-test: + test-configs: + - with-security + + - name: security-analytics + integ-test: + test-configs: + - with-security + - without-security + + - name: sql + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + script.context.field.max_compilations_rate: 1000/1m + plugins.query.datasources.encryption.masterkey: 4fc8fee6a3fd7d6ca01772e5 + bwc-test: + test-configs: + - with-security diff --git a/manifests/3.0.0/opensearch-3.0.0.yml b/manifests/3.0.0/opensearch-3.0.0.yml index b4cba5fd74..ab2886d7e4 100644 --- a/manifests/3.0.0/opensearch-3.0.0.yml +++ b/manifests/3.0.0/opensearch-3.0.0.yml @@ -156,3 +156,28 @@ components: checks: - gradle:properties:version - gradle:dependencies:opensearch.version: opensearch-sql-plugin + - name: asynchronous-search + repository: https://github.com/opensearch-project/asynchronous-search.git + ref: main + platforms: + - linux + - windows + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: security-analytics + repository: https://github.com/opensearch-project/security-analytics.git + ref: main + platforms: + - linux + - windows + checks: + - gradle:properties:version + - name: index-management + repository: https://github.com/opensearch-project/index-management.git + ref: main + platforms: + - linux + - windows + checks: + - gradle:properties:version diff --git a/manifests/3.0.0/opensearch-dashboards-3.0.0-test.yml b/manifests/3.0.0/opensearch-dashboards-3.0.0-test.yml index 9191335905..3a3ac1c35f 100644 --- a/manifests/3.0.0/opensearch-dashboards-3.0.0-test.yml +++ b/manifests/3.0.0/opensearch-dashboards-3.0.0-test.yml @@ -3,29 +3,37 @@ schema-version: '1.0' name: OpenSearch Dashboards ci: image: - name: opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v3 + name: opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v4 components: - name: OpenSearch-Dashboards - bwc-test: + integ-test: test-configs: - with-security - without-security - - name: notificationsDashboards + additional-cluster-configs: + vis_builder.enabled: true + data_source.enabled: true + - name: alertingDashboards integ-test: test-configs: - with-security - without-security - - name: observabilityDashboards + - name: anomalyDetectionDashboards integ-test: test-configs: - with-security - without-security - - name: reportsDashboards + - name: ganttChartDashboards integ-test: test-configs: - with-security - without-security - - name: ganttChartDashboards + - name: indexManagementDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: observabilityDashboards integ-test: test-configs: - with-security @@ -35,12 +43,17 @@ components: test-configs: - with-security - without-security + - name: reportsDashboards + integ-test: + test-configs: + - with-security + - without-security - name: securityDashboards integ-test: test-configs: - with-security - without-security - - name: searchRelevanceDashboards + - name: notificationsDashboards integ-test: test-configs: - with-security @@ -50,3 +63,20 @@ components: test-configs: - with-security - without-security + - name: searchRelevanceDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: securityAnalyticsDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: mlCommonsDashboards + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + ml_commons_dashboards.enabled: true diff --git a/manifests/3.0.0/opensearch-dashboards-3.0.0.yml b/manifests/3.0.0/opensearch-dashboards-3.0.0.yml index 8984e9e141..95808e3946 100644 --- a/manifests/3.0.0/opensearch-dashboards-3.0.0.yml +++ b/manifests/3.0.0/opensearch-dashboards-3.0.0.yml @@ -38,3 +38,18 @@ components: - name: customImportMapDashboards repository: https://github.com/opensearch-project/dashboards-maps.git ref: main + - name: anomalyDetectionDashboards + repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin + ref: main + - name: mlCommonsDashboards + repository: https://github.com/opensearch-project/ml-commons-dashboards.git + ref: main + - name: indexManagementDashboards + repository: https://github.com/opensearch-project/index-management-dashboards-plugin.git + ref: main + - name: alertingDashboards + repository: https://github.com/opensearch-project/alerting-dashboards-plugin.git + ref: main + - name: securityAnalyticsDashboards + repository: https://github.com/opensearch-project/security-analytics-dashboards-plugin.git + ref: main diff --git a/manifests/templates/opensearch/1.x/non-os-template-1.1.0.yml b/manifests/templates/opensearch/1.x/non-os-template-1.1.0.yml new file mode 100644 index 0000000000..76260c52d3 --- /dev/null +++ b/manifests/templates/opensearch/1.x/non-os-template-1.1.0.yml @@ -0,0 +1,30 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch + version: 1.1.0 +components: + - name: common-utils + repository: https://github.com/opensearch-project/common-utils.git + ref: tags/1.1.0.0 + checks: + - gradle:publish + - gradle:properties:version + - name: job-scheduler + repository: https://github.com/opensearch-project/job-scheduler.git + ref: tags/1.1.0.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: sql + repository: https://github.com/opensearch-project/sql.git + ref: tags/1.1.0.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: plugin + - name: alerting + repository: https://github.com/opensearch-project/alerting.git + ref: tags/1.1.0.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: alerting diff --git a/release-notes/opensearch-release-notes-1.3.12.md b/release-notes/opensearch-release-notes-1.3.12.md new file mode 100644 index 0000000000..3ac1f3e2bf --- /dev/null +++ b/release-notes/opensearch-release-notes-1.3.12.md @@ -0,0 +1,41 @@ +# OpenSearch and OpenSearch Dashboards 1.3.12 Release Notes + +## Release Details + +[OpenSearch and OpenSearch Dashboards 1.3.12](https://opensearch.org/versions/opensearch-1-3-12.html) includes the following bug fixes, infrastructure, enhancements and maintenance updates. + +OpenSearch [Release Notes](https://github.com/opensearch-project/OpenSearch/blob/main/release-notes/opensearch.release-notes-1.3.12.md). + +OpenSearch Dashboards [Release Notes](https://github.com/opensearch-project/OpenSearch-Dashboards/blob/main/release-notes/opensearch-dashboards.release-notes-1.3.12.md). + +## BUG FIXES + +### OpenSearch Dashboards Observability +* CVE fixes for tough-cookie, sem-ver, word-wrap ([#776](https://github.com/opensearch-project/dashboards-observability/pull/776)) + +### OpenSearch Dashboards Query Workbench +* Bump dependencies for tough-cookie, semver, word-wrap ([#101](https://github.com/opensearch-project/dashboards-query-workbench/pull/101)) + +### OpenSearch Dashboards Reporting +* Bump dependencies to fix cve ([#155](https://github.com/opensearch-project/dashboards-reporting/pull/155)) + +### Opensearch Dashboards Security +* Switch to new tenant after loading a copied long URL ([#1450](https://github.com/opensearch-project/security-dashboards-plugin/pull/1450)) +* Add the tenant into the short URL once the short URL is resolved ([#1462](https://github.com/opensearch-project/security-dashboards-plugin/pull/1462)) [#1516](https://github.com/opensearch-project/security-dashboards-plugin/pull/1516) + +### OpenSearch Dashboards Visualizations +* Fix CVEs Tough Cookie, Semver, word-wrap ([#219](https://github.com/opensearch-project/dashboards-visualizations/pull/219)) + +### OpenSearch Observability +* Bump guava version for cve ([#1559](https://github.com/opensearch-project/observability/pull/1559)) + +## MAINTENANCE + +### OpenSearch Dashboards Observability +* Remove unused files ([#650](https://github.com/opensearch-project/dashboards-observability/pull/650)) +* Update tests and snapshots ([#612](https://github.com/opensearch-project/dashboards-observability/pull/612)) + +### Opensearch Security +* Bump BouncyCastle from jdk15on to jdk15to18 ([#2901](https://github.com/opensearch-project/security/pull/2901)) ([#2931](https://github.com/opensearch-project/security/pull/2931)) +* Update guava to address CVE-2023-2976 ([#3060](https://github.com/opensearch-project/security/pull/3060)) +* Bump the version of kafka and spring-kafka-test (CVE Related) ([#3087](https://github.com/opensearch-project/security/pull/3087)) diff --git a/scripts/pkg/build_templates/opensearch/deb/debian/postinst b/scripts/pkg/build_templates/opensearch/deb/debian/postinst index cc84020eb7..a192b660b1 100755 --- a/scripts/pkg/build_templates/opensearch/deb/debian/postinst +++ b/scripts/pkg/build_templates/opensearch/deb/debian/postinst @@ -37,11 +37,17 @@ if ! grep -q '## OpenSearch Performance Analyzer' ${config_dir}/jvm.options; the echo "--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED" >> ${config_dir}/jvm.options fi -# Set owner -chown -R opensearch.opensearch ${product_dir} +# Set ownership and permissions +# FIXME: the opensearch service should not have w permission in the config directory chown -R opensearch.opensearch ${config_dir} -chown -R opensearch.opensearch ${log_dir} +chmod -R u=rwX,g=rX,o= ${config_dir} + +chown -R opensearch.adm ${log_dir} +chmod 750 ${log_dir} + chown -R opensearch.opensearch ${data_dir} +chmod 750 ${data_dir} + chown -R opensearch.opensearch ${pid_dir} # Reload systemctl daemon diff --git a/scripts/pkg/build_templates/opensearch/deb/debmake_opensearch_install.sh b/scripts/pkg/build_templates/opensearch/deb/debmake_opensearch_install.sh index 206ca6d6d6..ff87e544d5 100755 --- a/scripts/pkg/build_templates/opensearch/deb/debmake_opensearch_install.sh +++ b/scripts/pkg/build_templates/opensearch/deb/debmake_opensearch_install.sh @@ -41,6 +41,7 @@ ln -s ${data_dir} ${buildroot}${product_dir}/data ln -s ${log_dir} ${buildroot}${product_dir}/logs # Change Permissions -chmod -Rf a+rX,u+w,g-w,o-w ${buildroot}/* +chmod -Rf g-s ${buildroot}/* +chmod -Rf u=rwX,g=rX,o=rX ${buildroot}/* exit 0 diff --git a/scripts/pkg/build_templates/opensearch/rpm/opensearch.rpm.spec b/scripts/pkg/build_templates/opensearch/rpm/opensearch.rpm.spec index f2df1879e0..ea2ddb2108 100644 --- a/scripts/pkg/build_templates/opensearch/rpm/opensearch.rpm.spec +++ b/scripts/pkg/build_templates/opensearch/rpm/opensearch.rpm.spec @@ -68,7 +68,8 @@ if [ ! -f %{buildroot}%{data_dir}/performance_analyzer_enabled.conf ]; then echo 'true' > %{buildroot}%{data_dir}/performance_analyzer_enabled.conf fi # Change Permissions -chmod -Rf a+rX,u+w,g-w,o-w %{buildroot}/* +chmod -Rf g-s %{buildroot}/* +chmod -Rf u=rwX,g=rX,o= %{buildroot}/etc exit 0 %pre @@ -150,13 +151,6 @@ exit 0 # Permissions %defattr(-, %{name}, %{name}) -# Root dirs/docs/licenses -%dir %{product_dir} -%doc %{product_dir}/NOTICE.txt -%doc %{product_dir}/README.md -%license %{product_dir}/LICENSE.txt -%{product_dir}/manifest.yml - # Config dirs/files %dir %{config_dir} %{config_dir}/jvm.options.d @@ -175,6 +169,20 @@ exit 0 %attr(0644, root, root) %config(noreplace) %{_prefix}/lib/sysctl.d/%{name}.conf %attr(0644, root, root) %config(noreplace) %{_prefix}/lib/tmpfiles.d/%{name}.conf +%dir %attr(750, %{name}, %{name}) %{data_dir} +%attr(750, %{name}, %{name}) %{log_dir} +%attr(750, %{name}, %{name}) %{pid_dir} + +# Permissions +%defattr(-, root, root) + +# Root dirs/docs/licenses +%dir %{product_dir} +%doc %{product_dir}/NOTICE.txt +%doc %{product_dir}/README.md +%license %{product_dir}/LICENSE.txt +%{product_dir}/manifest.yml + # Main dirs %{product_dir}/bin %{product_dir}/jdk @@ -182,9 +190,6 @@ exit 0 %{product_dir}/modules %{product_dir}/performance-analyzer-rca %{product_dir}/plugins -%{log_dir} -%{pid_dir} -%dir %{data_dir} # Symlinks %{product_dir}/data diff --git a/src/build_workflow/build_args.py b/src/build_workflow/build_args.py index 253f4b8690..af096edf68 100644 --- a/src/build_workflow/build_args.py +++ b/src/build_workflow/build_args.py @@ -26,6 +26,7 @@ class BuildArgs: platform: str architecture: str distribution: str + continue_on_error: bool def __init__(self) -> None: parser = argparse.ArgumentParser(description="Build an OpenSearch Distribution") @@ -95,6 +96,13 @@ def __init__(self) -> None: default="tar", dest="distribution" ) + parser.add_argument( + "--continue-on-error", + dest="continue_on_error", + default=False, + action="store_true", + help="Do not fail the distribution build on any plugin component failure.", + ) args = parser.parse_args() self.logging_level = args.logging_level @@ -107,6 +115,7 @@ def __init__(self) -> None: self.architecture = args.architecture self.distribution = args.distribution self.script_path = sys.argv[0].replace("/src/run_build.py", "/build.sh") + self.continue_on_error = args.continue_on_error def component_command(self, name: str) -> str: return " ".join( diff --git a/src/manifests/test_run_manifest.py b/src/manifests/test_report_manifest.py similarity index 94% rename from src/manifests/test_run_manifest.py rename to src/manifests/test_report_manifest.py index 863629a2a0..73a5740c5a 100644 --- a/src/manifests/test_run_manifest.py +++ b/src/manifests/test_report_manifest.py @@ -10,9 +10,9 @@ from manifests.component_manifest import Component, ComponentManifest, Components -class TestRunManifest(ComponentManifest['TestRunManifest', 'TestComponents']): +class TestReportManifest(ComponentManifest['TestReportManifest', 'TestComponents']): """ - TestRunManifest contains the aggregated test results for the components. + TestReportManifest contains the aggregated test results for the components. The format for schema version 1.0 is: schema-version: '1.0' @@ -149,7 +149,7 @@ def __to_dict__(self) -> dict: } -TestRunManifest.VERSIONS = {"1.0": TestRunManifest} +TestReportManifest.VERSIONS = {"1.0": TestReportManifest} TestComponent.__test__ = False # type: ignore[attr-defined] -TestRunManifest.__test__ = False # type: ignore[attr-defined] +TestReportManifest.__test__ = False # type: ignore[attr-defined] diff --git a/src/report_workflow/README.md b/src/report_workflow/README.md new file mode 100644 index 0000000000..e348cc1c6b --- /dev/null +++ b/src/report_workflow/README.md @@ -0,0 +1,24 @@ +#### Generate test-report manifest for each test. +As the name specifies, the test report workflow helps to automatically generate a consolidated report of the tests run at distribution level along with commands to reproduce the error and associated failures. + +*Usage* +``` +./report.sh --artifact-paths opensearch=<...> opensearch-dashboards=<...> --test-run-id <...> --test-type integ-test --base-path <...> +``` +e.g. +``` +./report.sh manifests/2.9.0/opensearch-2.9.0-test.yml -p opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8172/linux/x64/tar --test-run-id 5328 --test-type integ-test --base-path https://ci.opensearch.org/ci/dbc/integ-test/2.9.0/8172/linux/x64/tar +``` +The following options are available. + +| name | description | +|---------------------------------|--------------------------------------------------------------------------------| +| test-manifest-path | Specify a test manifest path. | +| -p, --artifact-paths | Artifact paths of distributions used for testing. | +| --base-path | Base paths of testing logs. | +| --test-type | Type of tests report generates on. | +| --output-path | Specify the path location for the test-report manifest. | +| --test-run-id | Specify the unique execution id that matches the id of the test. | +| --component | Specify a specific component or components instead of the entire distribution. | +| --verbose | Show more verbose output. | + diff --git a/src/report_workflow/report_args.py b/src/report_workflow/report_args.py index 9b85d5663d..9a8b807a5b 100644 --- a/src/report_workflow/report_args.py +++ b/src/report_workflow/report_args.py @@ -30,7 +30,7 @@ def __init__(self) -> None: parser.add_argument("--base-path", type=str, default="", help="Specify base paths for the integration test logs.") parser.add_argument("--test-type", type=str, default="integ-test", help="Specify test type of this.") - parser.add_argument("--output-path", type=str, help="Specify the path location for the test-run manifest.") + parser.add_argument("--output-path", type=str, help="Specify the path location for the test-report manifest.") parser.add_argument("--test-run-id", type=int, help="The unique execution id for the test") parser.add_argument("--component", type=str, dest="components", nargs='*', help="Test a specific component or components instead of the entire distribution.") parser.add_argument( diff --git a/src/report_workflow/test_run_runner.py b/src/report_workflow/test_report_runner.py similarity index 88% rename from src/report_workflow/test_run_runner.py rename to src/report_workflow/test_report_runner.py index 2af4f7d189..f53cb413bd 100644 --- a/src/report_workflow/test_run_runner.py +++ b/src/report_workflow/test_report_runner.py @@ -15,22 +15,22 @@ import yaml from manifests.test_manifest import TestManifest -from manifests.test_run_manifest import TestRunManifest +from manifests.test_report_manifest import TestReportManifest from report_workflow.report_args import ReportArgs -class TestRunRunner: +class TestReportRunner: args: ReportArgs test_manifest: TestManifest tests_dir: str - test_run_manifest: TestRunManifest + test_report_manifest: TestReportManifest test_run_data: dict def __init__(self, args: ReportArgs, test_manifest: TestManifest) -> None: self.args = args self.base_path = args.base_path self.test_manifest = test_manifest - self.test_run_data = self.test_run_manifest_data_template("manifest") + self.test_run_data = self.test_report_manifest_data_template("manifest") self.product_name = test_manifest.__to_dict__().get("name") self.name = self.product_name.replace(" ", "-").lower() self.components = self.args.components @@ -63,13 +63,13 @@ def update_test_run_data(self) -> dict: return test_run_data def generate_report(self, data: dict, output_dir: str) -> Any: - test_run_manifest = TestRunManifest(data) - test_run_manifetest_run_manifest_file = os.path.join(output_dir, "test-run.yml") - logging.info(f"Generating test-run.yml in {output_dir}") - return test_run_manifest.to_file(test_run_manifetest_run_manifest_file) + test_report_manifest = TestReportManifest(data) + test_report_manifest_file = os.path.join(output_dir, "test-report.yml") + logging.info(f"Generating test-report.yml in {output_dir}") + return test_report_manifest.to_file(test_report_manifest_file) def component_entry(self, component_name: str) -> Any: - component = self.test_run_manifest_data_template("component") + component = self.test_report_manifest_data_template("component") component["name"] = component_name component["command"] = generate_test_command(self.test_type, self.test_manifest_path, self.artifact_paths, component_name) @@ -102,7 +102,7 @@ def component_entry(self, component_name: str) -> Any: component["configs"].append(config_dict) return component - def test_run_manifest_data_template(self, template_type: str) -> Any: + def test_report_manifest_data_template(self, template_type: str) -> Any: templates = { "manifest": { "schema-version": "1.0", @@ -134,4 +134,4 @@ def generate_test_command(test_type: str, test_manifest_path: str, artifacts_pat return command -TestRunRunner.__test__ = False # type:ignore +TestReportRunner.__test__ = False # type:ignore diff --git a/src/run_build.py b/src/run_build.py index 8ca9af7ec4..cd47c506f4 100755 --- a/src/run_build.py +++ b/src/run_build.py @@ -24,6 +24,7 @@ def main() -> int: args = BuildArgs() console.configure(level=args.logging_level) manifest = InputManifest.from_file(args.manifest) + failed_plugins = [] if args.ref_manifest: manifest = manifest.stable() @@ -70,10 +71,15 @@ def main() -> int: logging.info(f"Successfully built {component.name}") except: logging.error(f"Error building {component.name}, retry with: {args.component_command(component.name)}") - raise + if args.continue_on_error and component.name not in ['OpenSearch', 'job-scheduler', 'common-utils', 'OpenSearch-Dashboards']: + failed_plugins.append(component.name) + continue + else: + raise build_recorder.write_manifest() - + if len(failed_plugins) > 0: + logging.error(f"Failed plugins are {failed_plugins}") logging.info("Done.") return 0 diff --git a/src/run_test_report.py b/src/run_test_report.py index f9bd14daa9..1ae5adc3ba 100644 --- a/src/run_test_report.py +++ b/src/run_test_report.py @@ -10,7 +10,7 @@ from manifests.test_manifest import TestManifest from report_workflow.report_args import ReportArgs -from report_workflow.test_run_runner import TestRunRunner +from report_workflow.test_report_runner import TestReportRunner from system import console @@ -21,11 +21,11 @@ def main() -> Any: test_manifest = TestManifest.from_path(args.test_manifest_path) - test_run_runner = TestRunRunner(args, test_manifest) + test_report_runner = TestReportRunner(args, test_manifest) - test_run_data = test_run_runner.update_data() + test_report_data = test_report_runner.update_data() - test_run = test_run_runner.generate_report(test_run_data, args.output_path or os.getcwd()) + test_run = test_report_runner.generate_report(test_report_data, args.output_path or os.getcwd()) return test_run diff --git a/src/sign_workflow/sign_args.py b/src/sign_workflow/sign_args.py index 22980aa66e..778beed4c8 100644 --- a/src/sign_workflow/sign_args.py +++ b/src/sign_workflow/sign_args.py @@ -13,7 +13,7 @@ class SignArgs: ACCEPTED_SIGNATURE_FILE_TYPES = [".sig", ".asc"] - ACCEPTED_PLATFORM = ["linux", "windows", "mac"] + ACCEPTED_PLATFORM = ["linux", "windows", "mac", "jar_signer"] target: Path components: List[str] diff --git a/src/sign_workflow/signer_jar.py b/src/sign_workflow/signer_jar.py new file mode 100644 index 0000000000..4ca80b8a1a --- /dev/null +++ b/src/sign_workflow/signer_jar.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +import os +from pathlib import Path + +from sign_workflow.signer import Signer + +""" +This class is responsible for signing jar and taco files using the OpenSearch-signer-client and verifying its signature. +""" + + +class SignerJar(Signer): + ACCEPTED_FILE_TYPES = [".jar", ".taco"] + + def generate_signature_and_verify(self, artifact: str, basepath: Path, signature_type: str) -> None: + filename = os.path.join(basepath, artifact) + signed_filename = filename if self.overwrite else os.path.join(basepath, "signed_" + artifact) + self.sign(artifact, basepath, signature_type) + self.verify(signed_filename) + + def is_valid_file_type(self, file_name: str) -> bool: + return any( + file_name.endswith(x) for x in SignerJar.ACCEPTED_FILE_TYPES + ) + + def sign(self, artifact: str, basepath: Path, signature_type: str) -> None: + filename = os.path.join(basepath, artifact) + signed_filename = filename if self.overwrite else os.path.join(basepath, "signed_" + artifact) + signing_cmd = [ + "./opensearch-signer-client", + "-i", + filename, + "-o", + signed_filename, + "-p", + "jar_signer", + "-r", + str(self.overwrite) + ] + self.git_repo.execute(" ".join(signing_cmd)) + + def verify(self, filename: str) -> None: + verify_cmd = ["jarsigner", "-verify", filename, "-verbose", "-certs", "-strict"] + signature = self.git_repo.output(" ".join(verify_cmd)) + if signature.find('jar verified') == -1: + raise ValueError(f"Cannot verify the signature for {filename}") diff --git a/src/sign_workflow/signers.py b/src/sign_workflow/signers.py index 4b566b87a6..88216e6f22 100644 --- a/src/sign_workflow/signers.py +++ b/src/sign_workflow/signers.py @@ -8,6 +8,7 @@ from sign_workflow.signer import Signer +from sign_workflow.signer_jar import SignerJar from sign_workflow.signer_mac import SignerMac from sign_workflow.signer_pgp import SignerPGP from sign_workflow.signer_windows import SignerWindows @@ -18,6 +19,7 @@ class Signers: "windows": SignerWindows, "linux": SignerPGP, "mac": SignerMac, + "jar_signer": SignerJar } @classmethod diff --git a/src/test_workflow/README.md b/src/test_workflow/README.md index feb9bc6f74..5e7294c730 100644 --- a/src/test_workflow/README.md +++ b/src/test_workflow/README.md @@ -7,6 +7,7 @@ - [Identifying Regressions in Nightly Performance Tests](#identifying-regressions-in-nightly-performance-tests) - [Identifying Issues in Longevity Tests](#identifying-issues-in-longevity-tests) - [Benchmark Tests](#benchmarking-tests) + - [Onboard feature for nightly benchmark runs](#onboard-feature-for-nightly-benchmark-runs) - [Testing in CI/CD](#testing-in-cicd) - [Test Workflow (in development)](#test-workflow-in-development) - [Component-Level Details](#component-level-details) @@ -28,7 +29,7 @@ Testing is run via `./test.sh`. The following options are available. | name | description | -| ---------------------- | ----------------------------------------------------------------------- | +|------------------------|-------------------------------------------------------------------------| | test-type | Run tests of a test suite. [integ-test, bwc-test, perf-test] | | test-manifest-path | Specify a test manifest path. | | --paths | Location of manifest(s). | @@ -193,6 +194,29 @@ At a high-level the benchmarking test workflow uses [opensearch-cluster-cdk](htt The benchmarking tests will be run nightly and if you have a feature in any released/un-released OpenSearch version that you want to benchmark periodically please create an issue and the team will reach out to you. In case you want to run the benchmarking test locally you can use `opensearch-cluster-cdk` repo to spin up an OS cluster in your personal AWS account and then use `opensearch-benchmark` to run performance test against it. The detailed instructions are available on respective GitHub repositories. +#### Onboard feature for nightly benchmark runs: + +1. Checkout [opensearch-build](https://github.com/opensearch-project/opensearch-build) repo and open `jenkins/opensearch/benchmark-test.jenkinsfile` file. +2. You will then need add an entry in `parameterizedCron` section of the jenkinsfile. +3. The structure of the `parameterizedCron` section as follows: + 1. Schedule: `H * * *`, edit the `HOUR` section to any hour of the day, 0-24. `H` adds a jitter to the cron to make sure multiple crons are not started together. + 2. BUNDLE_MANIFEST_URL: The distribution manifest URL that contains the artifact details such as tar location, arch, build id, commit-id, etc. + 3. TEST_WORKLOAD: This could be any workload that [opensearch-benchmark-workload](https://github.com/opensearch-project/opensearch-benchmark-workloads) repo provides, if not provided `nyc-taxis` is used as default. + 4. SINGLE_NODE_CLUSTER: Values are `true/false`. Do you want to run the benchmark against a single-node cluster or multi-node. + 5. USE_50_PERCENT_HEAP: Values are `true/false`. Recommended to use 50 percent physical memory as heap. Keep this `true`. + 6. MIN_DISTRIBUTION: Values are `true/false`. If the `BUNDLE_MANIFEST_URL` you provided is for a min/snapshot distribution then set this as `true` else don't provide this parameter. + 7. ADDITIONAL_CONFIG: The configuration that needs to be added to `opensearch.yml` to enable your feature. + 8. USER_TAGS: The metadata that needs to be added to the benchmark metrics ingested in datastore, this helps filter out the metrics for each use-case. Mandatory tags are `run-type:nightly,segrep:,arch:,instance-type:,major-version:<3x|2x>,cluster-config:--` + 9. WORKLOAD_PARAMS: Additional parameters that need to be passed to opensearch-benchmark workload. + 10. To get more information on each parameter and explore more options please visit [here](https://github.com/opensearch-project/opensearch-build/blob/main/jenkins/opensearch/benchmark-test.jenkinsfile#L140-L247) + +Here's the sample entry for enabling nightly runs for `remote-store` feature +``` +H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} +``` + +Once you have added the configuration in the jenkinsfile please raise the PR and opensearch-infra team will review it. + ## Testing in CI/CD The CI/CD infrastructure is divided into two main workflows - `build` and `test`. The `build` workflow automates the process to generate all OpenSearch and OpenSearch Dashboards artifacts, and provide them as distributions to the `test` workflow, which runs exhaustive testing on the artifacts based on the artifact type. The next section talks in detail about the test workflow. diff --git a/src/test_workflow/benchmark_test/benchmark_args.py b/src/test_workflow/benchmark_test/benchmark_args.py index 8c4cfa965f..c0e265b0c2 100644 --- a/src/test_workflow/benchmark_test/benchmark_args.py +++ b/src/test_workflow/benchmark_test/benchmark_args.py @@ -33,6 +33,7 @@ class BenchmarkArgs: ml_node_storage: int jvm_sys_props: str additional_config: str + data_instance_type: str use_50_percent_heap: bool enable_remote_store: bool workload: str @@ -40,7 +41,8 @@ class BenchmarkArgs: benchmark_config: IO user_tag: str target_hosts: str - capture_node_stat: bool + telemetry: list + telemetry_params: str logging_level: int def __init__(self) -> None: @@ -80,6 +82,8 @@ def __init__(self) -> None: help="User provided data-node ebs block storage size, defaults to 100Gb") parser.add_argument("--enable-remote-store", dest="enable_remote_store", action="store_true", help="Enable Remote Store feature in OpenSearch") + parser.add_argument("--data-instance-type", dest="data_instance_type", + help="EC2 instance type for data node, defaults to r5.xlarge.") parser.add_argument("--workload", dest="workload", required=True, help="Name of the workload that OpenSearch Benchmark should run") parser.add_argument("--benchmark-config", dest="benchmark_config", @@ -89,8 +93,13 @@ def __init__(self) -> None: parser.add_argument("--workload-params", dest="workload_params", help="With this parameter you can inject variables into workloads. Parameters differs " "for each workload type. e.g., --workload-params \"number_of_replicas:1,number_of_shards:5\"") - parser.add_argument("--capture-node-stat", dest="capture_node_stat", action="store_true", + parser.add_argument("--capture-node-stat", dest="telemetry", action="append_const", const="node-stats", help="Enable opensearch-benchmark to capture node stat metrics such as cpu, mem, jvm etc as well.") + parser.add_argument("--capture-segment-replication-stat", dest="telemetry", action="append_const", + const="segment-replication-stats", + help="Enable opensearch-benchmark to segment_replication stat metrics such as replication lag.") + parser.add_argument("--telemetry-params", dest="telemetry_params", + help="Allows to set parameters for telemetry devices. Accepts json input.") parser.add_argument("-v", "--verbose", help="Show more verbose output.", action="store_const", default=logging.INFO, const=logging.DEBUG, dest="logging_level") @@ -112,11 +121,13 @@ def __init__(self) -> None: self.data_node_storage = args.data_node_storage if args.data_node_storage else None self.ml_node_storage = args.ml_node_storage if args.ml_node_storage else None self.enable_remote_store = args.enable_remote_store + self.data_instance_type = args.data_instance_type if args.data_instance_type else None self.workload = args.workload self.workload_params = args.workload_params if args.workload_params else None self.benchmark_config = args.benchmark_config if args.benchmark_config else None self.user_tag = args.user_tag if args.user_tag else None self.additional_config = json.dumps(args.additional_config) if args.additional_config is not None else None self.use_50_percent_heap = args.use_50_percent_heap - self.capture_node_stat = args.capture_node_stat + self.telemetry = args.telemetry + self.telemetry_params = args.telemetry_params if args.telemetry_params else None self.logging_level = args.logging_level diff --git a/src/test_workflow/benchmark_test/benchmark_test_cluster.py b/src/test_workflow/benchmark_test/benchmark_test_cluster.py index 39e2190c03..74b3545438 100644 --- a/src/test_workflow/benchmark_test/benchmark_test_cluster.py +++ b/src/test_workflow/benchmark_test/benchmark_test_cluster.py @@ -137,6 +137,7 @@ def setup_cdk_params(self, config: dict) -> dict: "serverAccessType": config["Constants"]["serverAccessType"], "restrictServerAccessTo": config["Constants"]["restrictServerAccessTo"], "additionalConfig": self.args.additional_config, + "dataInstanceType": self.args.data_instance_type, "managerNodeCount": self.args.manager_node_count, "dataNodeCount": self.args.data_node_count, "clientNodeCount": self.args.client_node_count, diff --git a/src/test_workflow/benchmark_test/benchmark_test_suite.py b/src/test_workflow/benchmark_test/benchmark_test_suite.py index 91d82b76a9..4e2788f77c 100644 --- a/src/test_workflow/benchmark_test/benchmark_test_suite.py +++ b/src/test_workflow/benchmark_test/benchmark_test_suite.py @@ -35,21 +35,25 @@ def __init__( self.args = args # Pass the cluster endpoints with -t for multi-cluster use cases(e.g. cross-cluster-replication) self.command = 'docker run --rm' - if args.benchmark_config: + if self.args.benchmark_config: self.command += f" -v {args.benchmark_config}:/opensearch-benchmark/.benchmark/benchmark.ini" self.command += f" opensearchproject/opensearch-benchmark:latest execute-test --workload={self.args.workload} " \ f"--pipeline=benchmark-only --target-hosts={endpoint}" - if args.workload_params: + if self.args.workload_params: logging.info(f"Workload Params are {args.workload_params}") self.command += f" --workload-params '{args.workload_params}'" - if args.user_tag: + if self.args.user_tag: user_tag = f"--user-tag=\"{args.user_tag}\"" self.command += f" {user_tag}" - if args.capture_node_stat: - self.command += " --telemetry node-stats" + if self.args.telemetry: + self.command += " --telemetry " + for value in self.args.telemetry: + self.command += f"{value}," + if self.args.telemetry_params: + self.command += f" --telemetry-params '{self.args.telemetry_params}'" def execute(self) -> None: if self.security: diff --git a/src/test_workflow/bwc_test/bwc_test_runner.py b/src/test_workflow/bwc_test/bwc_test_runner.py index db8f0f0be9..7fb5c9725d 100644 --- a/src/test_workflow/bwc_test/bwc_test_runner.py +++ b/src/test_workflow/bwc_test/bwc_test_runner.py @@ -33,7 +33,7 @@ def __init__(self, args: TestArgs, test_manifest: TestManifest, components: Buil self.tests_dir = os.path.join(os.getcwd(), "test-results") os.makedirs(self.tests_dir, exist_ok=True) - self.test_recorder = TestRecorder(self.args.test_run_id, "bwc-test", self.tests_dir) + self.test_recorder = TestRecorder(self.args.test_run_id, "bwc-test", self.tests_dir, args.base_path) def run(self) -> TestSuiteResults: with TemporaryDirectory(keep=self.args.keep, chdir=True) as work_dir: diff --git a/tests/jenkins/TestDockerBuild.groovy b/tests/jenkins/TestDockerBuild.groovy index 25a0504269..53810b0bbf 100644 --- a/tests/jenkins/TestDockerBuild.groovy +++ b/tests/jenkins/TestDockerBuild.groovy @@ -21,6 +21,7 @@ class TestDockerBuild extends BuildPipelineTest { String dockerBuildGitRespository = 'https://github.com/opensearch-project/opensearch-build' String dockerBuildGitRespositoryReference = 'main' String dockerBuildScriptwithCommands = 'bash docker/ci/build-image-multi-arch.sh -v -f ' + String dockerBuildOS = 'linux' @Override @Before @@ -41,6 +42,7 @@ class TestDockerBuild extends BuildPipelineTest { binding.setVariable('DOCKER_BUILD_GIT_REPOSITORY', dockerBuildGitRespository) binding.setVariable('DOCKER_BUILD_GIT_REPOSITORY_REFERENCE', dockerBuildGitRespositoryReference) binding.setVariable('DOCKER_BUILD_SCRIPT_WITH_COMMANDS', dockerBuildScriptwithCommands) + binding.setVariable('DOCKER_BUILD_OS', dockerBuildOS) } @@ -65,6 +67,9 @@ class TestDockerBuild extends BuildPipelineTest { // Validate the docker-build.sh is called with correct predefined credential assertCallStack().contains("docker-build.sh(echo Account: jenkins-staging-dockerhub-credential)") + // Make sure dockerBuildOS is deciding agent_node docker_nodes docker_args correctly + assertCallStack().contains("docker-build.echo(Executing on agent [docker:[alwaysPull:true, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder, image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.9.1-qemu5.0-awscli1.22-jdk11-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])") + printCallStack() } diff --git a/tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy b/tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy index 41accf4c41..632c9e4fbd 100644 --- a/tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy +++ b/tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy @@ -24,7 +24,7 @@ class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest { helper.registerSharedLibrary( library().name('jenkins') - .defaultVersion('5.4.1') + .defaultVersion('5.6.0') .allowOverride(true) .implicit(true) .targetPath('vars') @@ -38,6 +38,8 @@ class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest { def buildId = 215 def buildManifest = "tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml" def buildManifestUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/3.0.0/${buildId}/linux/x64/tar/builds/opensearch-dashboards/opensearch-dashboards-3.0.0-linux-x64.tar.gz" + def buildManifestOpenSearch = "tests/jenkins/data/opensearch-3.0.0-build.yml" + def buildManifestUrlOpenSearch = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/${buildId}/linux/x64/tar/dist/opensearch/opensearch-3.0.0-linux-x64.tar.gz" def agentLabel = "Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host" def bucketName = 'job-s3-bucket-name' @@ -55,12 +57,16 @@ class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest { binding.setVariable('WEBHOOK_URL', 'htth://WEBHOOK_URL_dummy.com') binding.setVariable('TEST_MANIFEST', testManifest) binding.setVariable('BUILD_MANIFEST_URL', buildManifestUrl) + binding.setVariable('BUILD_MANIFEST_URL_OPENSEARCH', buildManifestUrlOpenSearch) binding.setVariable('BUILD_NUMBER', '215') binding.setVariable('ARTIFACT_BUCKET_NAME', bucketName) binding.setVariable('RUN_DISPLAY_URL', 'https://some/url/redirect') binding.setVariable('AGENT_LABEL', agentLabel) binding.setVariable('BUILD_MANIFEST', buildManifest) + binding.setVariable('BUILD_MANIFEST_OPENSEARCH', buildManifestOpenSearch) binding.setVariable('BUILD_ID', "${buildId}") + binding.setVariable('distribution', 'tar' ) + binding.setVariable('COMPONENT_NAME', '' ) binding.getVariable('currentBuild').upstreamBuilds = [[fullProjectName: jobName]] def env = binding.getVariable('env') env['DOCKER_AGENT'] = [image:'opensearchstaging/opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2', args:'-e JAVA_HOME=/opt/java/openjdk-11'] @@ -103,6 +109,7 @@ class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest { 'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-3.0.0-test.yml --component reportsDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/3.0.0/215/linux/x64/tar '.toString(), 'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-3.0.0-test.yml --component observabilityDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/3.0.0/215/linux/x64/tar '.toString() )) + assertThat(getCommandExecutions('sh', 'report.sh'), hasItems('./report.sh manifests/tests/jenkins/data/opensearch-dashboards-3.0.0-test.yml --artifact-paths opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/215/linux/x64/tar opensearch-dashboards=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/3.0.0/215/linux/x64/tar --test-run-id 215 --test-type integ-test --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/3.0.0/215/linux/x64/tar ')) } @Test diff --git a/tests/jenkins/TestOpenSearchIntegTest.groovy b/tests/jenkins/TestOpenSearchIntegTest.groovy index 1c2d5d1b4b..cb574b94cf 100644 --- a/tests/jenkins/TestOpenSearchIntegTest.groovy +++ b/tests/jenkins/TestOpenSearchIntegTest.groovy @@ -23,7 +23,7 @@ class TestOpenSearchIntegTest extends BuildPipelineTest { helper.registerSharedLibrary( library().name('jenkins') - .defaultVersion('5.4.1') + .defaultVersion('5.6.0') .allowOverride(true) .implicit(true) .targetPath('vars') @@ -34,16 +34,16 @@ class TestOpenSearchIntegTest extends BuildPipelineTest { super.setUp() def jobName = "dummy_job" - def testManifest = "tests/jenkins/data/opensearch-1.3.0-test.yml" - def buildId = 717 - def buildManifest = "tests/jenkins/data/opensearch-1.3.0-build.yml" - def buildManifestUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/${buildId}/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz" + def testManifest = "tests/jenkins/data/opensearch-3.0.0-test.yml" + def buildId = 8184 + def buildManifest = "tests/jenkins/data/opensearch-3.0.0-build.yml" + def buildManifestUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/${buildId}/linux/x64/tar/dist/opensearch/opensearch-3.0.0-linux-x64.tar.gz" def agentLabel = "Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host" def bucketName = 'job-s3-bucket-name' binding.setVariable('ARTIFACT_DOWNLOAD_ROLE_NAME', 'Dummy_Download_Role') binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account') - binding.setVariable('env', ['BUILD_NUMBER': '234', 'PUBLIC_ARTIFACT_URL': 'DUMMY_PUBLIC_ARTIFACT_URL', 'JOB_NAME': 'dummy_job', 'DOCKER_AGENT':[image:'opensearchstaging/ci-runner:ci-runner-centos7-v1', args:'-e JAVA_HOME=/opt/java/openjdk-11'] ]) + binding.setVariable('env', ['BUILD_NUMBER': '234', 'PUBLIC_ARTIFACT_URL': 'DUMMY_PUBLIC_ARTIFACT_URL', 'JOB_NAME': 'dummy_job', 'DOCKER_AGENT':[image:'opensearchstaging/ci-runner:ci-runner-centos7-v1', args:'-e JAVA_HOME=/opt/java/openjdk-11']]) binding.setVariable('ARTIFACT_BUCKET_NAME', 'DUMMY_BUCKET_NAME') binding.setVariable('AWS_ACCOUNT_PUBLIC', 'DUMMY_AWS_ACCOUNT_PUBLIC') binding.setVariable('ARTIFACT_BUCKET_NAME', 'DUMMY_ARTIFACT_BUCKET_NAME') @@ -59,6 +59,8 @@ class TestOpenSearchIntegTest extends BuildPipelineTest { binding.setVariable('BUILD_MANIFEST', buildManifest) binding.setVariable('ARTIFACT_BUCKET_NAME', bucketName) binding.setVariable('RUN_DISPLAY_URL', 'https://some/url/redirect') + binding.setVariable('distribution', 'tar' ) + binding.setVariable('COMPONENT_NAME', '' ) binding.getVariable('currentBuild').upstreamBuilds = [[fullProjectName: jobName]] helper.registerAllowedMethod("s3Download", [Map]) helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure -> @@ -83,55 +85,56 @@ class TestOpenSearchIntegTest extends BuildPipelineTest { void integTests_runs_consistently() { super.testPipeline('jenkins/opensearch/integ-test.jenkinsfile', 'tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile') - assertThat(getCommandExecutions('sh', 'test.sh'), hasItem('env PATH=$PATH JAVA_HOME=/opt/java/openjdk-17 ./test.sh integ-test manifests/tests/jenkins/data/opensearch-1.3.0-test.yml --component OpenSearch --test-run-id 234 --paths opensearch=/tmp/workspace/tar --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/1.3.0/717/linux/x64/tar ')) + assertThat(getCommandExecutions('sh', 'test.sh'), hasItem('env PATH=$PATH JAVA_HOME=/opt/java/openjdk-17 ./test.sh integ-test manifests/tests/jenkins/data/opensearch-3.0.0-test.yml --component OpenSearch --test-run-id 234 --paths opensearch=/tmp/workspace/tar --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/3.0.0/8184/linux/x64/tar ')) + assertThat(getCommandExecutions('sh', 'report.sh'), hasItem('./report.sh manifests/tests/jenkins/data/opensearch-3.0.0-test.yml --artifact-paths opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/8184/linux/x64/tar --test-run-id 234 --test-type integ-test --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/3.0.0/8184/linux/x64/tar ')) } @Test void checkUploadResults() { runScript('jenkins/opensearch/integ-test.jenkinsfile') - assertThat(getCommandExecutions('s3Upload', ''), hasItem('{file=test-results, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/1.3.0/717/linux/x64/tar/test-results}')) + assertThat(getCommandExecutions('s3Upload', ''), hasItem('{file=test-results, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/3.0.0/8184/linux/x64/tar/test-results}')) } @Test void checkError() { - helper.addFileExistsMock('manifests/tests/jenkins/data/opensearch-1.3.0-test.yml', false) + helper.addFileExistsMock('manifests/tests/jenkins/data/opensearch-3.0.0-test.yml', false) runScript('jenkins/opensearch/integ-test.jenkinsfile') - assertThat(getCommandExecutions('error', ''), hasItem('Integration Tests failed to start. Test manifest was not provided or not found in manifests/tests/jenkins/data/opensearch-1.3.0-test.yml.')) + assertThat(getCommandExecutions('error', ''), hasItem('Integration Tests failed to start. Test manifest was not provided or not found in manifests/tests/jenkins/data/opensearch-3.0.0-test.yml.')) assertJobStatusFailure() } @Test void checkGHissueCreation() { super.setUp() - helper.addShMock('env PATH=$PATH JAVA_HOME=/opt/java/openjdk-17 ./test.sh integ-test manifests/tests/jenkins/data/opensearch-1.3.0-test.yml --component OpenSearch --test-run-id 234 --paths opensearch=/tmp/workspace/tar --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/1.3.0/717/linux/x64/tar', '', 1) - helper.addShMock('gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S "[AUTOCUT] Integration Test failed for OpenSearch: 1.3.0 tar distribution in:title" --label autocut,v1.3.0,integ-test-failure', '', 0) + helper.addShMock('env PATH=$PATH JAVA_HOME=/opt/java/openjdk-17 ./test.sh integ-test manifests/tests/jenkins/data/opensearch-3.0.0-test.yml --component OpenSearch --test-run-id 234 --paths opensearch=/tmp/workspace/tar --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/3.0.0/8184/linux/x64/tar', '', 1) + helper.addShMock('gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S "[AUTOCUT] Integration Test failed for OpenSearch: 3.0.0 tar distribution in:title" --label autocut,v3.0.0,integ-test-failure', '', 0) assertThrows(Exception) { runScript('jenkins/opensearch/integ-test.jenkinsfile') } assertJobStatusFailure() - assertThat(getCommandExecutions('sh', 'create'), hasItem('{script=gh issue create --title \"[AUTOCUT] Integration Test failed for OpenSearch: 1.3.0 tar distribution\" --body \"The integration test failed at distribution level for component OpenSearch
Version: 1.3.0
Distribution: tar
Architecture: x64
Platform: linux

Please check the logs: https://some/url/redirect

* Steps to reproduce: See https://github.com/opensearch-project/opensearch-build/tree/main/src/test_workflow#integration-tests
* Access components yml file:
- [With security](https://ci.opensearch.org/ci/dbc/dummy_job/1.3.0/717/linux/x64/tar/test-results/234/integ-test/OpenSearch/with-security/OpenSearch.yml) (if applicable)
- [Without security](https://ci.opensearch.org/ci/dbc/dummy_job/1.3.0/717/linux/x64/tar/test-results/234/integ-test/OpenSearch/without-security/OpenSearch.yml) (if applicable)

_Note: All in one test report manifest with all the details coming soon. See https://github.com/opensearch-project/opensearch-build/issues/1274_\" --label autocut,v1.3.0,integ-test-failure --label \"untriaged\" --repo https://github.com/opensearch-project/OpenSearch.git, returnStdout=true}')) + assertThat(getCommandExecutions('sh', 'create'), hasItem('{script=gh issue create --title \"[AUTOCUT] Integration Test failed for OpenSearch: 3.0.0 tar distribution\" --body \"The integration test failed at distribution level for component OpenSearch
Version: 3.0.0
Distribution: tar
Architecture: x64
Platform: linux

Please check the logs: https://some/url/redirect

* Steps to reproduce: See https://github.com/opensearch-project/opensearch-build/tree/main/src/test_workflow#integration-tests
* Access components yml file:
- [With security](https://ci.opensearch.org/ci/dbc/dummy_job/3.0.0/8184/linux/x64/tar/test-results/234/integ-test/OpenSearch/with-security/OpenSearch.yml) (if applicable)
- [Without security](https://ci.opensearch.org/ci/dbc/dummy_job/3.0.0/8184/linux/x64/tar/test-results/234/integ-test/OpenSearch/without-security/OpenSearch.yml) (if applicable)

_Note: All in one test report manifest with all the details coming soon. See https://github.com/opensearch-project/opensearch-build/issues/1274_\" --label autocut,v3.0.0,integ-test-failure --label \"untriaged\" --repo https://github.com/opensearch-project/OpenSearch.git, returnStdout=true}')) } @Test void CheckCloseGHissue() { - helper.addShMock('env PATH=$PATH JAVA_HOME=/opt/java/openjdk-17 ./test.sh integ-test manifests/tests/jenkins/data/opensearch-1.3.0-test.yml --component OpenSearch --test-run-id 234 --paths opensearch=/tmp/workspace/tar --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/1.3.0/717/linux/x64/tar', '', 1) + helper.addShMock('env PATH=$PATH JAVA_HOME=/opt/java/openjdk-17 ./test.sh integ-test manifests/tests/jenkins/data/opensearch-3.0.0-test.yml --component OpenSearch --test-run-id 234 --paths opensearch=/tmp/workspace/tar --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/3.0.0/8184/linux/x64/tar', '', 1) runScript('jenkins/opensearch/integ-test.jenkinsfile') - assertThat(getCommandExecutions('sh', 'script'), hasItem("{script=gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S \"[AUTOCUT] Integration Test failed for OpenSearch: 1.3.0 tar distribution in:title\" --label autocut,v1.3.0,integ-test-failure --json number --jq '.[0].number', returnStdout=true}")) - assertThat(getCommandExecutions('sh', 'script'), hasItem("{script=gh issue close bbb\nccc -R opensearch-project/OpenSearch --comment \"Closing the issue as the Integration Test passed for OpenSearch
Version: 1.3.0
Distribution: tar
Architecture: x64
Platform: linux

Please check the logs: https://some/url/redirect

*\", returnStdout=true}")) + assertThat(getCommandExecutions('sh', 'script'), hasItem("{script=gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S \"[AUTOCUT] Integration Test failed for OpenSearch: 3.0.0 tar distribution in:title\" --label autocut,v3.0.0,integ-test-failure --json number --jq '.[0].number', returnStdout=true}")) + assertThat(getCommandExecutions('sh', 'script'), hasItem("{script=gh issue close bbb\nccc -R opensearch-project/OpenSearch --comment \"Closing the issue as the Integration Test passed for OpenSearch
Version: 3.0.0
Distribution: tar
Architecture: x64
Platform: linux

Please check the logs: https://some/url/redirect

*\", returnStdout=true}")) } @Test void checkGHexistingIssue() { super.setUp() - helper.addShMock('env PATH=$PATH JAVA_HOME=/opt/java/openjdk-17 ./test.sh integ-test manifests/tests/jenkins/data/opensearch-1.3.0-test.yml --component OpenSearch --test-run-id 234 --paths opensearch=/tmp/workspace/tar --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/1.3.0/717/linux/x64/tar ', '', 1) + helper.addShMock('env PATH=$PATH JAVA_HOME=/opt/java/openjdk-17 ./test.sh integ-test manifests/tests/jenkins/data/opensearch-3.0.0-test.yml --component OpenSearch --test-run-id 234 --paths opensearch=/tmp/workspace/tar --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/3.0.0/8184/linux/x64/tar ', '', 1) assertThrows(Exception) { runScript('jenkins/opensearch/integ-test.jenkinsfile') } assertJobStatusFailure() assertThat(getCommandExecutions('println', 'Issue'), hasItem('Issue already exists, adding a comment.')) - assertThat(getCommandExecutions('sh', 'script'), hasItem("{script=gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S \"[AUTOCUT] Integration Test failed for OpenSearch: 1.3.0 tar distribution in:title\" --label autocut,v1.3.0,integ-test-failure, returnStdout=true}")) - assertThat(getCommandExecutions('sh', 'script'), hasItem("{script=gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S \"[AUTOCUT] Integration Test failed for OpenSearch: 1.3.0 tar distribution in:title\" --label autocut,v1.3.0,integ-test-failure --json number --jq '.[0].number', returnStdout=true}")) - assertThat(getCommandExecutions('sh', 'script'), hasItem("{script=gh issue comment bbb\nccc --repo https://github.com/opensearch-project/OpenSearch.git --body \"The integration test failed at distribution level for component OpenSearch
Version: 1.3.0
Distribution: tar
Architecture: x64
Platform: linux

Please check the logs: https://some/url/redirect

* Steps to reproduce: See https://github.com/opensearch-project/opensearch-build/tree/main/src/test_workflow#integration-tests
* Access components yml file:
- [With security](https://ci.opensearch.org/ci/dbc/dummy_job/1.3.0/717/linux/x64/tar/test-results/234/integ-test/OpenSearch/with-security/OpenSearch.yml) (if applicable)
- [Without security](https://ci.opensearch.org/ci/dbc/dummy_job/1.3.0/717/linux/x64/tar/test-results/234/integ-test/OpenSearch/without-security/OpenSearch.yml) (if applicable)

_Note: All in one test report manifest with all the details coming soon. See https://github.com/opensearch-project/opensearch-build/issues/1274_\", returnStdout=true}")) + assertThat(getCommandExecutions('sh', 'script'), hasItem("{script=gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S \"[AUTOCUT] Integration Test failed for OpenSearch: 3.0.0 tar distribution in:title\" --label autocut,v3.0.0,integ-test-failure, returnStdout=true}")) + assertThat(getCommandExecutions('sh', 'script'), hasItem("{script=gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S \"[AUTOCUT] Integration Test failed for OpenSearch: 3.0.0 tar distribution in:title\" --label autocut,v3.0.0,integ-test-failure --json number --jq '.[0].number', returnStdout=true}")) + assertThat(getCommandExecutions('sh', 'script'), hasItem("{script=gh issue comment bbb\nccc --repo https://github.com/opensearch-project/OpenSearch.git --body \"The integration test failed at distribution level for component OpenSearch
Version: 3.0.0
Distribution: tar
Architecture: x64
Platform: linux

Please check the logs: https://some/url/redirect

* Steps to reproduce: See https://github.com/opensearch-project/opensearch-build/tree/main/src/test_workflow#integration-tests
* Access components yml file:
- [With security](https://ci.opensearch.org/ci/dbc/dummy_job/3.0.0/8184/linux/x64/tar/test-results/234/integ-test/OpenSearch/with-security/OpenSearch.yml) (if applicable)
- [Without security](https://ci.opensearch.org/ci/dbc/dummy_job/3.0.0/8184/linux/x64/tar/test-results/234/integ-test/OpenSearch/without-security/OpenSearch.yml) (if applicable)

_Note: All in one test report manifest with all the details coming soon. See https://github.com/opensearch-project/opensearch-build/issues/1274_\", returnStdout=true}")) } def getCommandExecutions(methodName, command) { diff --git a/tests/jenkins/TestRunNonSecBenchmarkTestScript.groovy b/tests/jenkins/TestRunNonSecBenchmarkTestScript.groovy index 25ca5f467f..47818f61d7 100644 --- a/tests/jenkins/TestRunNonSecBenchmarkTestScript.groovy +++ b/tests/jenkins/TestRunNonSecBenchmarkTestScript.groovy @@ -26,7 +26,7 @@ class TestRunNonSecBenchmarkTestScript extends BuildPipelineTest{ void setUp() { helper.registerSharedLibrary( library().name('jenkins') - .defaultVersion('4.3.0') + .defaultVersion('5.5.0') .allowOverride(true) .implicit(true) .targetPath('vars') @@ -80,8 +80,10 @@ class TestRunNonSecBenchmarkTestScript extends BuildPipelineTest{ binding.setVariable('ML_NODE_COUNT', '') binding.setVariable('DATA_NODE_STORAGE', '100') binding.setVariable('ML_NODE_STORAGE', '') + binding.setVariable('DATA_INSTANCE_TYPE', 'r5-4xlarge') binding.setVariable('JVM_SYS_PROPS', '') binding.setVariable('CAPTURE_NODE_STAT', 'false') + binding.setVariable('CAPTURE_SEGMENT_REPLICATION_STAT', 'false') binding.setVariable('JOB_NAME', 'benchmark-test') binding.setVariable('BENCHMARK_TEST_CONFIG_LOCATION', 'test_config') binding.setVariable('PUBLIC_ARTIFACT_URL', 'test://artifact.url') @@ -132,7 +134,7 @@ class TestRunNonSecBenchmarkTestScript extends BuildPipelineTest{ assertThat(testScriptCommands.size(), equalTo(1)) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-node-storage 100 ".toString() + "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --data-node-storage 100 ".toString() )) } diff --git a/tests/jenkins/TestRunSecureBenchmarkTestScript.groovy b/tests/jenkins/TestRunSecureBenchmarkTestScript.groovy index 23eedd3b34..412102894b 100644 --- a/tests/jenkins/TestRunSecureBenchmarkTestScript.groovy +++ b/tests/jenkins/TestRunSecureBenchmarkTestScript.groovy @@ -26,7 +26,7 @@ class TestRunSecureBenchmarkTestScript extends BuildPipelineTest{ void setUp() { helper.registerSharedLibrary( library().name('jenkins') - .defaultVersion('4.3.0') + .defaultVersion('5.5.0') .allowOverride(true) .implicit(true) .targetPath('vars') @@ -80,8 +80,10 @@ class TestRunSecureBenchmarkTestScript extends BuildPipelineTest{ binding.setVariable('ML_NODE_COUNT', '') binding.setVariable('DATA_NODE_STORAGE', '100') binding.setVariable('ML_NODE_STORAGE', '') + binding.setVariable('DATA_INSTANCE_TYPE', '') binding.setVariable('JVM_SYS_PROPS', '') binding.setVariable('CAPTURE_NODE_STAT', 'false') + binding.setVariable('CAPTURE_SEGMENT_REPLICATION_STAT', 'true') binding.setVariable('JOB_NAME', 'benchmark-test') binding.setVariable('BENCHMARK_TEST_CONFIG_LOCATION', 'test_config') binding.setVariable('PUBLIC_ARTIFACT_URL', 'test://artifact.url') @@ -133,8 +135,8 @@ class TestRunSecureBenchmarkTestScript extends BuildPipelineTest{ assertThat(testScriptCommands.size(), equalTo(2)) assertThat(testScriptCommands, hasItems( - "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:true --use-50-percent-heap --enable-remote-store --suffix 307-secure --manager-node-count 3 --data-node-count 3 --data-node-storage 100 ".toString(), - "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false --without-security --use-50-percent-heap --enable-remote-store --suffix 307 --manager-node-count 3 --data-node-count 3 --data-node-storage 100 ".toString() + "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:true --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --data-node-storage 100 ".toString(), + "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false --without-security --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --data-node-storage 100 ".toString() )) } diff --git a/tests/jenkins/data/opensearch-3.0.0-build.yml b/tests/jenkins/data/opensearch-3.0.0-build.yml new file mode 100644 index 0000000000..2fbccc84b0 --- /dev/null +++ b/tests/jenkins/data/opensearch-3.0.0-build.yml @@ -0,0 +1,23 @@ +--- +name: OpenSearch +build: + name: OpenSearch + version: 3.0.0 + platform: linux + architecture: x64 + distribution: tar + id: '8184' +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: 1164221ee2b8ba3560f0ff492309867beea28433 + commit_id: 1164221ee2b8ba3560f0ff492309867beea28433 + artifacts: + maven: + - maven/org/opensearch/opensearch-dissect/maven-metadata.xml + - maven/org/opensearch/opensearch-dissect/maven-metadata.xml.sha1 + - maven/org/opensearch/opensearch-dissect/maven-metadata.xml.md5 + - maven/org/opensearch/opensearch-dissect/maven-metadata.xml.sha256 + - maven/org/opensearch/opensearch-dissect/maven-metadata.xml.sha512 + - maven/org/opensearch/opensearch-dissect/2.9.0/opensearch-dissect-2.9.0.jar + - maven/org/opensearch/opensearch-dissect/2.9.0/opensearch-dissect-2.9.0.jar.sha1 diff --git a/tests/jenkins/data/opensearch-3.0.0-test.yml b/tests/jenkins/data/opensearch-3.0.0-test.yml new file mode 100644 index 0000000000..42b50bb82b --- /dev/null +++ b/tests/jenkins/data/opensearch-3.0.0-test.yml @@ -0,0 +1,132 @@ +--- +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 + args: -e JAVA_HOME=/opt/java/openjdk-17 +components: + - name: alerting + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + plugins.destination.host.deny_list: [10.0.0.0/8, 127.0.0.1] + bwc-test: + test-configs: + - with-security + + - name: anomaly-detection + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + + - name: asynchronous-search + integ-test: + test-configs: + - with-security + - without-security + + - name: cross-cluster-replication + integ-test: + topology: + - cluster_name: leader + data_nodes: 1 + cluster_manager_nodes: 0 + - cluster_name: follower + data_nodes: 1 + cluster_manager_nodes: 0 + test-configs: + - with-security + - without-security + + - name: geospatial + integ-test: + test-configs: + - with-security + - without-security + + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + bwc-test: + test-configs: + - with-security + + - name: k-NN + integ-test: + test-configs: + - with-security + - without-security + + - name: ml-commons + integ-test: + test-configs: + - with-security + - without-security + + - name: neural-search + integ-test: + test-configs: + - with-security + - without-security + + - name: notifications + working-directory: notifications + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + + - name: opensearch-observability + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + + - name: opensearch-reports + integ-test: + test-configs: + - with-security + - without-security + + - name: security + integ-test: + test-configs: + - with-security + + - name: security-analytics + integ-test: + test-configs: + - with-security + - without-security + + - name: sql + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + script.context.field.max_compilations_rate: 1000/1m + plugins.query.datasources.encryption.masterkey: 4fc8fee6a3fd7d6ca01772e5 + bwc-test: + test-configs: + - with-security diff --git a/tests/jenkins/jenkinsjob-regression-files/cross-cluster-replication/perf-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/cross-cluster-replication/perf-test.jenkinsfile.txt index c7b4d360d4..a5f39a6c59 100644 --- a/tests/jenkins/jenkinsjob-regression-files/cross-cluster-replication/perf-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/cross-cluster-replication/perf-test.jenkinsfile.txt @@ -4,11 +4,6 @@ perf-test.pipeline(groovy.lang.Closure) perf-test.timeout({time=10, unit=HOURS}) perf-test.echo(Executing on agent [label:none]) - perf-test.parameterizedCron( - H 0 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.5.0/latest/linux/x64/tar/dist/opensearch/manifest.yml; - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.5.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml; - - ) perf-test.stage(validate-and-set-parameters, groovy.lang.Closure) perf-test.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]]) perf-test.script(groovy.lang.Closure) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/bwc-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/bwc-test.jenkinsfile.txt index 6ab5465885..74d889ffde 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/bwc-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/bwc-test.jenkinsfile.txt @@ -3,6 +3,8 @@ bwc-test.library({identifier=jenkins@1.0.4, retriever=null}) bwc-test.pipeline(groovy.lang.Closure) bwc-test.timeout({time=4, unit=HOURS}) + bwc-test.logRotator({daysToKeepStr=30}) + bwc-test.buildDiscarder(null) bwc-test.echo(Executing on agent [label:none]) bwc-test.stage(verify-parameters, groovy.lang.Closure) bwc-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/integ-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/integ-test.jenkinsfile.txt index 2c8891e97b..cd833b7002 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/integ-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/integ-test.jenkinsfile.txt @@ -1,9 +1,11 @@ integ-test.run() integ-test.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) - integ-test.library({identifier=jenkins@5.4.1, retriever=null}) + integ-test.library({identifier=jenkins@5.6.0, retriever=null}) integ-test.pipeline(groovy.lang.Closure) integ-test.credentials(jenkins-artifact-bucket-name) integ-test.timeout({time=4, unit=HOURS}) + integ-test.logRotator({daysToKeepStr=30}) + integ-test.buildDiscarder(null) integ-test.echo(Executing on agent [label:none]) integ-test.stage(verify-parameters, groovy.lang.Closure) integ-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) @@ -11,19 +13,19 @@ integ-test.fileExists(manifests/tests/jenkins/data/opensearch-dashboards-3.0.0-test.yml) integ-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/3.0.0/215/linux/x64/tar/builds/opensearch-dashboards/opensearch-dashboards-3.0.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@5.4.1, retriever=null}) + downloadBuildManifest.library({identifier=jenkins@5.5.0, retriever=null}) downloadBuildManifest.sh(curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/3.0.0/215/linux/x64/tar/builds/opensearch-dashboards/opensearch-dashboards-3.0.0-linux-x64.tar.gz --output tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) BuildManifest.asBoolean() - integ-test.downloadBuildManifest({url=null, path=build-manifest-opensearch.yml}) + integ-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/215/linux/x64/tar/dist/opensearch/opensearch-3.0.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-3.0.0-build.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@5.4.1, retriever=null}) - downloadBuildManifest.sh(curl -sSL null --output build-manifest-opensearch.yml) - downloadBuildManifest.readYaml({file=build-manifest-opensearch.yml}) + downloadBuildManifest.library({identifier=jenkins@5.5.0, retriever=null}) + downloadBuildManifest.sh(curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/215/linux/x64/tar/dist/opensearch/opensearch-3.0.0-linux-x64.tar.gz --output tests/jenkins/data/opensearch-3.0.0-build.yml) + downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-3.0.0-build.yml}) BuildManifest.asBoolean() integ-test.readYaml({file=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) BuildManifest.asBoolean() - integ-test.readYaml({file=build-manifest-opensearch.yml}) + integ-test.readYaml({file=tests/jenkins/data/opensearch-3.0.0-build.yml}) BuildManifest.asBoolean() BuildManifest.getArtifactArchitecture() BuildManifest.getArtifactBuildId() @@ -39,7 +41,7 @@ integ-test.script(groovy.lang.Closure) integ-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/3.0.0/215/linux/x64/tar/builds/opensearch-dashboards/opensearch-dashboards-3.0.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@5.4.1, retriever=null}) + downloadBuildManifest.library({identifier=jenkins@5.5.0, retriever=null}) downloadBuildManifest.sh(curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/3.0.0/215/linux/x64/tar/builds/opensearch-dashboards/opensearch-dashboards-3.0.0-linux-x64.tar.gz --output tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) BuildManifest.asBoolean() @@ -78,7 +80,7 @@ integ-test.sh(rm -rf test-results) integ-test.runIntegTestScript({jobName=distribution-build-opensearch-dashboards, componentName=ganttChartDashboards, buildManifest=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml, testManifest=manifests/tests/jenkins/data/opensearch-dashboards-3.0.0-test.yml, localPath=/tmp/workspace/tar, switchUserNonRoot=false}) runIntegTestScript.legacySCM(groovy.lang.Closure) - runIntegTestScript.library({identifier=jenkins@5.4.1, retriever=null}) + runIntegTestScript.library({identifier=jenkins@5.5.0, retriever=null}) runIntegTestScript.readYaml({file=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) BuildManifest.asBoolean() BuildManifest.getDistribution() @@ -104,7 +106,7 @@ ccc -R opensearch-project/dashboards-visualizations --comment "Closing the issue integ-test.echo(Completed running integtest for component ganttChartDashboards) integ-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml, jobName=dummy_job}) uploadTestResults.legacySCM(groovy.lang.Closure) - uploadTestResults.library({identifier=jenkins@5.4.1, retriever=null}) + uploadTestResults.library({identifier=jenkins@5.5.0, retriever=null}) uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) BuildManifest.asBoolean() uploadTestResults.echo(Build Id: 215) @@ -129,7 +131,7 @@ ccc -R opensearch-project/dashboards-visualizations --comment "Closing the issue integ-test.sh(rm -rf test-results) integ-test.runIntegTestScript({jobName=distribution-build-opensearch-dashboards, componentName=indexManagementDashboards, buildManifest=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml, testManifest=manifests/tests/jenkins/data/opensearch-dashboards-3.0.0-test.yml, localPath=/tmp/workspace/tar, switchUserNonRoot=false}) runIntegTestScript.legacySCM(groovy.lang.Closure) - runIntegTestScript.library({identifier=jenkins@5.4.1, retriever=null}) + runIntegTestScript.library({identifier=jenkins@5.5.0, retriever=null}) runIntegTestScript.readYaml({file=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) BuildManifest.asBoolean() BuildManifest.getDistribution() @@ -155,7 +157,7 @@ ccc -R opensearch-project/index-management-dashboards-plugin --comment "Closing integ-test.echo(Completed running integtest for component indexManagementDashboards) integ-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml, jobName=dummy_job}) uploadTestResults.legacySCM(groovy.lang.Closure) - uploadTestResults.library({identifier=jenkins@5.4.1, retriever=null}) + uploadTestResults.library({identifier=jenkins@5.5.0, retriever=null}) uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) BuildManifest.asBoolean() uploadTestResults.echo(Build Id: 215) @@ -180,7 +182,7 @@ ccc -R opensearch-project/index-management-dashboards-plugin --comment "Closing integ-test.sh(rm -rf test-results) integ-test.runIntegTestScript({jobName=distribution-build-opensearch-dashboards, componentName=anomalyDetectionDashboards, buildManifest=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml, testManifest=manifests/tests/jenkins/data/opensearch-dashboards-3.0.0-test.yml, localPath=/tmp/workspace/tar, switchUserNonRoot=false}) runIntegTestScript.legacySCM(groovy.lang.Closure) - runIntegTestScript.library({identifier=jenkins@5.4.1, retriever=null}) + runIntegTestScript.library({identifier=jenkins@5.5.0, retriever=null}) runIntegTestScript.readYaml({file=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) BuildManifest.asBoolean() BuildManifest.getDistribution() @@ -206,7 +208,7 @@ ccc -R opensearch-project/anomaly-detection-dashboards-plugin --comment "Closing integ-test.echo(Completed running integtest for component anomalyDetectionDashboards) integ-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml, jobName=dummy_job}) uploadTestResults.legacySCM(groovy.lang.Closure) - uploadTestResults.library({identifier=jenkins@5.4.1, retriever=null}) + uploadTestResults.library({identifier=jenkins@5.5.0, retriever=null}) uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) BuildManifest.asBoolean() uploadTestResults.echo(Build Id: 215) @@ -231,7 +233,7 @@ ccc -R opensearch-project/anomaly-detection-dashboards-plugin --comment "Closing integ-test.sh(rm -rf test-results) integ-test.runIntegTestScript({jobName=distribution-build-opensearch-dashboards, componentName=OpenSearch-Dashboards, buildManifest=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml, testManifest=manifests/tests/jenkins/data/opensearch-dashboards-3.0.0-test.yml, localPath=/tmp/workspace/tar, switchUserNonRoot=false}) runIntegTestScript.legacySCM(groovy.lang.Closure) - runIntegTestScript.library({identifier=jenkins@5.4.1, retriever=null}) + runIntegTestScript.library({identifier=jenkins@5.5.0, retriever=null}) runIntegTestScript.readYaml({file=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) BuildManifest.asBoolean() BuildManifest.getDistribution() @@ -257,7 +259,7 @@ ccc -R opensearch-project/OpenSearch-Dashboards --comment "Closing the issue as integ-test.echo(Completed running integtest for component OpenSearch-Dashboards) integ-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml, jobName=dummy_job}) uploadTestResults.legacySCM(groovy.lang.Closure) - uploadTestResults.library({identifier=jenkins@5.4.1, retriever=null}) + uploadTestResults.library({identifier=jenkins@5.5.0, retriever=null}) uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) BuildManifest.asBoolean() uploadTestResults.echo(Build Id: 215) @@ -282,7 +284,7 @@ ccc -R opensearch-project/OpenSearch-Dashboards --comment "Closing the issue as integ-test.sh(rm -rf test-results) integ-test.runIntegTestScript({jobName=distribution-build-opensearch-dashboards, componentName=reportsDashboards, buildManifest=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml, testManifest=manifests/tests/jenkins/data/opensearch-dashboards-3.0.0-test.yml, localPath=/tmp/workspace/tar, switchUserNonRoot=false}) runIntegTestScript.legacySCM(groovy.lang.Closure) - runIntegTestScript.library({identifier=jenkins@5.4.1, retriever=null}) + runIntegTestScript.library({identifier=jenkins@5.5.0, retriever=null}) runIntegTestScript.readYaml({file=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) BuildManifest.asBoolean() BuildManifest.getDistribution() @@ -308,7 +310,7 @@ ccc -R opensearch-project/dashboards-reports --comment "Closing the issue as the integ-test.echo(Completed running integtest for component reportsDashboards) integ-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml, jobName=dummy_job}) uploadTestResults.legacySCM(groovy.lang.Closure) - uploadTestResults.library({identifier=jenkins@5.4.1, retriever=null}) + uploadTestResults.library({identifier=jenkins@5.5.0, retriever=null}) uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) BuildManifest.asBoolean() uploadTestResults.echo(Build Id: 215) @@ -333,7 +335,7 @@ ccc -R opensearch-project/dashboards-reports --comment "Closing the issue as the integ-test.sh(rm -rf test-results) integ-test.runIntegTestScript({jobName=distribution-build-opensearch-dashboards, componentName=queryWorkbenchDashboards, buildManifest=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml, testManifest=manifests/tests/jenkins/data/opensearch-dashboards-3.0.0-test.yml, localPath=/tmp/workspace/tar, switchUserNonRoot=false}) runIntegTestScript.legacySCM(groovy.lang.Closure) - runIntegTestScript.library({identifier=jenkins@5.4.1, retriever=null}) + runIntegTestScript.library({identifier=jenkins@5.5.0, retriever=null}) runIntegTestScript.readYaml({file=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) BuildManifest.asBoolean() BuildManifest.getDistribution() @@ -359,7 +361,7 @@ ccc -R opensearch-project/sql --comment "Closing the issue as the Integration Te integ-test.echo(Completed running integtest for component queryWorkbenchDashboards) integ-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml, jobName=dummy_job}) uploadTestResults.legacySCM(groovy.lang.Closure) - uploadTestResults.library({identifier=jenkins@5.4.1, retriever=null}) + uploadTestResults.library({identifier=jenkins@5.5.0, retriever=null}) uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) BuildManifest.asBoolean() uploadTestResults.echo(Build Id: 215) @@ -384,7 +386,7 @@ ccc -R opensearch-project/sql --comment "Closing the issue as the Integration Te integ-test.sh(rm -rf test-results) integ-test.runIntegTestScript({jobName=distribution-build-opensearch-dashboards, componentName=observabilityDashboards, buildManifest=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml, testManifest=manifests/tests/jenkins/data/opensearch-dashboards-3.0.0-test.yml, localPath=/tmp/workspace/tar, switchUserNonRoot=false}) runIntegTestScript.legacySCM(groovy.lang.Closure) - runIntegTestScript.library({identifier=jenkins@5.4.1, retriever=null}) + runIntegTestScript.library({identifier=jenkins@5.5.0, retriever=null}) runIntegTestScript.readYaml({file=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) BuildManifest.asBoolean() BuildManifest.getDistribution() @@ -410,7 +412,7 @@ ccc -R opensearch-project/dashboards-observability --comment "Closing the issue integ-test.echo(Completed running integtest for component observabilityDashboards) integ-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml, jobName=dummy_job}) uploadTestResults.legacySCM(groovy.lang.Closure) - uploadTestResults.library({identifier=jenkins@5.4.1, retriever=null}) + uploadTestResults.library({identifier=jenkins@5.5.0, retriever=null}) uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) BuildManifest.asBoolean() uploadTestResults.echo(Build Id: 215) @@ -428,6 +430,42 @@ ccc -R opensearch-project/dashboards-observability --comment "Closing the issue uploadTestResults.stash({includes=messages/*, name=messages-DUMMY_STAGE_NAME}) integ-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + integ-test.script(groovy.lang.Closure) + integ-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) + integ-test.unstash(integtest-opensearch-dashboards-215) + integ-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/215/linux/x64/tar/dist/opensearch/opensearch-3.0.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-3.0.0-build.yml}) + downloadBuildManifest.legacySCM(groovy.lang.Closure) + downloadBuildManifest.library({identifier=jenkins@5.5.0, retriever=null}) + downloadBuildManifest.sh(curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/215/linux/x64/tar/dist/opensearch/opensearch-3.0.0-linux-x64.tar.gz --output tests/jenkins/data/opensearch-3.0.0-build.yml) + downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-3.0.0-build.yml}) + BuildManifest.asBoolean() + integ-test.createUploadTestReportManifest({testManifest=manifests/tests/jenkins/data/opensearch-dashboards-3.0.0-test.yml, buildManifest=tests/jenkins/data/opensearch-3.0.0-build.yml, dashboardsBuildManifest=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml, testRunID=215, testType=integ-test, componentName=}) + createUploadTestReportManifest.legacySCM(groovy.lang.Closure) + createUploadTestReportManifest.library({identifier=jenkins@main, retriever=null}) + createUploadTestReportManifest.readYaml({file=manifests/tests/jenkins/data/opensearch-dashboards-3.0.0-test.yml}) + TestManifest.asBoolean() + createUploadTestReportManifest.readYaml({file=tests/jenkins/data/opensearch-3.0.0-build.yml}) + BuildManifest.asBoolean() + createUploadTestReportManifest.readYaml({file=tests/jenkins/data/opensearch-dashboards-3.0.0-build.yml}) + BuildManifest.asBoolean() + BuildManifest.getDistribution() + createUploadTestReportManifest.echo(Start Reporting workflow for test type: tar) + BuildManifest.getArtifactRootUrl(distribution-build-opensearch, 215) + BuildManifest.getArtifactRootUrl(distribution-build-opensearch-dashboards, 215) + createUploadTestReportManifest.echo(Artifact root URL: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/215/linux/x64/tar) + createUploadTestReportManifest.echo(Paths: opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/215/linux/x64/tar opensearch-dashboards=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/3.0.0/215/linux/x64/tar) + createUploadTestReportManifest.echo(Base Path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/3.0.0/215/linux/x64/tar) + createUploadTestReportManifest.echo(Component: ) + createUploadTestReportManifest.echo(Run command: ./report.sh manifests/tests/jenkins/data/opensearch-dashboards-3.0.0-test.yml --artifact-paths opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/215/linux/x64/tar opensearch-dashboards=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/3.0.0/215/linux/x64/tar --test-run-id 215 --test-type integ-test --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/3.0.0/215/linux/x64/tar ) + createUploadTestReportManifest.sh(./report.sh manifests/tests/jenkins/data/opensearch-dashboards-3.0.0-test.yml --artifact-paths opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/215/linux/x64/tar opensearch-dashboards=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/3.0.0/215/linux/x64/tar --test-run-id 215 --test-type integ-test --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/3.0.0/215/linux/x64/tar ) + createUploadTestReportManifest.echo(Build Id: 215) + BuildManifest.getArtifactRoot(dummy_job, 215) + createUploadTestReportManifest.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + createUploadTestReportManifest.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + createUploadTestReportManifest.withCredentials([ARTIFACT_BUCKET_NAME, AWS_ACCOUNT_PUBLIC], groovy.lang.Closure) + createUploadTestReportManifest.echo(Uploading to s3://dummy_job/3.0.0/215/linux/x64/tar/test-results/215/integ-test/test-report.yml) + createUploadTestReportManifest.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + createUploadTestReportManifest.s3Upload({file=/tmp/workspace/test-report.yml, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/3.0.0/215/linux/x64/tar/test-results/215/integ-test/test-report.yml}) integ-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) integ-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt index 24f36dc5c8..e1b9a3d5f2 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt @@ -1,73 +1,76 @@ benchmark-test.run() benchmark-test.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) - benchmark-test.library({identifier=jenkins@5.1.0, retriever=null}) + benchmark-test.library({identifier=jenkins@5.5.0, retriever=null}) benchmark-test.pipeline(groovy.lang.Closure) benchmark-test.timeout({time=24, unit=HOURS}) + benchmark-test.logRotator({daysToKeepStr=30}) + benchmark-test.buildDiscarder(null) benchmark-test.echo(Executing on agent [label:none]) benchmark-test.parameterizedCron( - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} + H 0 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.7.0/7771/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 0 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.7.0/7771/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 0 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 0 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} + H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + + H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-0-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-2-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-4-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-6-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-0-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-2-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-4-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-6-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-0-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-2-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-4-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-6-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-0-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-2-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-4-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-6-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - + H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 15 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-arm64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0"} H 15 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0"} H 15 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-arm64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0"} @@ -77,6 +80,28 @@ H 15 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-arm64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0"} H 15 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-1-shard;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-0-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-2-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-4-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-6-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-1-shard;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-0-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-2-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-4-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-6-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-1-shard;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-0-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-2-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-4-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-6-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-1-shard;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-0-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-2-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-4-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-6-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 18 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-arm64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-min-distribution-zstd-no-dict-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd_no_dict","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} H 18 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-zstd-no-dict-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd_no_dict","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} H 18 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-arm64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-min-distribution-zstd-no-dict-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd_no_dict","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} @@ -91,7 +116,7 @@ benchmark-test.script(groovy.lang.Closure) benchmark-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@4.3.0, retriever=null}) + downloadBuildManifest.library({identifier=jenkins@5.5.0, retriever=null}) downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) BuildManifest.asBoolean() @@ -110,16 +135,16 @@ benchmark-test.script(groovy.lang.Closure) benchmark-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@4.3.0, retriever=null}) + downloadBuildManifest.library({identifier=jenkins@5.5.0, retriever=null}) downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) BuildManifest.asBoolean() benchmark-test.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml) benchmark-test.echo(BUILD_ID: 1234) benchmark-test.echo(Architecture: x64) - benchmark-test.runBenchmarkTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml, workload=nyc-taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, userTag=run-type:test,security-enabled:false, workloadParams=, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, captureNodeStat=false}) + benchmark-test.runBenchmarkTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml, workload=nyc-taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5-4xlarge, userTag=run-type:test,security-enabled:false, workloadParams=, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=false}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) - runBenchmarkTestScript.library({identifier=jenkins@4.3.0, retriever=null}) + runBenchmarkTestScript.library({identifier=jenkins@5.5.0, retriever=null}) runBenchmarkTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) BuildManifest.asBoolean() runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) @@ -136,7 +161,7 @@ BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-node-storage 100 ) + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --data-node-storage 100 ) Messages.asBoolean() Messages.asBoolean() Messages.get([benchmark-test]) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/bwc-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/bwc-test.jenkinsfile.txt index 30e90a16cf..24e8a29348 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/bwc-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/bwc-test.jenkinsfile.txt @@ -3,6 +3,8 @@ bwc-test.library({identifier=jenkins@1.0.4, retriever=null}) bwc-test.pipeline(groovy.lang.Closure) bwc-test.timeout({time=3, unit=HOURS}) + bwc-test.logRotator({daysToKeepStr=30}) + bwc-test.buildDiscarder(null) bwc-test.echo(Executing on agent [label:none]) bwc-test.stage(verify-parameters, groovy.lang.Closure) bwc-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt index f080bb0f8b..22c285f26e 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt @@ -1,51 +1,59 @@ integ-test.run() integ-test.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) - integ-test.library({identifier=jenkins@5.4.1, retriever=null}) + integ-test.library({identifier=jenkins@5.6.0, retriever=null}) integ-test.pipeline(groovy.lang.Closure) integ-test.credentials(jenkins-artifact-bucket-name) integ-test.timeout({time=4, unit=HOURS}) + integ-test.logRotator({daysToKeepStr=30}) + integ-test.buildDiscarder(null) integ-test.echo(Executing on agent [label:none]) + integ-test.parameterizedCron( + H 3 * * * %TEST_MANIFEST=2.10.0/opensearch-2.10.0-concurrent-search-test.yml;BUILD_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/builds/opensearch/manifest.yml + H 3 * * * %TEST_MANIFEST=2.10.0/opensearch-2.10.0-concurrent-search-test.yml;BUILD_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/builds/opensearch/manifest.yml + ) integ-test.stage(verify-parameters, groovy.lang.Closure) integ-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) integ-test.script(groovy.lang.Closure) - integ-test.fileExists(manifests/tests/jenkins/data/opensearch-1.3.0-test.yml) - integ-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/717/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-1.3.0-build.yml}) + integ-test.fileExists(manifests/tests/jenkins/data/opensearch-3.0.0-test.yml) + integ-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/8184/linux/x64/tar/dist/opensearch/opensearch-3.0.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-3.0.0-build.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@5.4.1, retriever=null}) - downloadBuildManifest.sh(curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/717/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz --output tests/jenkins/data/opensearch-1.3.0-build.yml) - downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-build.yml}) + downloadBuildManifest.library({identifier=jenkins@5.5.0, retriever=null}) + downloadBuildManifest.sh(curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/8184/linux/x64/tar/dist/opensearch/opensearch-3.0.0-linux-x64.tar.gz --output tests/jenkins/data/opensearch-3.0.0-build.yml) + downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-3.0.0-build.yml}) BuildManifest.asBoolean() - integ-test.readYaml({file=tests/jenkins/data/opensearch-1.3.0-build.yml}) + integ-test.readYaml({file=tests/jenkins/data/opensearch-3.0.0-build.yml}) BuildManifest.asBoolean() BuildManifest.getNames() BuildManifest.getArtifactArchitecture() BuildManifest.getArtifactBuildId() BuildManifest.getDistribution() - BuildManifest.getArtifactRoot(distribution-build-opensearch, 717) - integ-test.echo(Version: 1.3.0, Agent: Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, BuildId: 717, Distribution: tar, Components: [OpenSearch]) + BuildManifest.getArtifactRoot(distribution-build-opensearch, 8184) + integ-test.echo(Version: 3.0.0, Agent: Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, BuildId: 8184, Distribution: tar, Components: [OpenSearch]) integ-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) integ-test.stage(integ-test, groovy.lang.Closure) integ-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) integ-test.script(groovy.lang.Closure) - integ-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/717/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-1.3.0-build.yml}) + integ-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/8184/linux/x64/tar/dist/opensearch/opensearch-3.0.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-3.0.0-build.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@5.4.1, retriever=null}) - downloadBuildManifest.sh(curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/717/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz --output tests/jenkins/data/opensearch-1.3.0-build.yml) - downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-build.yml}) + downloadBuildManifest.library({identifier=jenkins@5.5.0, retriever=null}) + downloadBuildManifest.sh(curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/8184/linux/x64/tar/dist/opensearch/opensearch-3.0.0-linux-x64.tar.gz --output tests/jenkins/data/opensearch-3.0.0-build.yml) + downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-3.0.0-build.yml}) BuildManifest.asBoolean() - integ-test.readYaml({file=tests/jenkins/data/opensearch-1.3.0-build.yml}) + integ-test.readYaml({file=tests/jenkins/data/opensearch-3.0.0-build.yml}) BuildManifest.asBoolean() BuildManifest.getNames() integ-test.echo(switch_user_non_root: false) integ-test.echo(componentList: [OpenSearch]) - integ-test.echo(Downloading from S3: distribution-build-opensearch/1.3.0/717/linux/x64/tar) - integ-test.downloadFromS3({assumedRoleName=opensearch-bundle, roleAccountNumberCred=jenkins-aws-account-public, downloadPath=distribution-build-opensearch/1.3.0/717/linux/x64/tar/, bucketName=job-s3-bucket-name, localPath=/tmp/workspace/artifacts, force=true}) + integ-test.echo(Downloading from S3: distribution-build-opensearch/3.0.0/8184/linux/x64/tar) + integ-test.downloadFromS3({assumedRoleName=opensearch-bundle, roleAccountNumberCred=jenkins-aws-account-public, downloadPath=distribution-build-opensearch/3.0.0/8184/linux/x64/tar/, bucketName=job-s3-bucket-name, localPath=/tmp/workspace/artifacts, force=true}) downloadFromS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_NUMBER}) downloadFromS3.withCredentials([AWS_ACCOUNT_NUMBER], groovy.lang.Closure) downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure) - downloadFromS3.s3Download({file=/tmp/workspace/artifacts, bucket=job-s3-bucket-name, path=distribution-build-opensearch/1.3.0/717/linux/x64/tar/, force=true}) - integ-test.sh(mv -v /tmp/workspace/artifacts/distribution-build-opensearch/1.3.0/717/linux/x64/tar /tmp/workspace) + downloadFromS3.s3Download({file=/tmp/workspace/artifacts, bucket=job-s3-bucket-name, path=distribution-build-opensearch/3.0.0/8184/linux/x64/tar/, force=true}) + integ-test.sh(ls) + integ-test.sh(ls /tmp/workspace/artifacts) + integ-test.sh(mv -v /tmp/workspace/artifacts/distribution-build-opensearch/3.0.0/8184/linux/x64/tar /tmp/workspace) integ-test.stash({includes=**, name=integtest-opensearch-234}) integ-test.echo(Add Component: OpenSearch) integ-test.parallel({Run Integtest OpenSearch=groovy.lang.Closure}) @@ -54,70 +62,98 @@ integ-test.sh(echo OpenSearch with index 0 will sleep 0 seconds to reduce load && sleep 0) integ-test.unstash(integtest-opensearch-234) integ-test.sh(rm -rf test-results) - integ-test.runIntegTestScript({jobName=distribution-build-opensearch, componentName=OpenSearch, buildManifest=tests/jenkins/data/opensearch-1.3.0-build.yml, testManifest=manifests/tests/jenkins/data/opensearch-1.3.0-test.yml, localPath=/tmp/workspace/tar, switchUserNonRoot=false}) + integ-test.runIntegTestScript({jobName=distribution-build-opensearch, componentName=OpenSearch, buildManifest=tests/jenkins/data/opensearch-3.0.0-build.yml, testManifest=manifests/tests/jenkins/data/opensearch-3.0.0-test.yml, localPath=/tmp/workspace/tar, switchUserNonRoot=false}) runIntegTestScript.legacySCM(groovy.lang.Closure) - runIntegTestScript.library({identifier=jenkins@5.4.1, retriever=null}) - runIntegTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-build.yml}) + runIntegTestScript.library({identifier=jenkins@5.5.0, retriever=null}) + runIntegTestScript.readYaml({file=tests/jenkins/data/opensearch-3.0.0-build.yml}) BuildManifest.asBoolean() BuildManifest.getDistribution() runIntegTestScript.echo(Start integTest for distribution type: tar) - runIntegTestScript.detectTestDockerAgent({testManifest=manifests/tests/jenkins/data/opensearch-1.3.0-test.yml}) + runIntegTestScript.detectTestDockerAgent({testManifest=manifests/tests/jenkins/data/opensearch-3.0.0-test.yml}) detectTestDockerAgent.legacySCM(groovy.lang.Closure) - detectTestDockerAgent.library({identifier=jenkins@5.4.1, retriever=null}) - detectTestDockerAgent.readYaml({file=manifests/tests/jenkins/data/opensearch-1.3.0-test.yml}) + detectTestDockerAgent.library({identifier=jenkins@5.5.0, retriever=null}) + detectTestDockerAgent.readYaml({file=manifests/tests/jenkins/data/opensearch-3.0.0-test.yml}) TestManifest.asBoolean() detectTestDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:ci-runner-centos7-v1 (null)) detectTestDockerAgent.echo(Using java version openjdk-17) runIntegTestScript.echo(Possible Java Home: JAVA_HOME=/opt/java/openjdk-17) - runIntegTestScript.echo(Build Id: 717) - BuildManifest.getArtifactRootUrl(distribution-build-opensearch, 717) - runIntegTestScript.echo(Artifact root URL: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/717/linux/x64/tar) + runIntegTestScript.echo(Build Id: 8184) + BuildManifest.getArtifactRootUrl(distribution-build-opensearch, 8184) + runIntegTestScript.echo(Artifact root URL: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/8184/linux/x64/tar) runIntegTestScript.echo(User provides localPath, use local artifacts: /tmp/workspace/tar) runIntegTestScript.echo(Paths: opensearch=/tmp/workspace/tar) - runIntegTestScript.echo(Base Path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/1.3.0/717/linux/x64/tar) + runIntegTestScript.echo(Base Path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/3.0.0/8184/linux/x64/tar) runIntegTestScript.echo(Component: OpenSearch) runIntegTestScript.echo(Switch User to Non-Root (uid=1000): false) - runIntegTestScript.echo(Run command: env PATH=$PATH JAVA_HOME=/opt/java/openjdk-17 ./test.sh integ-test manifests/tests/jenkins/data/opensearch-1.3.0-test.yml --component OpenSearch --test-run-id 234 --paths opensearch=/tmp/workspace/tar --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/1.3.0/717/linux/x64/tar ) - runIntegTestScript.sh(env PATH=$PATH JAVA_HOME=/opt/java/openjdk-17 ./test.sh integ-test manifests/tests/jenkins/data/opensearch-1.3.0-test.yml --component OpenSearch --test-run-id 234 --paths opensearch=/tmp/workspace/tar --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/1.3.0/717/linux/x64/tar ) + runIntegTestScript.echo(Run command: env PATH=$PATH JAVA_HOME=/opt/java/openjdk-17 ./test.sh integ-test manifests/tests/jenkins/data/opensearch-3.0.0-test.yml --component OpenSearch --test-run-id 234 --paths opensearch=/tmp/workspace/tar --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/3.0.0/8184/linux/x64/tar ) + runIntegTestScript.sh(env PATH=$PATH JAVA_HOME=/opt/java/openjdk-17 ./test.sh integ-test manifests/tests/jenkins/data/opensearch-3.0.0-test.yml --component OpenSearch --test-run-id 234 --paths opensearch=/tmp/workspace/tar --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/3.0.0/8184/linux/x64/tar ) BuildManifest.getRepo(OpenSearch) - integ-test.closeGithubIssue({repoUrl=https://github.com/opensearch-project/OpenSearch.git, issueTitle=[AUTOCUT] Integration Test failed for OpenSearch: 1.3.0 tar distribution, closeComment=Closing the issue as the Integration Test passed for OpenSearch
Version: 1.3.0
Distribution: tar
Architecture: x64
Platform: linux

Please check the logs: https://some/url/redirect

*, label=autocut,v1.3.0,integ-test-failure}) + integ-test.closeGithubIssue({repoUrl=https://github.com/opensearch-project/OpenSearch.git, issueTitle=[AUTOCUT] Integration Test failed for OpenSearch: 3.0.0 tar distribution, closeComment=Closing the issue as the Integration Test passed for OpenSearch
Version: 3.0.0
Distribution: tar
Architecture: x64
Platform: linux

Please check the logs: https://some/url/redirect

*, label=autocut,v3.0.0,integ-test-failure}) closeGithubIssue.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) closeGithubIssue.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - closeGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S "[AUTOCUT] Integration Test failed for OpenSearch: 1.3.0 tar distribution in:title" --label autocut,v1.3.0,integ-test-failure --json number --jq '.[0].number', returnStdout=true}) + closeGithubIssue.sh({script=gh issue list --repo https://github.com/opensearch-project/OpenSearch.git -S "[AUTOCUT] Integration Test failed for OpenSearch: 3.0.0 tar distribution in:title" --label autocut,v3.0.0,integ-test-failure --json number --jq '.[0].number', returnStdout=true}) closeGithubIssue.sh({script=gh issue close bbb -ccc -R opensearch-project/OpenSearch --comment "Closing the issue as the Integration Test passed for OpenSearch
Version: 1.3.0
Distribution: tar
Architecture: x64
Platform: linux

Please check the logs: https://some/url/redirect

*", returnStdout=true}) +ccc -R opensearch-project/OpenSearch --comment "Closing the issue as the Integration Test passed for OpenSearch
Version: 3.0.0
Distribution: tar
Architecture: x64
Platform: linux

Please check the logs: https://some/url/redirect

*", returnStdout=true}) integ-test.echo(Completed running integtest for component OpenSearch) - integ-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-build.yml, jobName=dummy_job}) + integ-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-3.0.0-build.yml, jobName=dummy_job}) uploadTestResults.legacySCM(groovy.lang.Closure) - uploadTestResults.library({identifier=jenkins@5.4.1, retriever=null}) - uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-1.3.0-build.yml}) + uploadTestResults.library({identifier=jenkins@5.5.0, retriever=null}) + uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-3.0.0-build.yml}) BuildManifest.asBoolean() - uploadTestResults.echo(Build Id: 717) - BuildManifest.getArtifactRoot(dummy_job, 717) + uploadTestResults.echo(Build Id: 8184) + BuildManifest.getArtifactRoot(dummy_job, 8184) uploadTestResults.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) uploadTestResults.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) uploadTestResults.withCredentials([ARTIFACT_BUCKET_NAME, AWS_ACCOUNT_PUBLIC], groovy.lang.Closure) - uploadTestResults.echo(Uploading to s3://ARTIFACT_BUCKET_NAME/dummy_job/1.3.0/717/linux/x64/tar) + uploadTestResults.echo(Uploading to s3://ARTIFACT_BUCKET_NAME/dummy_job/3.0.0/8184/linux/x64/tar) uploadTestResults.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - uploadTestResults.s3Upload({file=test-results, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/1.3.0/717/linux/x64/tar/test-results}) + uploadTestResults.s3Upload({file=test-results, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/3.0.0/8184/linux/x64/tar/test-results}) BuildManifest.getArtifactRootUrl(DUMMY_PUBLIC_ARTIFACT_URL, dummy_job) Messages.asBoolean() - Messages.add(DUMMY_STAGE_NAME, https://ci.opensearch.org/ci/dbc/DUMMY_PUBLIC_ARTIFACT_URL/1.3.0/dummy_job/linux/x64/tar/test-results/) - uploadTestResults.writeFile({file=messages/DUMMY_STAGE_NAME.msg, text=https://ci.opensearch.org/ci/dbc/DUMMY_PUBLIC_ARTIFACT_URL/1.3.0/dummy_job/linux/x64/tar/test-results/}) + Messages.add(DUMMY_STAGE_NAME, https://ci.opensearch.org/ci/dbc/DUMMY_PUBLIC_ARTIFACT_URL/3.0.0/dummy_job/linux/x64/tar/test-results/) + uploadTestResults.writeFile({file=messages/DUMMY_STAGE_NAME.msg, text=https://ci.opensearch.org/ci/dbc/DUMMY_PUBLIC_ARTIFACT_URL/3.0.0/dummy_job/linux/x64/tar/test-results/}) uploadTestResults.stash({includes=messages/*, name=messages-DUMMY_STAGE_NAME}) integ-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + integ-test.script(groovy.lang.Closure) + integ-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) + integ-test.unstash(integtest-opensearch-234) + integ-test.createUploadTestReportManifest({testManifest=manifests/tests/jenkins/data/opensearch-3.0.0-test.yml, buildManifest=tests/jenkins/data/opensearch-3.0.0-build.yml, testRunID=234, testType=integ-test, componentName=}) + createUploadTestReportManifest.legacySCM(groovy.lang.Closure) + createUploadTestReportManifest.library({identifier=jenkins@main, retriever=null}) + createUploadTestReportManifest.readYaml({file=manifests/tests/jenkins/data/opensearch-3.0.0-test.yml}) + TestManifest.asBoolean() + createUploadTestReportManifest.readYaml({file=tests/jenkins/data/opensearch-3.0.0-build.yml}) + BuildManifest.asBoolean() + BuildManifest.getDistribution() + createUploadTestReportManifest.echo(Start Reporting workflow for test type: tar) + BuildManifest.getArtifactRootUrl(distribution-build-opensearch, 8184) + createUploadTestReportManifest.echo(Artifact root URL: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/8184/linux/x64/tar) + createUploadTestReportManifest.echo(Paths: opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/8184/linux/x64/tar) + createUploadTestReportManifest.echo(Base Path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/3.0.0/8184/linux/x64/tar) + createUploadTestReportManifest.echo(Component: ) + createUploadTestReportManifest.echo(Run command: ./report.sh manifests/tests/jenkins/data/opensearch-3.0.0-test.yml --artifact-paths opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/8184/linux/x64/tar --test-run-id 234 --test-type integ-test --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/3.0.0/8184/linux/x64/tar ) + createUploadTestReportManifest.sh(./report.sh manifests/tests/jenkins/data/opensearch-3.0.0-test.yml --artifact-paths opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/8184/linux/x64/tar --test-run-id 234 --test-type integ-test --base-path DUMMY_PUBLIC_ARTIFACT_URL/dummy_job/3.0.0/8184/linux/x64/tar ) + createUploadTestReportManifest.echo(Build Id: 8184) + BuildManifest.getArtifactRoot(dummy_job, 8184) + createUploadTestReportManifest.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + createUploadTestReportManifest.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + createUploadTestReportManifest.withCredentials([ARTIFACT_BUCKET_NAME, AWS_ACCOUNT_PUBLIC], groovy.lang.Closure) + createUploadTestReportManifest.echo(Uploading to s3://dummy_job/3.0.0/8184/linux/x64/tar/test-results/234/integ-test/test-report.yml) + createUploadTestReportManifest.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + createUploadTestReportManifest.s3Upload({file=/tmp/workspace/test-report.yml, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/3.0.0/8184/linux/x64/tar/test-results/234/integ-test/test-report.yml}) integ-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) integ-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) integ-test.script(groovy.lang.Closure) + integ-test.echo(I'm still running here**********) Messages.asBoolean() Messages.get([integ-test]) integ-test.unstash({name=messages-integ-test}) integ-test.findFiles({excludes=, glob=messages/*}) integ-test.dir(messages, groovy.lang.Closure) integ-test.deleteDir() - integ-test.publishNotification({icon=:white_check_mark:, message=Integration Tests Successful, extra=, credentialsId=jenkins-integ-test-webhook, manifest=tests/jenkins/data/opensearch-1.3.0-test.yml}) + integ-test.publishNotification({icon=:white_check_mark:, message=Integration Tests Successful, extra=, credentialsId=jenkins-integ-test-webhook, manifest=tests/jenkins/data/opensearch-3.0.0-test.yml}) publishNotification.string({credentialsId=jenkins-integ-test-webhook, variable=WEBHOOK_URL}) publishNotification.withCredentials([WEBHOOK_URL], groovy.lang.Closure) publishNotification.sh(curl -XPOST --header "Content-Type: application/json" --data '{"result_text":":white_check_mark: @@ -125,7 +161,7 @@ JOB_NAME=dummy_job BUILD_NUMBER=[234] MESSAGE=Integration Tests Successful BUILD_URL: htth://BUILD_URL_dummy.com -MANIFEST: tests/jenkins/data/opensearch-1.3.0-test.yml +MANIFEST: tests/jenkins/data/opensearch-3.0.0-test.yml "}' "WEBHOOK_URL") integ-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt index 809c4e99ed..ae080cbe8f 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt @@ -3,6 +3,8 @@ perf-test.library({identifier=jenkins@1.0.4, retriever=null}) perf-test.pipeline(groovy.lang.Closure) perf-test.timeout({time=14, unit=DAYS}) + perf-test.logRotator({daysToKeepStr=30}) + perf-test.buildDiscarder(null) perf-test.echo(Executing on agent [label:none]) perf-test.parameterizedCron( H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.11/8029/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt index 9aa361d257..edd9871539 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt @@ -1,73 +1,76 @@ benchmark-test.run() benchmark-test.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) - benchmark-test.library({identifier=jenkins@5.1.0, retriever=null}) + benchmark-test.library({identifier=jenkins@5.5.0, retriever=null}) benchmark-test.pipeline(groovy.lang.Closure) benchmark-test.timeout({time=24, unit=HOURS}) + benchmark-test.logRotator({daysToKeepStr=30}) + benchmark-test.buildDiscarder(null) benchmark-test.echo(Executing on agent [label:none]) benchmark-test.parameterizedCron( - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} + H 0 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.7.0/7771/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 0 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.7.0/7771/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 0 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 0 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} + H 2 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 6 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + + H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-0-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-2-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-4-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-6-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-0-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-2-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-4-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-6-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-0-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-2-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-4-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-concurrent-search-6-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-0-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-2-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-4-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 11 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-concurrent-search-6-slice-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;ENABLE_REMOTE_STORE=true;USER_TAGS=run-type:nightly,segrep:enabled-with-remote-store,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=opensearch.experimental.feature.remote_store.enabled:true cluster.remote_store.enabled:true opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.10.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} - H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"2","number_of_shards":"3"} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} + H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8184/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=false;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-zstd-codec-1-shard;WORKLOAD_PARAMS={"number_of_shards":"1","index_settings":{"codec":"zstd","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} - + H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"} + H 15 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-arm64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0"} H 15 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0"} H 15 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-arm64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0"} @@ -77,6 +80,28 @@ H 15 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-arm64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:3x,cluster-config:arm64-r6g.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0"} H 15 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-1-shard;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-0-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-2-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-4-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-6-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-1-shard;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-0-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-2-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-4-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 16 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-6-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-1-shard;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-0-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-2-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-4-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-6-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-1-shard;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-0-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-2-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:2;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-4-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-single-node-min-distribution-concurrent-search-1-shard-6-slice;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:6;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"} + H 18 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-arm64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-min-distribution-zstd-no-dict-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd_no_dict","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} H 18 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution-zstd-no-dict-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd_no_dict","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} H 18 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.10.0-SNAPSHOT/opensearch-min-2.10.0-SNAPSHOT-linux-arm64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-min-distribution-zstd-no-dict-codec;WORKLOAD_PARAMS={"index_settings":{"codec":"zstd_no_dict","refresh_interval":"30s","translog":{"flush_threshold_size":"4g"}}} @@ -91,7 +116,7 @@ benchmark-test.script(groovy.lang.Closure) benchmark-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@4.3.0, retriever=null}) + downloadBuildManifest.library({identifier=jenkins@5.5.0, retriever=null}) downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() @@ -109,16 +134,16 @@ benchmark-test.script(groovy.lang.Closure) benchmark-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@4.3.0, retriever=null}) + downloadBuildManifest.library({identifier=jenkins@5.5.0, retriever=null}) downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() benchmark-test.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-bundle.yml) benchmark-test.echo(BUILD_ID: 1234) benchmark-test.echo(Architecture: x64) - benchmark-test.runBenchmarkTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, workload=nyc-taxis, insecure=false, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307-secure, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, userTag=run-type:test,security-enabled:true, workloadParams=, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, captureNodeStat=false}) + benchmark-test.runBenchmarkTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, workload=nyc-taxis, insecure=false, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307-secure, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=run-type:test,security-enabled:true, workloadParams=, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) - runBenchmarkTestScript.library({identifier=jenkins@4.3.0, retriever=null}) + runBenchmarkTestScript.library({identifier=jenkins@5.5.0, retriever=null}) runBenchmarkTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) @@ -135,7 +160,7 @@ BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:true --use-50-percent-heap --enable-remote-store --suffix 307-secure --manager-node-count 3 --data-node-count 3 --data-node-storage 100 ) + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:true --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --data-node-storage 100 ) Messages.asBoolean() Messages.asBoolean() Messages.get([benchmark-test]) @@ -155,16 +180,16 @@ Benchmark tests with security for 1234 completed}) benchmark-test.script(groovy.lang.Closure) benchmark-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@4.3.0, retriever=null}) + downloadBuildManifest.library({identifier=jenkins@5.5.0, retriever=null}) downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() benchmark-test.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-bundle.yml) benchmark-test.echo(BUILD_ID: 1234) benchmark-test.echo(Architecture: x64) - benchmark-test.runBenchmarkTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, workload=nyc-taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, userTag=run-type:test,security-enabled:false, workloadParams=, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, captureNodeStat=false}) + benchmark-test.runBenchmarkTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, workload=nyc-taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=run-type:test,security-enabled:false, workloadParams=, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) - runBenchmarkTestScript.library({identifier=jenkins@4.3.0, retriever=null}) + runBenchmarkTestScript.library({identifier=jenkins@5.5.0, retriever=null}) runBenchmarkTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) @@ -181,7 +206,7 @@ Benchmark tests with security for 1234 completed}) BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false --without-security --use-50-percent-heap --enable-remote-store --suffix 307 --manager-node-count 3 --data-node-count 3 --data-node-storage 100 ) + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false --without-security --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --data-node-storage 100 ) Messages.asBoolean() Messages.asBoolean() Messages.get([benchmark-test]) diff --git a/tests/test_run_build.py b/tests/test_run_build.py index 583493808f..bf7f11ab4b 100644 --- a/tests/test_run_build.py +++ b/tests/test_run_build.py @@ -38,6 +38,7 @@ def test_usage(self) -> None: OPENSEARCH_MANIFEST = os.path.realpath(os.path.join(MANIFESTS, "templates", "opensearch", "1.x", "os-template-1.1.0.yml")) OPENSEARCH_MANIFEST_1_2 = os.path.realpath(os.path.join(MANIFESTS, "templates", "opensearch", "1.x", "os-template-1.2.0.yml")) + NON_OPENSEARCH_MANIFEST = os.path.realpath(os.path.join(MANIFESTS, "templates", "opensearch", "1.x", "non-os-template-1.1.0.yml")) @patch("argparse._sys.argv", ["run_build.py", OPENSEARCH_MANIFEST, "-p", "linux"]) @patch("run_build.Builders.builder_from", return_value=MagicMock()) @@ -160,3 +161,59 @@ def test_main_manifest_new_lock_with_overrides(self, mock_logging: Mock, mock_to mock_stable.assert_called_with() mock_to_file.assert_called_with(self.OPENSEARCH_MANIFEST + ".lock") mock_logging.assert_called_with(f"Creating {self.OPENSEARCH_MANIFEST}.lock") + + @patch("argparse._sys.argv", ["run_build.py", OPENSEARCH_MANIFEST, "-p", "linux", "--continue-on-error"]) + @patch("run_build.Builders.builder_from", return_value=MagicMock()) + @patch("run_build.BuildRecorder", return_value=MagicMock()) + @patch("run_build.TemporaryDirectory") + @patch("run_build.logging.error") + def test_fail_core_component_continue_on_error(self, mock_logging_error: Mock, mock_temp: Mock, mock_recorder: Mock, mock_builder_from: Mock, *mocks: Any) -> None: + mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir() + mock_builder = Mock() + mock_builder.build.side_effect = Exception("Error during build") + mock_builder_from.return_value = mock_builder + with pytest.raises(Exception, match="Error during build"): + main() + mock_logging_error.assert_called_with(f"Error building OpenSearch, retry with: run_build.py {self.OPENSEARCH_MANIFEST} --component OpenSearch") + + @patch("argparse._sys.argv", ["run_build.py", NON_OPENSEARCH_MANIFEST, "-p", "linux", "--continue-on-error"]) + @patch("run_build.Builders.builder_from", return_value=MagicMock()) + @patch("run_build.BuildRecorder", return_value=MagicMock()) + @patch("run_build.TemporaryDirectory") + @patch("run_build.logging.error") + def test_common_utils_failure_continue_on_error(self, mock_logging_error: Mock, mock_temp: Mock, mock_recorder: Mock, mock_builder_from: Mock, *mocks: Any) -> None: + mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir() + mock_builder = Mock() + mock_builder.build.side_effect = Exception("Error building") + mock_builder_from.return_value = mock_builder + with pytest.raises(Exception, match="Error building"): + main() + mock_logging_error.assert_called_with(f"Error building common-utils, retry with: run_build.py {self.NON_OPENSEARCH_MANIFEST} --component common-utils") + + @patch("argparse._sys.argv", ["run_build.py", NON_OPENSEARCH_MANIFEST, "-p", "linux", "--continue-on-error", "--component", "sql", "alerting"]) + @patch("run_build.Builders.builder_from", return_value=MagicMock()) + @patch("run_build.BuildRecorder", return_value=MagicMock()) + @patch("run_build.TemporaryDirectory") + @patch("run_build.logging.error") + def test_fail_plugins_continue_on_error(self, mock_logging_error: Mock, mock_temp: Mock, mock_recorder: Mock, mock_builder_from: Mock, *mocks: Any) -> None: + mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir() + mock_builder = Mock() + mock_builder.build.side_effect = Exception("Error during build") + mock_builder_from.return_value = mock_builder + + main() + mock_logging_error.assert_called_with("Failed plugins are ['sql', 'alerting']") + + @patch("argparse._sys.argv", ["run_build.py", NON_OPENSEARCH_MANIFEST, "-p", "linux"]) + @patch("run_build.Builders.builder_from", return_value=MagicMock()) + @patch("run_build.BuildRecorder", return_value=MagicMock()) + @patch("run_build.TemporaryDirectory") + @patch("run_build.logging.error") + def test_failed_plugins_default(self, mock_logging_error: Mock, mock_temp: Mock, mock_recorder: Mock, mock_builder_from: Mock, *mocks: Any) -> None: + mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir() + mock_builder = Mock() + mock_builder.build.side_effect = Exception("Error during build") + mock_builder_from.return_value = mock_builder + with pytest.raises(Exception, match="Error during build"): + main() + mock_logging_error.assert_called_with(f"Error building common-utils, retry with: run_build.py {self.NON_OPENSEARCH_MANIFEST} --component common-utils") diff --git a/tests/test_run_test_report.py b/tests/test_run_test_report.py index fe5a080612..6cd76fbbb5 100644 --- a/tests/test_run_test_report.py +++ b/tests/test_run_test_report.py @@ -39,7 +39,7 @@ def test_usage(self, *mocks: Any) -> None: self.assertTrue(out.startswith("usage:")) @patch("argparse._sys.argv", ["run_test_report.py", TEST_MANIFEST_PATH, "-p", "opensearch=foo"]) - @patch('run_test_report.TestRunRunner') + @patch('run_test_report.TestReportRunner') def test_main(self, runner_mock: Mock, *mocks: Any) -> None: main() diff --git a/tests/tests_build_workflow/test_build_args.py b/tests/tests_build_workflow/test_build_args.py index 9f85873af6..76e2956168 100644 --- a/tests/tests_build_workflow/test_build_args.py +++ b/tests/tests_build_workflow/test_build_args.py @@ -44,6 +44,14 @@ def test_keep_default(self) -> None: def test_keep_true(self) -> None: self.assertTrue(BuildArgs().keep) + @patch("argparse._sys.argv", [BUILD_PY, OPENSEARCH_MANIFEST]) + def test_continue_on_error_default(self) -> None: + self.assertFalse(BuildArgs().continue_on_error) + + @patch("argparse._sys.argv", [BUILD_PY, OPENSEARCH_MANIFEST, "--continue-on-error"]) + def test_continue_on_error_true(self) -> None: + self.assertTrue(BuildArgs().continue_on_error) + @patch("argparse._sys.argv", [BUILD_PY, OPENSEARCH_MANIFEST]) def test_snapshot_default(self) -> None: self.assertFalse(BuildArgs().snapshot) diff --git a/tests/tests_manifests/test_test_run_manifest.py b/tests/tests_manifests/test_test_run_manifest.py index 203bde3c53..2723bf8fd7 100644 --- a/tests/tests_manifests/test_test_run_manifest.py +++ b/tests/tests_manifests/test_test_run_manifest.py @@ -10,16 +10,16 @@ import yaml -from manifests.test_run_manifest import TestRunManifest +from manifests.test_report_manifest import TestReportManifest -class TestTestRunManifest(unittest.TestCase): +class TestTestReportManifest(unittest.TestCase): def setUp(self) -> None: self.maxDiff = None self.data_path = os.path.realpath(os.path.join(os.path.dirname(__file__), "data")) self.manifest_filename = os.path.join(self.data_path, "test-run.yml") - self.manifest = TestRunManifest.from_path(self.manifest_filename) + self.manifest = TestReportManifest.from_path(self.manifest_filename) def test_test_run(self) -> None: self.assertEqual(self.manifest.name, "OpenSearch") diff --git a/tests/tests_report_workflow/test_test_run_runner.py b/tests/tests_report_workflow/test_test_report_runner.py similarity index 79% rename from tests/tests_report_workflow/test_test_run_runner.py rename to tests/tests_report_workflow/test_test_report_runner.py index b8d1a50461..a6c1bde1ea 100644 --- a/tests/tests_report_workflow/test_test_run_runner.py +++ b/tests/tests_report_workflow/test_test_report_runner.py @@ -11,10 +11,11 @@ from unittest.mock import MagicMock, call, mock_open, patch from manifests.test_manifest import TestManifest -from report_workflow.test_run_runner import TestRunRunner +from report_workflow.test_report_runner import TestReportRunner +from system.temporary_directory import TemporaryDirectory -class TestTestRunRunner(unittest.TestCase): +class TestTestReportRunner(unittest.TestCase): TEST_MANIFEST_PATH = os.path.join( os.path.dirname(__file__), "data", "test_manifest.yml" ) @@ -35,12 +36,35 @@ def test_runner_init(self, report_args_mock: MagicMock, test_manifest_mock: Magi report_args_mock.test_run_id = 123 report_args_mock.test_type = "integ-test" - test_run_runner = TestRunRunner(report_args_mock, self.TEST_MANIFEST) + test_run_runner = TestReportRunner(report_args_mock, self.TEST_MANIFEST) self.assertEqual(test_run_runner.name, "opensearch") self.assertEqual(test_run_runner.test_run_id, 123) self.assertEqual(test_run_runner.test_type, "integ-test") self.assertEqual(test_run_runner.test_manifest_path, self.TEST_MANIFEST_PATH) + @patch("yaml.safe_load") + @patch("urllib.request.urlopen") + @patch("validators.url") + @patch("report_workflow.report_args.ReportArgs") + def test_generate_file(self, report_args_mock: MagicMock, validators_mock: MagicMock, urlopen_mock: MagicMock, yaml_safe_load_mock: MagicMock) -> None: + report_args_mock.test_manifest_path = self.TEST_MANIFEST_PATH + report_args_mock.artifact_paths = {"opensearch": "foo/bar"} + report_args_mock.test_run_id = 123 + report_args_mock.base_path = "https://ci.opensearch.org/ci/dbc/mock" + report_args_mock.test_type = "integ-test" + + validators_mock.return_value = True + yaml_safe_load_mock.return_value = {"test_result": "PASS"} + urlopen_mock.return_value = MagicMock() + + test_run_runner = TestReportRunner(report_args_mock, self.TEST_MANIFEST) + test_run_runner_data = test_run_runner.update_data() + + with TemporaryDirectory() as path: + output_path = os.path.join(path.name, "test-report.yml") + test_run_runner.generate_report(test_run_runner_data, path.name) + self.assertTrue(os.path.isfile(output_path)) + @patch("report_workflow.report_args.ReportArgs") @patch("manifests.test_manifest.TestManifest") def test_runner_update_test_run_data_local(self, report_args_mock: MagicMock, test_manifest_mock: MagicMock) -> None: @@ -49,7 +73,7 @@ def test_runner_update_test_run_data_local(self, report_args_mock: MagicMock, te report_args_mock.test_run_id = 123 report_args_mock.test_type = "integ-test" - test_run_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).update_test_run_data() + test_run_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST).update_test_run_data() self.assertEqual(test_run_dict.get("Command"), " ".join(["./test.sh", "integ-test", self.TEST_MANIFEST_PATH, "--paths", "opensearch=foo/bar"])) self.assertEqual(test_run_dict.get("TestType"), "integ-test") self.assertEqual(test_run_dict.get("TestManifest"), self.TEST_MANIFEST_PATH) @@ -64,7 +88,7 @@ def test_runner_update_test_run_data_url(self, report_args_mock: MagicMock, test report_args_mock.test_run_id = 123 report_args_mock.test_type = "integ-test" - test_run_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).update_test_run_data() + test_run_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST).update_test_run_data() self.assertEqual(test_run_dict.get("Command"), " ".join(["./test.sh", "integ-test", self.TEST_MANIFEST_PATH, "--paths", "opensearch=https://foo/bar"])) self.assertEqual(test_run_dict.get("TestType"), "integ-test") self.assertEqual(test_run_dict.get("TestManifest"), self.TEST_MANIFEST_PATH) @@ -86,7 +110,7 @@ def test_runner_component_entry_url(self, report_args_mock: MagicMock, validator yaml_safe_load_mock.return_value = {"test_result": "PASS"} urlopen_mock.return_value = MagicMock() - test_run_component_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") + test_run_component_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") urlopen_mock.assert_has_calls([call('https://ci.opensearch.org/ci/dbc/mock/test-results/123/integ-test/geospatial/with-security/geospatial.yml')]) self.assertEqual(test_run_component_dict.get("configs")[0]["status"], "PASS") self.assertEqual(test_run_component_dict.get("configs")[0]["name"], "with-security") @@ -107,7 +131,7 @@ def test_runner_component_entry_local(self, report_args_mock: MagicMock, validat yaml_safe_load_mock.return_value = {"test_result": "PASS"} mock_open.return_value = MagicMock() - test_run_component_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") + test_run_component_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") mock_open.assert_has_calls([call('https://ci.opensearch.org/ci/dbc/mock/test-results/123/integ-test/geospatial/with-security/geospatial.yml', 'r', encoding='utf8')]) self.assertEqual(test_run_component_dict.get("configs")[0]["status"], "PASS") self.assertEqual(test_run_component_dict.get("configs")[0]["name"], "with-security") @@ -125,7 +149,7 @@ def test_runner_component_entry_url_invalid(self, report_args_mock: MagicMock, v validators_mock.return_value = True - test_run_component_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") + test_run_component_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") self.assertEqual(test_run_component_dict.get("configs")[0]["status"], "Not Available") self.assertEqual(test_run_component_dict.get("configs")[0]["name"], "with-security") self.assertEqual(test_run_component_dict.get("configs")[0]["yml"], "URL not available") @@ -145,7 +169,7 @@ def test_runner_component_entry_local_invalid(self, report_args_mock: MagicMock, yaml_safe_load_mock.return_value = {"test_result": "PASS"} mock_open.side_effect = FileNotFoundError - test_run_component_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") + test_run_component_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") mock_open.assert_has_calls([call('https://ci.opensearch.org/ci/dbc/mock/test-results/123/integ-test/geospatial/with-security/geospatial.yml', 'r', encoding='utf8')]) self.assertEqual(test_run_component_dict.get("configs")[0]["status"], "Not Available") self.assertEqual(test_run_component_dict.get("configs")[0]["name"], "with-security") diff --git a/tests/tests_sign_workflow/test_signer_jar.py b/tests/tests_sign_workflow/test_signer_jar.py new file mode 100644 index 0000000000..7eba00e8dc --- /dev/null +++ b/tests/tests_sign_workflow/test_signer_jar.py @@ -0,0 +1,61 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + + +import os +import unittest +from pathlib import Path +from unittest.mock import MagicMock, Mock, call, patch + +from sign_workflow.signer_jar import SignerJar + + +class TestSignerJar(unittest.TestCase): + + @patch("sign_workflow.signer.GitRepository") + def test_accepted_file_types(self, git_repo: Mock) -> None: + artifacts = [ + "the-msi.msi", + "the-zip.zip", + "the-jar.jar", + "the-taco.taco", + "the-sys.sys", + "something-1.0.0.0.jar", + ] + expected = [ + call("the-jar.jar", Path("path"), 'null'), + call("the-taco.taco", Path("path"), 'null'), + call("something-1.0.0.0.jar", Path("path"), 'null') + ] + signer = SignerJar(True) + signer.sign = MagicMock() # type: ignore + signer.sign_artifacts(artifacts, Path("path"), 'null') + self.assertEqual(signer.sign.call_args_list, expected) + + @patch("sign_workflow.signer.GitRepository") + @patch('os.rename') + @patch('os.mkdir') + def test_signer_sign(self, mock_os_mkdir: Mock, mock_os_rename: Mock, mock_repo: Mock) -> None: + signer = SignerJar(False) + signer.sign("the-jar.jar", Path("/path/"), "null") + command = "./opensearch-signer-client -i " + os.path.join(Path("/path/"), 'the-jar.jar') + " -o " + os.path.join(Path("/path/"), 'signed_the-jar.jar') + " -p jar_signer -r False" + mock_repo.assert_has_calls( + [call().execute(command)]) + + @patch("sign_workflow.signer.GitRepository") + def test_sign_command_for_overwrite(self, mock_repo: Mock) -> None: + signer = SignerJar(True) + signer.sign("the-taco.taco", Path("/path/"), 'null') + command = "./opensearch-signer-client -i " + os.path.join(Path("/path/"), 'the-taco.taco') + " -o " + os.path.join(Path("/path/"), 'the-taco.taco') + " -p jar_signer" + " -r True" + mock_repo.assert_has_calls( + [call().execute(command)]) + + @patch("sign_workflow.signer.GitRepository") + def test_signer_verify(self, mock_repo: Mock) -> None: + signer = SignerJar(True) + signer.verify('/path/the-jar.jar') + mock_repo.assert_has_calls([call().output("jarsigner -verify /path/the-jar.jar -verbose -certs -strict")]) diff --git a/tests/tests_sign_workflow/test_signers.py b/tests/tests_sign_workflow/test_signers.py index 1c28f4d807..492d106c0d 100644 --- a/tests/tests_sign_workflow/test_signers.py +++ b/tests/tests_sign_workflow/test_signers.py @@ -8,6 +8,7 @@ import unittest from unittest.mock import Mock, patch +from sign_workflow.signer_jar import SignerJar from sign_workflow.signer_mac import SignerMac from sign_workflow.signer_pgp import SignerPGP from sign_workflow.signer_windows import SignerWindows @@ -31,6 +32,11 @@ def test_signer_macos(self, mock_repo: Mock) -> None: signer = Signers.create("mac", True) self.assertIs(type(signer), SignerMac) + @patch("sign_workflow.signer.GitRepository") + def test_signer_jar(self, mock_repo: Mock) -> None: + signer = Signers.create("jar_signer", True) + self.assertIs(type(signer), SignerJar) + def test_signer_invalid(self) -> None: with self.assertRaises(ValueError) as ctx: Signers.create("java", False) diff --git a/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_cluster.py b/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_cluster.py index bce511d696..54e0493de8 100644 --- a/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_cluster.py +++ b/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_cluster.py @@ -65,6 +65,7 @@ def test_port(self) -> None: @patch("test_workflow.benchmark_test.benchmark_test_cluster.BenchmarkTestCluster.wait_for_processing") def test_create_single_node_insecure(self, mock_wait_for_processing: Optional[Mock]) -> None: self.args.insecure = True + self.args.data_instance_type = 'r5.4xlarge' TestBenchmarkTestCluster.setUp(self, self.args) mock_file = MagicMock(side_effect=[{"opensearch-infra-stack-test-suffix-007-x64": {"loadbalancerurl": "www.example.com"}}]) with patch("subprocess.check_call") as mock_check_call: @@ -76,6 +77,7 @@ def test_create_single_node_insecure(self, mock_wait_for_processing: Optional[Mo self.assertEqual(self.benchmark_test_cluster.endpoint_with_port, 'www.example.com:80') self.assertEqual(self.benchmark_test_cluster.port, 80) self.assertTrue("securityDisabled=true" in self.benchmark_test_cluster.params) + self.assertTrue("dataInstanceType=r5.4xlarge" in self.benchmark_test_cluster.params) @patch("test_workflow.benchmark_test.benchmark_test_cluster.BenchmarkTestCluster.wait_for_processing") def test_create_multi_node(self, mock_wait_for_processing: Optional[Mock]) -> None: diff --git a/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_suite.py b/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_suite.py index 395bc53082..44895e0d73 100644 --- a/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_suite.py +++ b/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_suite.py @@ -14,13 +14,14 @@ class TestBenchmarkTestSuite(unittest.TestCase): def setUp(self, config: Optional[str] = None, tag: Optional[str] = None, - workload_params: Optional[str] = None, telemetry: Optional[bool] = None) -> None: + workload_params: Optional[str] = None, telemetry: Optional[list] = None, telemetry_params: Optional[str] = None) -> None: self.args = Mock() self.args.workload = "nyc_taxis" self.args.benchmark_config = config self.args.user_tag = tag self.args.workload_params = workload_params - self.args.capture_node_stat = telemetry + self.args.telemetry = telemetry + self.args.telemetry_params = telemetry_params self.endpoint = "abc.com" self.benchmark_test_suite = BenchmarkTestSuite(endpoint=self.endpoint, security=False, args=self.args) @@ -44,7 +45,7 @@ def test_execute_security_enabled(self) -> None: 'verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'admin\'"') def test_execute_default_with_optional_args(self) -> None: - TestBenchmarkTestSuite.setUp(self, "/home/test/benchmark.ini", "key1:value1,key2:value2", "{\"number_of_replicas\":\"1\"}", True) + TestBenchmarkTestSuite.setUp(self, "/home/test/benchmark.ini", "key1:value1,key2:value2", "{\"number_of_replicas\":\"1\"}", ['node-stats', 'test'], "{\"example_key\":\"example_value\"}") with patch("subprocess.check_call") as mock_check_call: self.benchmark_test_suite.execute() self.assertEqual(mock_check_call.call_count, 1) @@ -54,4 +55,19 @@ def test_execute_default_with_optional_args(self) -> None: '--workload=nyc_taxis ' '--pipeline=benchmark-only --target-hosts=abc.com ' '--workload-params \'{"number_of_replicas":"1"}\' ' - '--user-tag="key1:value1,key2:value2" --telemetry node-stats --client-options="timeout:300"') + '--user-tag="key1:value1,key2:value2" --telemetry node-stats,test, --telemetry-params \'{"example_key":"example_value"}\' ' + '--client-options="timeout:300"') + + def test_execute_default_with_no_telemetry_params(self) -> None: + TestBenchmarkTestSuite.setUp(self, "/home/test/benchmark.ini", "key1:value1,key2:value2", "{\"number_of_replicas\":\"1\"}", ['node-stats', 'test']) + with patch("subprocess.check_call") as mock_check_call: + self.benchmark_test_suite.execute() + self.assertEqual(mock_check_call.call_count, 1) + self.assertEqual(self.benchmark_test_suite.command, 'docker run --rm -v /home/test/benchmark.ini:' + '/opensearch-benchmark/.benchmark/benchmark.ini ' + 'opensearchproject/opensearch-benchmark:latest execute-test ' + '--workload=nyc_taxis ' + '--pipeline=benchmark-only --target-hosts=abc.com ' + '--workload-params \'{"number_of_replicas":"1"}\' ' + '--user-tag="key1:value1,key2:value2" --telemetry node-stats,test, ' + '--client-options="timeout:300"') From 44d0f8a1a51613e58aefd4a0ebb723274b56a57f Mon Sep 17 00:00:00 2001 From: Zelin Hao Date: Wed, 23 Aug 2023 11:41:00 -0700 Subject: [PATCH 3/3] Fix tests Signed-off-by: Zelin Hao --- .../opensearch/integ-test.jenkinsfile.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt index 22c285f26e..96f9612017 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt @@ -51,8 +51,6 @@ downloadFromS3.withCredentials([AWS_ACCOUNT_NUMBER], groovy.lang.Closure) downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure) downloadFromS3.s3Download({file=/tmp/workspace/artifacts, bucket=job-s3-bucket-name, path=distribution-build-opensearch/3.0.0/8184/linux/x64/tar/, force=true}) - integ-test.sh(ls) - integ-test.sh(ls /tmp/workspace/artifacts) integ-test.sh(mv -v /tmp/workspace/artifacts/distribution-build-opensearch/3.0.0/8184/linux/x64/tar /tmp/workspace) integ-test.stash({includes=**, name=integtest-opensearch-234}) integ-test.echo(Add Component: OpenSearch) @@ -146,7 +144,6 @@ ccc -R opensearch-project/OpenSearch --comment "Closing the issue as the Integra postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) integ-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) integ-test.script(groovy.lang.Closure) - integ-test.echo(I'm still running here**********) Messages.asBoolean() Messages.get([integ-test]) integ-test.unstash({name=messages-integ-test})