Skip to content

Commit

Permalink
Auto generate test manifest alongside input manifest in manifest update
Browse files Browse the repository at this point in the history
Signed-off-by: Peter Zhu <[email protected]>
  • Loading branch information
peterzhuamazon committed Aug 29, 2024
1 parent d52e7a2 commit a59c2f3
Show file tree
Hide file tree
Showing 28 changed files with 788 additions and 74 deletions.
22 changes: 17 additions & 5 deletions jenkins/check-for-build.jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,10 @@ pipeline {
}
triggers {
parameterizedCron '''
H 1 * * * %INPUT_MANIFEST=2.16.1/opensearch-2.16.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
H 1 * * * %INPUT_MANIFEST=2.17.0/opensearch-2.17.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
H 1 * * * %INPUT_MANIFEST=2.17.0/opensearch-dashboards-2.17.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
H 1 * * * %INPUT_MANIFEST=2.16.1/opensearch-2.16.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
H 1 * * * %INPUT_MANIFEST=2.17.0/opensearch-2.17.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;TEST_MANIFEST=2.17.0/opensearch-2.17.0-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar
H 1 * * * %INPUT_MANIFEST=2.17.0/opensearch-dashboards-2.17.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;TEST_MANIFEST=2.17.0/opensearch-dashboards-2.17.0-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar
H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
'''
}
Expand Down Expand Up @@ -56,6 +56,16 @@ pipeline {
description: 'Distribution to build',
trim: true
)
string(
name: 'TEST_PLATFORM',
description: 'Platform to test',
trim: true
)
string(
name: 'TEST_DISTRIBUTION',
description: 'Distribution to test',
trim: true
)
}
stages {
stage('detect docker image + args') {
Expand Down Expand Up @@ -101,7 +111,9 @@ pipeline {
string(name: 'INPUT_MANIFEST', value: "${INPUT_MANIFEST}"),
string(name: 'TEST_MANIFEST', value: "${TEST_MANIFEST}"),
string(name: 'BUILD_PLATFORM', value: "${BUILD_PLATFORM}"),
string(name: 'BUILD_DISTRIBUTION', value: "${BUILD_DISTRIBUTION}")
string(name: 'BUILD_DISTRIBUTION', value: "${BUILD_DISTRIBUTION}"),
string(name: 'TEST_PLATFORM', value: "${TEST_PLATFORM}"),
string(name: 'TEST_DISTRIBUTION', value: "${TEST_DISTRIBUTION}")
], wait: true

echo "Build succeeded, uploading build SHA for that job"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -946,6 +946,7 @@ pipeline {
node(AGENT_LINUX_X64) {
checkout scm
script {
sleep 10
def rc = (params.RC_NUMBER.toInteger() > 0)
publishDistributionBuildResults(
failureMessages: buildMessage(search: 'Error building'),
Expand Down
1 change: 1 addition & 0 deletions jenkins/opensearch-dashboards/integ-test.jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,7 @@ pipeline {
}
retry(5) {
node(agent_nodes['linux_x64']) {
sleep 10
def rc = (params.RC_NUMBER.toInteger() > 0)
sh "mkdir -p test-results-osd-${env.BUILD_NUMBER}"
sh "curl -sSL https://ci.opensearch.org/ci/dbc/integ-test-opensearch-dashboards/${env.version}/${env.buildId}/${env.platform}/${env.architecture}/${env.distribution}/test-results/${env.BUILD_NUMBER}/integ-test/test-report.yml --output test-results-osd-${env.BUILD_NUMBER}/test-report.yml"
Expand Down
1 change: 1 addition & 0 deletions jenkins/opensearch/distribution-build.jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -891,6 +891,7 @@ pipeline {
node(AGENT_LINUX_X64) {
checkout scm
script {
sleep 10
def rc = (params.RC_NUMBER.toInteger() > 0)
publishDistributionBuildResults(
failureMessages: buildMessage(search: 'Error building'),
Expand Down
1 change: 1 addition & 0 deletions jenkins/opensearch/integ-test.jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,7 @@ pipeline {
}
retry(5) {
node(agent_nodes['linux_x64']) {
sleep 10
def rc = (params.RC_NUMBER.toInteger() > 0)
sh "mkdir -p test-results-os-${env.BUILD_NUMBER}"
sh "curl -sSL https://ci.opensearch.org/ci/dbc/integ-test/${env.version}/${env.buildId}/${env.platform}/${env.architecture}/${env.distribution}/test-results/${env.BUILD_NUMBER}/integ-test/test-report.yml --output test-results-os-${env.BUILD_NUMBER}/test-report.yml"
Expand Down
30 changes: 15 additions & 15 deletions manifests/1.3.19/opensearch-1.3.19.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@ ci:
components:
- name: OpenSearch
repository: https://github.com/opensearch-project/OpenSearch.git
ref: '1.3'
ref: tags/1.3.19
checks:
- gradle:publish
- gradle:properties:version
- name: common-utils
repository: https://github.com/opensearch-project/common-utils.git
ref: '1.3'
ref: tags/1.3.19.0
checks:
- gradle:publish
- gradle:properties:version
Expand All @@ -25,7 +25,7 @@ components:
- windows
- name: job-scheduler
repository: https://github.com/opensearch-project/job-scheduler.git
ref: '1.3'
ref: tags/1.3.19.0
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version
Expand All @@ -34,7 +34,7 @@ components:
- windows
- name: security
repository: https://github.com/opensearch-project/security.git
ref: '1.3'
ref: tags/1.3.19.0
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version
Expand All @@ -43,7 +43,7 @@ components:
- windows
- name: ml-commons
repository: https://github.com/opensearch-project/ml-commons.git
ref: '1.3'
ref: tags/1.3.19.0
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version: opensearch-ml-plugin
Expand All @@ -54,15 +54,15 @@ components:
- common-utils
- name: performance-analyzer
repository: https://github.com/opensearch-project/performance-analyzer.git
ref: '1.3'
ref: tags/1.3.19.0
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version
platforms:
- linux
- name: alerting
repository: https://github.com/opensearch-project/alerting.git
ref: '1.3'
ref: tags/1.3.19.0
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version: alerting
Expand All @@ -73,7 +73,7 @@ components:
- common-utils
- name: opensearch-reports
repository: https://github.com/opensearch-project/reporting.git
ref: '1.3'
ref: tags/1.3.19.0
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version
Expand All @@ -85,7 +85,7 @@ components:
- job-scheduler
- name: cross-cluster-replication
repository: https://github.com/opensearch-project/cross-cluster-replication.git
ref: '1.3'
ref: tags/1.3.19.0
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version
Expand All @@ -96,7 +96,7 @@ components:
- common-utils
- name: opensearch-observability
repository: https://github.com/opensearch-project/observability.git
ref: '1.3'
ref: tags/1.3.19.0
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version
Expand All @@ -107,7 +107,7 @@ components:
- common-utils
- name: anomaly-detection
repository: https://github.com/opensearch-project/anomaly-detection.git
ref: '1.3'
ref: tags/1.3.19.0
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version
Expand All @@ -119,7 +119,7 @@ components:
- job-scheduler
- name: asynchronous-search
repository: https://github.com/opensearch-project/asynchronous-search.git
ref: '1.3'
ref: tags/1.3.19.0
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version
Expand All @@ -130,7 +130,7 @@ components:
- common-utils
- name: k-NN
repository: https://github.com/opensearch-project/k-NN.git
ref: '1.3'
ref: tags/1.3.19.0
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version
Expand All @@ -139,7 +139,7 @@ components:
- windows
- name: sql
repository: https://github.com/opensearch-project/sql.git
ref: '1.3'
ref: tags/1.3.19.0
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version: plugin
Expand All @@ -150,7 +150,7 @@ components:
- ml-commons
- name: index-management
repository: https://github.com/opensearch-project/index-management.git
ref: '1.3'
ref: tags/1.3.19.0
checks:
- gradle:properties:version
- gradle:dependencies:opensearch.version
Expand Down
20 changes: 10 additions & 10 deletions manifests/1.3.19/opensearch-dashboards-1.3.19.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,31 +9,31 @@ ci:
components:
- name: OpenSearch-Dashboards
repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git
ref: '1.3'
ref: tags/1.3.19
- name: functionalTestDashboards
repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git
ref: '1.3'
ref: tags/1.3.19
- name: anomalyDetectionDashboards
repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin
ref: '1.3'
ref: tags/1.3.19.0
- name: ganttChartDashboards
repository: https://github.com/opensearch-project/dashboards-visualizations.git
ref: '1.3'
ref: tags/1.3.19.0
- name: observabilityDashboards
repository: https://github.com/opensearch-project/dashboards-observability.git
ref: '1.3'
ref: tags/1.3.19.0
- name: alertingDashboards
repository: https://github.com/opensearch-project/alerting-dashboards-plugin.git
ref: '1.3'
ref: tags/1.3.19.0
- name: indexManagementDashboards
repository: https://github.com/opensearch-project/index-management-dashboards-plugin
ref: '1.3'
ref: tags/1.3.19.0
- name: reportsDashboards
repository: https://github.com/opensearch-project/dashboards-reporting.git
ref: '1.3'
ref: tags/1.3.19.0
- name: securityDashboards
repository: https://github.com/opensearch-project/security-dashboards-plugin.git
ref: '1.3'
ref: tags/1.3.19.0
- name: queryWorkbenchDashboards
repository: https://github.com/opensearch-project/dashboards-query-workbench.git
ref: '1.3'
ref: tags/1.3.19.0
143 changes: 143 additions & 0 deletions manifests/2.16.1/opensearch-2.16.1-test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
---
schema-version: '1.0'
name: OpenSearch
ci:
image:
name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1
args: -e JAVA_HOME=/opt/java/openjdk-21
components:
- name: alerting
integ-test:
test-configs:
- with-security
- without-security
additional-cluster-configs:
plugins.destination.host.deny_list:
- 10.0.0.0/8
- 127.0.0.1
bwc-test:
test-configs:
- with-security
- name: anomaly-detection
integ-test:
build-dependencies:
- job-scheduler
test-configs:
- with-security
- without-security
bwc-test:
test-configs:
- with-security
- name: flow-framework
integ-test:
test-configs:
- with-security
- without-security
- name: asynchronous-search
integ-test:
test-configs:
- with-security
- without-security
bwc-test:
test-configs:
- with-security
- name: cross-cluster-replication
integ-test:
topology:
- cluster_name: leader
data_nodes: 2
- cluster_name: follower
data_nodes: 2
test-configs:
- with-security
- without-security
- name: geospatial
integ-test:
test-configs:
- with-security
- without-security
- name: index-management
integ-test:
build-dependencies:
- job-scheduler
test-configs:
- with-security
- without-security
additional-cluster-configs:
path.repo:
- /tmp
bwc-test:
test-configs:
- with-security
- name: k-NN
integ-test:
test-configs:
- with-security
- without-security
- name: ml-commons
integ-test:
test-configs:
- with-security
- without-security
- name: neural-search
integ-test:
test-configs:
- with-security
- without-security
- name: notifications
working-directory: notifications
integ-test:
test-configs:
- with-security
- without-security
bwc-test:
test-configs:
- with-security
- name: opensearch-observability
integ-test:
test-configs:
- with-security
- without-security
bwc-test:
test-configs:
- with-security
- name: opensearch-reports
integ-test:
test-configs:
- with-security
- without-security
- name: security
integ-test:
test-configs:
- with-security
- name: security-analytics
integ-test:
test-configs:
- with-security
- without-security
- name: sql
integ-test:
test-configs:
- with-security
- without-security
additional-cluster-configs:
script.context.field.max_compilations_rate: 1000/1m
plugins.query.datasources.encryption.masterkey: 4fc8fee6a3fd7d6ca01772e5
bwc-test:
test-configs:
- with-security
- name: custom-codecs
integ-test:
test-configs:
- with-security
- without-security
- name: skills
integ-test:
test-configs:
- with-security
- without-security
- name: query-insights
integ-test:
test-configs:
- with-security
- without-security
Loading

0 comments on commit a59c2f3

Please sign in to comment.