From 071d6df03b2714a0f85a6c133e974cec0c1dc6e2 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad <61760125+gaiksaya@users.noreply.github.com> Date: Thu, 30 Jun 2022 12:17:33 -0700 Subject: [PATCH] Update remaining env vars with credentials in jenkins libraries (#2286) * Update getManifestSHA lib env vars with credentials Signed-off-by: Sayali Gaikawad --- tests/jenkins/TestCCRPerfTest.groovy | 2 +- tests/jenkins/TestPromoteYumRepos.groovy | 7 +- .../TestRunNonSecurityPerfTestScript.groovy | 2 +- tests/jenkins/TestRunPerfTestScript.groovy | 2 +- .../perf-test.jenkinsfile.txt | 7 +- .../perf-test-with-security.jenkinsfile.txt | 14 ++- .../opensearch/perf-test.jenkinsfile.txt | 7 +- .../BuildUploadManifestSHA_Jenkinsfile.txt | 7 +- ...ManifestSHA_Jenkinsfile_does_not_exist.txt | 7 +- .../GetManifestSHA_Jenkinsfile_exists.txt | 7 +- .../jobs/PromoteYumRepos_Jenkinsfile.txt | 109 +++++++++--------- vars/getManifestSHA.groovy | 9 +- vars/promoteYumRepos.groovy | 95 +++++++-------- vars/runPerfTestScript.groovy | 9 +- 14 files changed, 158 insertions(+), 126 deletions(-) diff --git a/tests/jenkins/TestCCRPerfTest.groovy b/tests/jenkins/TestCCRPerfTest.groovy index 9711e3804b..16946aff34 100644 --- a/tests/jenkins/TestCCRPerfTest.groovy +++ b/tests/jenkins/TestCCRPerfTest.groovy @@ -53,7 +53,7 @@ class TestCCRPerfTest extends BuildPipelineTest { assertThat(s3DownloadCommands.size(), equalTo(1)) assertThat(s3DownloadCommands, hasItem( - "{file=config.yml, bucket=test_bucket, path=test_config/config-ccr.yml, force=true}".toString() + "{file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config-ccr.yml, force=true}".toString() )) } diff --git a/tests/jenkins/TestPromoteYumRepos.groovy b/tests/jenkins/TestPromoteYumRepos.groovy index 4a171d82bc..c63309fbf9 100644 --- a/tests/jenkins/TestPromoteYumRepos.groovy +++ b/tests/jenkins/TestPromoteYumRepos.groovy @@ -20,9 +20,6 @@ class TestPromoteYumRepos extends BuildPipelineTest { super.setUp() binding.setVariable('PUBLIC_ARTIFACT_URL', 'https://ci.opensearch.org/dbc') - binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'artifactPromotionRole') - binding.setVariable('AWS_ACCOUNT_ARTIFACT', 'artifactsAccount') - binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'prod-bucket-name') binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'github_bot_token_name') def signer_client_creds = ["role": "dummy_role", "external_id": "dummy_ID", @@ -31,6 +28,10 @@ class TestPromoteYumRepos extends BuildPipelineTest { binding.setVariable('configs', signer_client_creds) helper.registerAllowedMethod("readJSON", [Map.class], {c -> signer_client_creds}) helper.registerAllowedMethod("git", [Map]) + helper.registerAllowedMethod("withCredentials", [Map, Closure], { args, closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure -> closure.delegate = delegate return helper.callClosure(closure) diff --git a/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy b/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy index 36bf2a0495..0457aafdfc 100644 --- a/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy +++ b/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy @@ -56,7 +56,7 @@ class TestRunNonSecurityPerfTestScript extends BuildPipelineTest { assertThat(s3DownloadCommands.size(), equalTo(1)) assertThat(s3DownloadCommands, hasItem( - "{file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}".toString() + "{file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true}".toString() )) } diff --git a/tests/jenkins/TestRunPerfTestScript.groovy b/tests/jenkins/TestRunPerfTestScript.groovy index 6bd1445a8b..3aaa792fb8 100644 --- a/tests/jenkins/TestRunPerfTestScript.groovy +++ b/tests/jenkins/TestRunPerfTestScript.groovy @@ -56,7 +56,7 @@ class TestRunPerfTestScript extends BuildPipelineTest { assertThat(s3DownloadCommands.size(), equalTo(2)) assertThat(s3DownloadCommands, hasItem( - "{file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}".toString() + "{file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true}".toString() )) } diff --git a/tests/jenkins/jenkinsjob-regression-files/cross-cluster-replication/perf-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/cross-cluster-replication/perf-test.jenkinsfile.txt index 7a5d5d356f..e9ac6bcc0b 100644 --- a/tests/jenkins/jenkinsjob-regression-files/cross-cluster-replication/perf-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/cross-cluster-replication/perf-test.jenkinsfile.txt @@ -42,8 +42,11 @@ pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" ) - runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config-ccr.yml, force=true}) + runPerfTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runPerfTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runPerfTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runPerfTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runPerfTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config-ccr.yml, force=true}) runPerfTestScript.usernamePassword({credentialsId=bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) runPerfTestScript.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) runPerfTestScript.sh(./test.sh perf-test --stack test-single-security-1236-x64-perf-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --component cross-cluster-replication) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt index 83754a284b..cbf489541a 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt @@ -42,8 +42,11 @@ pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" ) - runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) + runPerfTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runPerfTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runPerfTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runPerfTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runPerfTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true}) runPerfTestScript.usernamePassword({credentialsId=bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) runPerfTestScript.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) runPerfTestScript.sh(./test.sh perf-test --stack test-single-security-1236-x64-perf-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --workload nyc_taxis --test-iters 1 --warmup-iters 1 ) @@ -102,8 +105,11 @@ Performance tests with security for 1236 completed}) pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" ) - runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) + runPerfTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runPerfTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runPerfTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runPerfTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runPerfTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true}) runPerfTestScript.usernamePassword({credentialsId=bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) runPerfTestScript.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64-perf-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --without-security --workload nyc_taxis --test-iters 1 --warmup-iters 1 ) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt index 514d265b0b..9e7dc671ac 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt @@ -43,8 +43,11 @@ pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" ) - runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) + runPerfTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runPerfTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runPerfTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runPerfTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runPerfTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true}) runPerfTestScript.usernamePassword({credentialsId=bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) runPerfTestScript.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64-perf-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config config.yml --without-security --workload nyc_taxis --test-iters 1 --warmup-iters 1 ) diff --git a/tests/jenkins/jobs/BuildUploadManifestSHA_Jenkinsfile.txt b/tests/jenkins/jobs/BuildUploadManifestSHA_Jenkinsfile.txt index 24976b4fbb..c163c31312 100644 --- a/tests/jenkins/jobs/BuildUploadManifestSHA_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BuildUploadManifestSHA_Jenkinsfile.txt @@ -18,8 +18,11 @@ InputManifest.getSHAsRoot(get-manifest-sha-build) getManifestSHA.echo(Manifest lock: tests/jenkins/data/opensearch-1.3.0.yml.lock) getManifestSHA.echo(Manifest SHA path: get-manifest-sha-build/1.3.0/shas/sha1.yml) - getManifestSHA.withAWS({role=opensearch-bundle, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - getManifestSHA.s3DoesObjectExist({bucket=artifact-bucket, path=get-manifest-sha-build/1.3.0/shas/sha1.yml}) + getManifestSHA.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + getManifestSHA.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + getManifestSHA.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + getManifestSHA.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + getManifestSHA.s3DoesObjectExist({bucket=ARTIFACT_BUCKET_NAME, path=get-manifest-sha-build/1.3.0/shas/sha1.yml}) getManifestSHA.echo(Manifest SHA exists: false) buildUploadManifestSHA.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) buildUploadManifestSHA.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) diff --git a/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_does_not_exist.txt b/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_does_not_exist.txt index ac58fb01af..219eadbcc4 100644 --- a/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_does_not_exist.txt +++ b/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_does_not_exist.txt @@ -15,8 +15,11 @@ InputManifest.getSHAsRoot(get-manifest-sha-build) getManifestSHA.echo(Manifest lock: tests/jenkins/data/opensearch-1.3.0.yml.lock) getManifestSHA.echo(Manifest SHA path: get-manifest-sha-build/1.3.0/shas/sha1.yml) - getManifestSHA.withAWS({role=opensearch-bundle, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - getManifestSHA.s3DoesObjectExist({bucket=artifact-bucket, path=get-manifest-sha-build/1.3.0/shas/sha1.yml}) + getManifestSHA.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + getManifestSHA.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + getManifestSHA.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + getManifestSHA.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + getManifestSHA.s3DoesObjectExist({bucket=ARTIFACT_BUCKET_NAME, path=get-manifest-sha-build/1.3.0/shas/sha1.yml}) getManifestSHA.echo(Manifest SHA exists: false) GetManifestSHA_Jenkinsfile.echo(sha: sha1) GetManifestSHA_Jenkinsfile.echo(exists: false) diff --git a/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_exists.txt b/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_exists.txt index 81b97f07b5..5a18992ce9 100644 --- a/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_exists.txt +++ b/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_exists.txt @@ -15,8 +15,11 @@ InputManifest.getSHAsRoot(get-manifest-sha-build) getManifestSHA.echo(Manifest lock: tests/jenkins/data/opensearch-1.3.0.yml.lock) getManifestSHA.echo(Manifest SHA path: get-manifest-sha-build/1.3.0/shas/sha1.yml) - getManifestSHA.withAWS({role=opensearch-bundle, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - getManifestSHA.s3DoesObjectExist({bucket=artifact-bucket, path=get-manifest-sha-build/1.3.0/shas/sha1.yml}) + getManifestSHA.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + getManifestSHA.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + getManifestSHA.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + getManifestSHA.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + getManifestSHA.s3DoesObjectExist({bucket=ARTIFACT_BUCKET_NAME, path=get-manifest-sha-build/1.3.0/shas/sha1.yml}) getManifestSHA.echo(Manifest SHA exists: true) GetManifestSHA_Jenkinsfile.echo(sha: sha1) GetManifestSHA_Jenkinsfile.echo(exists: true) diff --git a/tests/jenkins/jobs/PromoteYumRepos_Jenkinsfile.txt b/tests/jenkins/jobs/PromoteYumRepos_Jenkinsfile.txt index f6d2d3e505..5bece22db1 100644 --- a/tests/jenkins/jobs/PromoteYumRepos_Jenkinsfile.txt +++ b/tests/jenkins/jobs/PromoteYumRepos_Jenkinsfile.txt @@ -8,47 +8,51 @@ promoteYumRepos.library({identifier=jenkins@20211123, retriever=null}) promoteYumRepos.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml}) InputManifest.asBoolean() - promoteYumRepos.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteYumRepos.println(Pulling Prod Yumrepo) - promoteYumRepos.sh(aws s3 sync s3://prod-bucket-name/releases/bundle/opensearch/1.x/yum/ /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ --no-progress) - promoteYumRepos.sh( - set -e - set +x - set +x - - echo "Pulling 1.3.0 rpms" - cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum - curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/x64/rpm/dist/opensearch/opensearch-1.3.0-linux-x64.rpm - curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/arm64/rpm/dist/opensearch/opensearch-1.3.0-linux-arm64.rpm - - ls -l - - rm -vf repodata/repomd.xml.asc - - echo "Update repo metadata" - createrepo --update . - - # Rename .xml to .pom for signing - # Please do not add .xml to signer filter - # As maven have many .xml and we do not want to sign them - # This is an outlier case for yum repo only - mv -v repodata/repomd.xml repodata/repomd.pom - - echo "Complete metadata update, awaiting signing repomd.xml" + promoteYumRepos.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) + promoteYumRepos.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) + promoteYumRepos.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) + promoteYumRepos.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) + promoteYumRepos.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteYumRepos.println(Pulling Prod Yumrepo) + promoteYumRepos.sh(aws s3 sync s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/yum/ /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ --no-progress) + promoteYumRepos.sh( + set -e + set +x + set +x + + echo "Pulling 1.3.0 rpms" + cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum + curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/x64/rpm/dist/opensearch/opensearch-1.3.0-linux-x64.rpm + curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/arm64/rpm/dist/opensearch/opensearch-1.3.0-linux-arm64.rpm + + ls -l + + rm -vf repodata/repomd.xml.asc + + echo "Update repo metadata" + createrepo --update . + + # Rename .xml to .pom for signing + # Please do not add .xml to signer filter + # As maven have many .xml and we do not want to sign them + # This is an outlier case for yum repo only + mv -v repodata/repomd.xml repodata/repomd.pom + + echo "Complete metadata update, awaiting signing repomd.xml" cd - ) - promoteYumRepos.signArtifacts({artifactPath=/tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom, sigtype=.sig, platform=linux}) - signArtifacts.echo(PGP Signature Signing) - signArtifacts.fileExists(/tmp/workspace/sign.sh) - signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) - signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) - signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) - signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=signer_client_creds}) - signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], signer_client_creds], groovy.lang.Closure) - signArtifacts.readJSON({text=signer_client_creds}) - signArtifacts.sh( + promoteYumRepos.signArtifacts({artifactPath=/tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom, sigtype=.sig, platform=linux}) + signArtifacts.echo(PGP Signature Signing) + signArtifacts.fileExists(/tmp/workspace/sign.sh) + signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) + signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) + signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=signer_client_creds}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], signer_client_creds], groovy.lang.Closure) + signArtifacts.readJSON({text=signer_client_creds}) + signArtifacts.sh( #!/bin/bash set +x export ROLE=dummy_role @@ -58,27 +62,26 @@ /tmp/workspace/sign.sh /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom --sigtype=.sig --platform=linux ) - promoteYumRepos.sh( - set -e - set +x + promoteYumRepos.sh( + set -e + set +x - cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/ + cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/ - ls -l + ls -l - mv -v repomd.pom repomd.xml - mv -v repomd.pom.sig repomd.xml.sig + mv -v repomd.pom repomd.xml + mv -v repomd.pom.sig repomd.xml.sig - # This step is required as yum only accept .asc and signing workflow only support .sig - cat repomd.xml.sig | gpg --enarmor | sed 's@ARMORED FILE@SIGNATURE@g' > repomd.xml.asc + # This step is required as yum only accept .asc and signing workflow only support .sig + cat repomd.xml.sig | gpg --enarmor | sed 's@ARMORED FILE@SIGNATURE@g' > repomd.xml.asc - rm -vf repomd.xml.sig + rm -vf repomd.xml.sig - ls -l - - cd - + ls -l + cd - ) - promoteYumRepos.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteYumRepos.println(Pushing Prod Yumrepo) - promoteYumRepos.sh(aws s3 sync /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ s3://prod-bucket-name/releases/bundle/opensearch/1.x/yum/ --no-progress) + promoteYumRepos.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteYumRepos.println(Pushing Prod Yumrepo) + promoteYumRepos.sh(aws s3 sync /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/yum/ --no-progress) diff --git a/vars/getManifestSHA.groovy b/vars/getManifestSHA.groovy index 23a810fde3..0ff2d7ea22 100644 --- a/vars/getManifestSHA.groovy +++ b/vars/getManifestSHA.groovy @@ -21,11 +21,14 @@ Map call(Map args = [:]) { echo "Manifest SHA path: ${manifestSHAPath}" Boolean manifestSHAExists = false - withAWS(role: 'opensearch-bundle', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { - if (s3DoesObjectExist(bucket: "${ARTIFACT_BUCKET_NAME}", path: manifestSHAPath)) { - manifestSHAExists = true + withCredentials([string(credentialsId: 'jenkins-aws-account-public', variable: 'AWS_ACCOUNT_PUBLIC'), + string(credentialsId: 'jenkins-artifact-bucket-name', variable: 'ARTIFACT_BUCKET_NAME')]) { + withAWS(role: 'opensearch-bundle', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { + if (s3DoesObjectExist(bucket: "${ARTIFACT_BUCKET_NAME}", path: manifestSHAPath)) { + manifestSHAExists = true } } + } echo "Manifest SHA exists: ${manifestSHAExists}" diff --git a/vars/promoteYumRepos.groovy b/vars/promoteYumRepos.groovy index a18be4835e..f1e1614b8c 100644 --- a/vars/promoteYumRepos.groovy +++ b/vars/promoteYumRepos.groovy @@ -17,7 +17,7 @@ void call(Map args = [:]) { String buildnumber = args.buildNumber ?: 'none' if (buildnumber == 'none') { - println("User did not enter build number in jenkins parameter, exit 1") + println('User did not enter build number in jenkins parameter, exit 1') System.exit(1) } @@ -40,72 +40,73 @@ void call(Map args = [:]) { String yumRepoProdPath = "releases/bundle/${filename}/${yumRepoVersion}/yum" String artifactPath = "${localPath}/${yumRepoProdPath}" + withCredentials([string(credentialsId: 'jenkins-artifact-promotion-role', variable: 'ARTIFACT_PROMOTION_ROLE_NAME'), + string(credentialsId: 'jenkins-aws-production-account', variable: 'AWS_ACCOUNT_ARTIFACT'), + string(credentialsId: 'jenkins-artifact-production-bucket-name', variable: 'ARTIFACT_PRODUCTION_BUCKET_NAME')]) { + withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { + println('Pulling Prod Yumrepo') + sh("aws s3 sync s3://${ARTIFACT_PRODUCTION_BUCKET_NAME}/${yumRepoProdPath}/ ${artifactPath}/ --no-progress") + } - withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { - println("Pulling Prod Yumrepo") - sh("aws s3 sync s3://${ARTIFACT_PRODUCTION_BUCKET_NAME}/${yumRepoProdPath}/ ${artifactPath}/ --no-progress") - } - - sh """ - set -e - set +x - set +x + sh """ + set -e + set +x + set +x - echo "Pulling ${revision} rpms" - cd ${artifactPath} - curl -SLO ${stagingYumPathX64} - curl -SLO ${stagingYumPathARM64} + echo "Pulling ${revision} rpms" + cd ${artifactPath} + curl -SLO ${stagingYumPathX64} + curl -SLO ${stagingYumPathARM64} - ls -l + ls -l - rm -vf repodata/repomd.xml.asc + rm -vf repodata/repomd.xml.asc - echo "Update repo metadata" - createrepo --update . + echo "Update repo metadata" + createrepo --update . - # Rename .xml to .pom for signing - # Please do not add .xml to signer filter - # As maven have many .xml and we do not want to sign them - # This is an outlier case for yum repo only - mv -v repodata/repomd.xml repodata/repomd.pom + # Rename .xml to .pom for signing + # Please do not add .xml to signer filter + # As maven have many .xml and we do not want to sign them + # This is an outlier case for yum repo only + mv -v repodata/repomd.xml repodata/repomd.pom - echo "Complete metadata update, awaiting signing repomd.xml" + echo "Complete metadata update, awaiting signing repomd.xml" cd - """ - signArtifacts( - artifactPath: "${artifactPath}/repodata/repomd.pom", - sigtype: '.sig', - platform: 'linux' - ) + signArtifacts( + artifactPath: "${artifactPath}/repodata/repomd.pom", + sigtype: '.sig', + platform: 'linux' + ) - sh """ - set -e - set +x + sh """ + set -e + set +x - cd ${artifactPath}/repodata/ + cd ${artifactPath}/repodata/ - ls -l + ls -l - mv -v repomd.pom repomd.xml - mv -v repomd.pom.sig repomd.xml.sig + mv -v repomd.pom repomd.xml + mv -v repomd.pom.sig repomd.xml.sig - # This step is required as yum only accept .asc and signing workflow only support .sig - cat repomd.xml.sig | gpg --enarmor | sed 's@ARMORED FILE@SIGNATURE@g' > repomd.xml.asc + # This step is required as yum only accept .asc and signing workflow only support .sig + cat repomd.xml.sig | gpg --enarmor | sed 's@ARMORED FILE@SIGNATURE@g' > repomd.xml.asc - rm -vf repomd.xml.sig + rm -vf repomd.xml.sig - ls -l - - cd - + ls -l + cd - """ - withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { - println("Pushing Prod Yumrepo") - sh("aws s3 sync ${artifactPath}/ s3://${ARTIFACT_PRODUCTION_BUCKET_NAME}/${yumRepoProdPath}/ --no-progress") - } - + withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { + println('Pushing Prod Yumrepo') + sh("aws s3 sync ${artifactPath}/ s3://${ARTIFACT_PRODUCTION_BUCKET_NAME}/${yumRepoProdPath}/ --no-progress") + } + } } diff --git a/vars/runPerfTestScript.groovy b/vars/runPerfTestScript.groovy index 550ad42c6e..6f31fa537f 100644 --- a/vars/runPerfTestScript.groovy +++ b/vars/runPerfTestScript.groovy @@ -4,9 +4,12 @@ void call(Map args = [:]) { install_opensearch_infra_dependencies() config_name = isNullOrEmpty(args.config) ? "config.yml" : args.config - withAWS(role: 'opensearch-test', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { - s3Download(file: "config.yml", bucket: "${ARTIFACT_BUCKET_NAME}", path: "${PERF_TEST_CONFIG_LOCATION}/${config_name}", force: true) - } + withCredentials([string(credentialsId: 'jenkins-aws-account-public', variable: 'AWS_ACCOUNT_PUBLIC'), + string(credentialsId: 'jenkins-artifact-bucket-name', variable: 'ARTIFACT_BUCKET_NAME')]) { + withAWS(role: 'opensearch-test', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { + s3Download(file: "config.yml", bucket: "${ARTIFACT_BUCKET_NAME}", path: "${PERF_TEST_CONFIG_LOCATION}/${config_name}", force: true) + } + } String stackNameSuffix = isNullOrEmpty(args.stackNameSuffix) ? 'perf-test' : args.stackNameSuffix