From a9be8be8d2cabedfbfde2929c1162251812271f9 Mon Sep 17 00:00:00 2001 From: awsluja Date: Fri, 12 May 2023 18:38:22 -0400 Subject: [PATCH 1/6] ci: remove dependance on webhooks --- .circleci/publish-codebuild.sh | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/.circleci/publish-codebuild.sh b/.circleci/publish-codebuild.sh index 3b8943248c0..6c655862271 100755 --- a/.circleci/publish-codebuild.sh +++ b/.circleci/publish-codebuild.sh @@ -1,4 +1,8 @@ #!/bin/bash -e +export BRANCH_NAME=$(git branch -a --contains HEAD | sed -n 2p | awk ‘{ printf $1 }’) +git checkout $BRANCH_NAME +echo "fetching tags" +git fetch --tags https://github.com/aws-amplify/amplify-cli # lerna has a bug (https://github.com/lerna/lerna/issues/1066) where failed publishes do not set the exit code properly # this causes the script to keep running even after failed publishes @@ -34,13 +38,13 @@ else git config --global user.name $GITHUB_USER fi -if [[ "$CIRCLE_BRANCH" =~ ^tagged-release ]]; then - if [[ "$CIRCLE_BRANCH" =~ ^tagged-release-without-e2e-tests\/.* ]]; then +if [[ "$BRANCH_NAME" =~ ^tagged-release ]]; then + if [[ "$BRANCH_NAME" =~ ^tagged-release-without-e2e-tests\/.* ]]; then # Remove tagged-release-without-e2e-tests/ - export NPM_TAG="${CIRCLE_BRANCH/tagged-release-without-e2e-tests\//}" - elif [[ "$CIRCLE_BRANCH" =~ ^tagged-release\/.* ]]; then + export NPM_TAG="${BRANCH_NAME/tagged-release-without-e2e-tests\//}" + elif [[ "$BRANCH_NAME" =~ ^tagged-release\/.* ]]; then # Remove tagged-release/ - export NPM_TAG="${CIRCLE_BRANCH/tagged-release\//}" + export NPM_TAG="${BRANCH_NAME/tagged-release\//}" fi if [ -z "$NPM_TAG" ]; then echo "Tag name is missing. Name your branch with either tagged-release/ or tagged-release-without-e2e-tests/" @@ -56,7 +60,7 @@ if [[ "$CIRCLE_BRANCH" =~ ^tagged-release ]]; then fi # @latest release -elif [[ "$CIRCLE_BRANCH" == "release" ]]; then +elif [[ "$BRANCH_NAME" == "release" ]]; then # create release commit and release tags npx lerna version --exact --conventional-commits --conventional-graduate --yes --no-push --include-merged-tags --message "chore(release): Publish latest [ci skip]" @@ -70,7 +74,7 @@ elif [[ "$CIRCLE_BRANCH" == "release" ]]; then fi # push release commit - git push origin "$CIRCLE_BRANCH" + git push origin "$BRANCH_NAME" # push release tags git tag --points-at HEAD | xargs git push origin @@ -88,33 +92,33 @@ elif [[ "$CIRCLE_BRANCH" == "release" ]]; then git push origin hotfix # release candidate or local publish for testing / building binary -elif [[ "$CIRCLE_BRANCH" =~ ^run-e2e-with-rc\/.* ]] || [[ "$CIRCLE_BRANCH" =~ ^release_rc\/.* ]] || [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then +elif [[ "$BRANCH_NAME" =~ ^run-e2e-with-rc\/.* ]] || [[ "$BRANCH_NAME" =~ ^release_rc\/.* ]] || [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then # force @aws-amplify/cli-internal to be versioned in case this pipeline run does not have any commits that modify the CLI packages if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then force_publish_local_args="--force-publish '@aws-amplify/cli-internal'" fi # create release commit and release tags - git checkout "${CODEBUILD_WEBHOOK_TRIGGER#branch/*}" && npx lerna version --preid=rc.$CODEBUILD_RESOLVED_SOURCE_VERSION --exact --conventional-prerelease --conventional-commits --yes --no-push --include-merged-tags --message "chore(release): Publish rc [ci skip]" $(echo $force_publish_local_args) --no-commit-hooks + npx lerna version --preid=rc.$(git rev-parse --short HEAD) --exact --conventional-prerelease --conventional-commits --yes --no-push --include-merged-tags --message "chore(release): Publish rc [ci skip]" $(echo $force_publish_local_args) --no-commit-hooks # if publishing locally to verdaccio if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then # publish to verdaccio with no dist tag (default to latest) - lernaPublishExitOnFailure from-package --git-head $CODEBUILD_RESOLVED_SOURCE_VERSION --yes --no-push + lernaPublishExitOnFailure from-git --yes --no-push echo "Published packages to verdaccio" echo "Exiting without pushing release commit or release tags" exit 0 fi # publish versions that were just computed - lernaPublishExitOnFailure from-package --git-head $CODEBUILD_RESOLVED_SOURCE_VERSION --yes --no-push --dist-tag rc + lernaPublishExitOnFailure from-git --yes --no-push --dist-tag rc # push release commit - git push origin "${CODEBUILD_WEBHOOK_TRIGGER#branch/*}" + git push origin "$BRANCH_NAME" # push release tags git tag --points-at HEAD | xargs git push origin else - echo "branch name" "${CODEBUILD_WEBHOOK_TRIGGER#branch/*}" "did not match any branch publish rules. Skipping publish" + echo "branch name" "$BRANCH_NAME" "did not match any branch publish rules. Skipping publish" fi From 856f3062b9fd5113a5cfec6ff55f9eaf319f9280 Mon Sep 17 00:00:00 2001 From: awsluja Date: Fri, 12 May 2023 19:49:57 -0400 Subject: [PATCH 2/6] ci: try again --- .circleci/publish-codebuild.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/publish-codebuild.sh b/.circleci/publish-codebuild.sh index 6c655862271..e8ad5412bd5 100755 --- a/.circleci/publish-codebuild.sh +++ b/.circleci/publish-codebuild.sh @@ -1,5 +1,8 @@ #!/bin/bash -e -export BRANCH_NAME=$(git branch -a --contains HEAD | sed -n 2p | awk ‘{ printf $1 }’) +BRANCH_NAME="$(git symbolic-ref HEAD --short 2>/dev/null)" +if [ "$BRANCH_NAME" = "" ] ; then + BRANCH_NAME="$(git rev-parse HEAD | xargs git name-rev | cut -d' ' -f2 | sed 's/remotes\/origin\///g')"; +fi git checkout $BRANCH_NAME echo "fetching tags" git fetch --tags https://github.com/aws-amplify/amplify-cli From 39b15ed2cd49d7574ec0ba55b9e467eaeb7cf030 Mon Sep 17 00:00:00 2001 From: awsluja Date: Mon, 15 May 2023 21:27:20 -0400 Subject: [PATCH 3/6] ci: migrate to manual cb triggers for e2e --- .circleci/publish-codebuild.sh | 2 +- package.json | 4 ++-- scripts/cloud-e2e-beta.sh | 18 ++++++++++++++++++ scripts/cloud-e2e.sh | 23 +++++++++++++++++++++++ scripts/sample.env | 4 ++++ 5 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 scripts/cloud-e2e-beta.sh create mode 100644 scripts/cloud-e2e.sh create mode 100644 scripts/sample.env diff --git a/.circleci/publish-codebuild.sh b/.circleci/publish-codebuild.sh index e8ad5412bd5..bf60e4be5cd 100755 --- a/.circleci/publish-codebuild.sh +++ b/.circleci/publish-codebuild.sh @@ -1,5 +1,5 @@ #!/bin/bash -e -BRANCH_NAME="$(git symbolic-ref HEAD --short 2>/dev/null)" +export BRANCH_NAME="$(git symbolic-ref HEAD --short 2>/dev/null)" if [ "$BRANCH_NAME" = "" ] ; then BRANCH_NAME="$(git rev-parse HEAD | xargs git name-rev | cut -d' ' -f2 | sed 's/remotes\/origin\///g')"; fi diff --git a/package.json b/package.json index 3b70714fef4..ad873462ae8 100644 --- a/package.json +++ b/package.json @@ -11,8 +11,8 @@ "check-version-conflicts:beta": "yarn ts-node ./scripts/check-version.ts beta", "check-version-conflicts:release": "yarn ts-node ./scripts/check-version.ts", "clean": "nx reset && nx run-many --target=clean --all && yarn rimraf tsconfig.tsbuildinfo", - "cloud-e2e-with-rc": "CURR_BRANCH=$(git branch | awk '/\\*/{printf \"%s\", $2}') && UPSTREAM_BRANCH=run-e2e-with-rc/$USER/$CURR_BRANCH && git push $(git remote -v | grep aws-amplify/amplify-cli | head -n1 | awk '{print $1;}') $CURR_BRANCH:$UPSTREAM_BRANCH --no-verify --force-with-lease && echo \"\n\n 🏃 E2E test are running at:\nhttps://app.circleci.com/pipelines/github/aws-amplify/amplify-cli?branch=$UPSTREAM_BRANCH\"", - "cloud-e2e": "CURR_BRANCH=$(git branch | awk '/\\*/{printf \"%s\", $2}') && UPSTREAM_BRANCH=run-e2e/$USER/$CURR_BRANCH && git push $(git remote -v | grep aws-amplify/amplify-cli | head -n1 | awk '{print $1;}') $CURR_BRANCH:$UPSTREAM_BRANCH --no-verify --force-with-lease && echo \"\n\n 🏃 E2E test are running at:\nhttps://app.circleci.com/pipelines/github/aws-amplify/amplify-cli?branch=$UPSTREAM_BRANCH\"", + "cloud-e2e": "source ./scripts/cloud-e2e.sh", + "cloud-e2e-beta": "source ./scripts/cloud-e2e-beta.sh", "commit": "git-cz", "coverage:collect": "ts-node ./scripts/collect-test-coverage.ts", "coverage": "codecov || exit 0", diff --git a/scripts/cloud-e2e-beta.sh b/scripts/cloud-e2e-beta.sh new file mode 100644 index 00000000000..40635ee02d7 --- /dev/null +++ b/scripts/cloud-e2e-beta.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# set exit on error to true +set -e + +set -o allexport +source ./scripts/.env set +PROFILE=AmplifyCLIE2EBeta + +ACCOUNT=$E2E_ACCOUNT_BETA +echo Submitting CodeBuild Request to AWS Account: $ACCOUNT +CURR_BRANCH=$(git branch --show-current) +echo Current branch is: $CURR_BRANCH +mwinit +ada cred update --profile="${PROFILE}" --account="${ACCOUNT}" --role=CodeBuildE2E --provider=isengard --once +echo credentials loaded +RESULT=$(aws codebuild start-build-batch --profile="${PROFILE}" --project-name AmplifyCLI-E2E-Testing --source-version=$UPSTREAM_BRANCH --query 'buildBatch.id' --output text) +echo "https://us-east-1.console.aws.amazon.com/codesuite/codebuild/$ACCOUNT/projects/AmplifyCLI-E2E-Testing/batch/$RESULT?region=us-east-1" diff --git a/scripts/cloud-e2e.sh b/scripts/cloud-e2e.sh new file mode 100644 index 00000000000..b9d9e19f65a --- /dev/null +++ b/scripts/cloud-e2e.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# set exit on error to true +set -e +# load .env +set -o allexport +source ./scripts/.env set +PROFILE=AmplifyCLIE2E + +ACCOUNT=$E2E_ACCOUNT_PROD +echo Submitting CodeBuild Request to AWS Account: $ACCOUNT +CURR_BRANCH=$(git branch --show-current) +UPSTREAM_BRANCH=run-e2e/$USER/$CURR_BRANCH +git push $(git remote -v | grep aws-amplify/amplify-cli | head -n1 | awk '{print $1;}') $CURR_BRANCH:$UPSTREAM_BRANCH --no-verify --force-with-lease +echo Current branch is: $CURR_BRANCH +echo Upstream branch is: $UPSTREAM_BRANCH +echo Authenticating terminal... +mwinit +echo Loading E2E account credentials... +ada cred update --profile="${PROFILE}" --account="${ACCOUNT}" --role=CodeBuildE2E --provider=isengard --once +echo Submitting Build Request +RESULT=$(aws codebuild start-build-batch --profile="${PROFILE}" --project-name AmplifyCLI-E2E-Testing --source-version=$CURR_BRANCH --query 'buildBatch.id' --output text) +echo "https://us-east-1.console.aws.amazon.com/codesuite/codebuild/$ACCOUNT/projects/AmplifyCLI-E2E-Testing/batch/$RESULT?region=us-east-1" diff --git a/scripts/sample.env b/scripts/sample.env new file mode 100644 index 00000000000..6fe0276386c --- /dev/null +++ b/scripts/sample.env @@ -0,0 +1,4 @@ +E2E_ACCOUNT_BETA= +E2E_ACCOUNT_PROD= +RELEASE_ACCOUNT_BETA= +RELEASE_ACCOUNT_PROD= From 7f5249e09a2de86af1b00b29552349ee6b35cf64 Mon Sep 17 00:00:00 2001 From: awsluja Date: Tue, 16 May 2023 13:27:26 -0400 Subject: [PATCH 4/6] ci: set region if not defined --- scripts/cloud-e2e-beta.sh | 1 + scripts/cloud-e2e.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/scripts/cloud-e2e-beta.sh b/scripts/cloud-e2e-beta.sh index 40635ee02d7..dff4d8de312 100644 --- a/scripts/cloud-e2e-beta.sh +++ b/scripts/cloud-e2e-beta.sh @@ -13,6 +13,7 @@ CURR_BRANCH=$(git branch --show-current) echo Current branch is: $CURR_BRANCH mwinit ada cred update --profile="${PROFILE}" --account="${ACCOUNT}" --role=CodeBuildE2E --provider=isengard --once +aws configure set region us-east-1 --profile $PROFILE echo credentials loaded RESULT=$(aws codebuild start-build-batch --profile="${PROFILE}" --project-name AmplifyCLI-E2E-Testing --source-version=$UPSTREAM_BRANCH --query 'buildBatch.id' --output text) echo "https://us-east-1.console.aws.amazon.com/codesuite/codebuild/$ACCOUNT/projects/AmplifyCLI-E2E-Testing/batch/$RESULT?region=us-east-1" diff --git a/scripts/cloud-e2e.sh b/scripts/cloud-e2e.sh index b9d9e19f65a..eb745931edb 100644 --- a/scripts/cloud-e2e.sh +++ b/scripts/cloud-e2e.sh @@ -12,6 +12,7 @@ echo Submitting CodeBuild Request to AWS Account: $ACCOUNT CURR_BRANCH=$(git branch --show-current) UPSTREAM_BRANCH=run-e2e/$USER/$CURR_BRANCH git push $(git remote -v | grep aws-amplify/amplify-cli | head -n1 | awk '{print $1;}') $CURR_BRANCH:$UPSTREAM_BRANCH --no-verify --force-with-lease +aws configure set region us-east-1 --profile $PROFILE echo Current branch is: $CURR_BRANCH echo Upstream branch is: $UPSTREAM_BRANCH echo Authenticating terminal... From 3fe4345df2de2990cd8252ee9f0bdf3da73c2224 Mon Sep 17 00:00:00 2001 From: awsluja Date: Tue, 16 May 2023 13:36:41 -0400 Subject: [PATCH 5/6] ci: cleanup --- scripts/cloud-e2e-beta.sh | 2 +- scripts/cloud-e2e.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/cloud-e2e-beta.sh b/scripts/cloud-e2e-beta.sh index dff4d8de312..7bdf92838d2 100644 --- a/scripts/cloud-e2e-beta.sh +++ b/scripts/cloud-e2e-beta.sh @@ -15,5 +15,5 @@ mwinit ada cred update --profile="${PROFILE}" --account="${ACCOUNT}" --role=CodeBuildE2E --provider=isengard --once aws configure set region us-east-1 --profile $PROFILE echo credentials loaded -RESULT=$(aws codebuild start-build-batch --profile="${PROFILE}" --project-name AmplifyCLI-E2E-Testing --source-version=$UPSTREAM_BRANCH --query 'buildBatch.id' --output text) +RESULT=$(aws codebuild start-build-batch --profile="${PROFILE}" --project-name AmplifyCLI-E2E-Testing --source-version=$CURR_BRANCH --query 'buildBatch.id' --output text) echo "https://us-east-1.console.aws.amazon.com/codesuite/codebuild/$ACCOUNT/projects/AmplifyCLI-E2E-Testing/batch/$RESULT?region=us-east-1" diff --git a/scripts/cloud-e2e.sh b/scripts/cloud-e2e.sh index eb745931edb..42bada5613a 100644 --- a/scripts/cloud-e2e.sh +++ b/scripts/cloud-e2e.sh @@ -20,5 +20,5 @@ mwinit echo Loading E2E account credentials... ada cred update --profile="${PROFILE}" --account="${ACCOUNT}" --role=CodeBuildE2E --provider=isengard --once echo Submitting Build Request -RESULT=$(aws codebuild start-build-batch --profile="${PROFILE}" --project-name AmplifyCLI-E2E-Testing --source-version=$CURR_BRANCH --query 'buildBatch.id' --output text) +RESULT=$(aws codebuild start-build-batch --profile="${PROFILE}" --project-name AmplifyCLI-E2E-Testing --source-version=$UPSTREAM_BRANCH --query 'buildBatch.id' --output text) echo "https://us-east-1.console.aws.amazon.com/codesuite/codebuild/$ACCOUNT/projects/AmplifyCLI-E2E-Testing/batch/$RESULT?region=us-east-1" From 3c2e0dfe3e3d2548a0e9d999c969636e3be01564 Mon Sep 17 00:00:00 2001 From: awsluja Date: Tue, 16 May 2023 18:55:00 -0400 Subject: [PATCH 6/6] ci: refactor --- package.json | 4 ++-- scripts/cloud-e2e-beta.sh | 19 --------------- scripts/cloud-e2e.sh | 49 +++++++++++++++++++++++++-------------- 3 files changed, 34 insertions(+), 38 deletions(-) delete mode 100644 scripts/cloud-e2e-beta.sh diff --git a/package.json b/package.json index ad873462ae8..2cd39e0911a 100644 --- a/package.json +++ b/package.json @@ -11,8 +11,8 @@ "check-version-conflicts:beta": "yarn ts-node ./scripts/check-version.ts beta", "check-version-conflicts:release": "yarn ts-node ./scripts/check-version.ts", "clean": "nx reset && nx run-many --target=clean --all && yarn rimraf tsconfig.tsbuildinfo", - "cloud-e2e": "source ./scripts/cloud-e2e.sh", - "cloud-e2e-beta": "source ./scripts/cloud-e2e-beta.sh", + "cloud-e2e": "source ./scripts/cloud-e2e.sh && cloudE2E", + "cloud-e2e-beta": "source ./scripts/cloud-e2e.sh && cloudE2EBeta", "commit": "git-cz", "coverage:collect": "ts-node ./scripts/collect-test-coverage.ts", "coverage": "codecov || exit 0", diff --git a/scripts/cloud-e2e-beta.sh b/scripts/cloud-e2e-beta.sh deleted file mode 100644 index 7bdf92838d2..00000000000 --- a/scripts/cloud-e2e-beta.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# set exit on error to true -set -e - -set -o allexport -source ./scripts/.env set -PROFILE=AmplifyCLIE2EBeta - -ACCOUNT=$E2E_ACCOUNT_BETA -echo Submitting CodeBuild Request to AWS Account: $ACCOUNT -CURR_BRANCH=$(git branch --show-current) -echo Current branch is: $CURR_BRANCH -mwinit -ada cred update --profile="${PROFILE}" --account="${ACCOUNT}" --role=CodeBuildE2E --provider=isengard --once -aws configure set region us-east-1 --profile $PROFILE -echo credentials loaded -RESULT=$(aws codebuild start-build-batch --profile="${PROFILE}" --project-name AmplifyCLI-E2E-Testing --source-version=$CURR_BRANCH --query 'buildBatch.id' --output text) -echo "https://us-east-1.console.aws.amazon.com/codesuite/codebuild/$ACCOUNT/projects/AmplifyCLI-E2E-Testing/batch/$RESULT?region=us-east-1" diff --git a/scripts/cloud-e2e.sh b/scripts/cloud-e2e.sh index 42bada5613a..6068e914b2d 100644 --- a/scripts/cloud-e2e.sh +++ b/scripts/cloud-e2e.sh @@ -5,20 +5,35 @@ set -e # load .env set -o allexport source ./scripts/.env set -PROFILE=AmplifyCLIE2E - -ACCOUNT=$E2E_ACCOUNT_PROD -echo Submitting CodeBuild Request to AWS Account: $ACCOUNT -CURR_BRANCH=$(git branch --show-current) -UPSTREAM_BRANCH=run-e2e/$USER/$CURR_BRANCH -git push $(git remote -v | grep aws-amplify/amplify-cli | head -n1 | awk '{print $1;}') $CURR_BRANCH:$UPSTREAM_BRANCH --no-verify --force-with-lease -aws configure set region us-east-1 --profile $PROFILE -echo Current branch is: $CURR_BRANCH -echo Upstream branch is: $UPSTREAM_BRANCH -echo Authenticating terminal... -mwinit -echo Loading E2E account credentials... -ada cred update --profile="${PROFILE}" --account="${ACCOUNT}" --role=CodeBuildE2E --provider=isengard --once -echo Submitting Build Request -RESULT=$(aws codebuild start-build-batch --profile="${PROFILE}" --project-name AmplifyCLI-E2E-Testing --source-version=$UPSTREAM_BRANCH --query 'buildBatch.id' --output text) -echo "https://us-east-1.console.aws.amazon.com/codesuite/codebuild/$ACCOUNT/projects/AmplifyCLI-E2E-Testing/batch/$RESULT?region=us-east-1" +export CURR_BRANCH=$(git branch --show-current) +function authenticate { + echo Authenticating terminal... + mwinit + echo Loading E2E account credentials... + ada cred update --profile="${CLOUD_E2E_PROFILE}" --account="${CLOUD_E2E_ACCOUNT}" --role=CodeBuildE2E --provider=isengard --once + aws configure set region us-east-1 --profile $CLOUD_E2E_PROFILE +} +function triggerBuild { + echo Submitting CodeBuild Request to AWS Account: $CLOUD_E2E_ACCOUNT + echo Current branch is: $CURR_BRANCH + echo E2E Target branch is: $TARGET_BRANCH + RESULT=$(aws codebuild start-build-batch --profile="${CLOUD_E2E_PROFILE}" --project-name AmplifyCLI-E2E-Testing --source-version=$TARGET_BRANCH --query 'buildBatch.id' --output text) + echo "https://us-east-1.console.aws.amazon.com/codesuite/codebuild/$CLOUD_E2E_ACCOUNT/projects/AmplifyCLI-E2E-Testing/batch/$RESULT?region=us-east-1" +} +function cloudE2EBeta { + echo Running Beta E2E Test Suite + export CLOUD_E2E_PROFILE=AmplifyCLIE2EBeta + export CLOUD_E2E_ACCOUNT=$E2E_ACCOUNT_BETA + export TARGET_BRANCH=$CURR_BRANCH + authenticate + triggerBuild +} +function cloudE2E { + echo Running Prod E2E Test Suite + export CLOUD_E2E_PROFILE=AmplifyCLIE2E + export CLOUD_E2E_ACCOUNT=$E2E_ACCOUNT_PROD + export TARGET_BRANCH=run-e2e/$USER/$CURR_BRANCH + git push $(git remote -v | grep aws-amplify/amplify-cli | head -n1 | awk '{print $1;}') $CURR_BRANCH:$TARGET_BRANCH --no-verify --force-with-lease + authenticate + triggerBuild +}