Skip to content

Commit

Permalink
whoops
Browse files Browse the repository at this point in the history
  • Loading branch information
@noahkawasaki-airbyte
noahkawasaki-airbyte committed May 27, 2022
1 parent 514fad9 commit 3bff0cf
Showing 1 changed file with 67 additions and 73 deletions.
140 changes: 67 additions & 73 deletions .github/workflows/publish-command.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,10 @@ on:
comment-id:
description: "The comment-id of the slash command. Used to update the comment with the status."
required: false
bump-version:
description: "Set to major, minor, or patch to automatically bump connectors version in Dockerfile, definitions.yaml and generate seed spec. You can also do this manually"
required: false
default: "patch"
options:
- "major"
- "minor"
- "patch"
force:
description: "Force entire build and bump pipeline to execute, even if bump as already happened"
default: "false"
required: false
auto-bump-version:
description: "after publishing, the workflow will automatically bump the connector version in definitions and generate seed spec"
required: true
default: "true"

jobs:
find_valid_pat:
Expand Down Expand Up @@ -73,8 +65,6 @@ jobs:
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
github-token: ${{ needs.find_valid_pat.outputs.pat }}
# 80 gb disk
ec2-image-id: ami-0d648081937c75a73
publish-image:
timeout-minutes: 240
needs: start-publish-image-runner
Expand Down Expand Up @@ -124,6 +114,7 @@ jobs:
source venv/bin/activate
pip install --quiet tox==3.24.4
- name: Install yq
if: github.event.inputs.auto-bump-version == 'true' && success()
run: |
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
sudo add-apt-repository ppa:rmescandon/yq
Expand All @@ -135,93 +126,45 @@ jobs:
source venv/bin/activate
tox -r -c ./tools/tox_ci.ini
pip install --quiet -e ./tools/ci_*
# Needed for Sentry steps
- name: Set Connector Name Env Variable
if: startsWith(github.event.inputs.connector, 'connectors')
run: |
echo "CONNECTOR_IMAGE_NAME=$(echo ${{ github.event.inputs.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV
- name: Write Integration Test Credentials for ${{ github.event.inputs.connector }}
run: |
source venv/bin/activate
ci_credentials ${{ github.event.inputs.connector }}
env:
GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
- name: Bump ${{ github.event.inputs.connector }} Version
id: bump-version
env:
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
FORCE_BUMP: ${{ github.event.inputs.force }}
run: |
./tools/integrations/manage.sh bump_version airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.bump-version }}
- name: Add Version Bump Success Comment
if: github.event.inputs.comment-id && success() && steps.bump-version.outputs.bumped == 'true'
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ github.event.inputs.comment-id }}
body: |
> :rocket: Auto-bumped version for ${{github.event.inputs.connector}}
- name: Add Version Bump Failure Comment
if: github.event.inputs.comment-id && !success() && steps.bump-version.outputs.bumped == 'true'
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ github.event.inputs.comment-id }}
body: |
> :x: Couldn't auto-bump version for ${{github.event.inputs.connector}}
- name: Build ${{ github.event.inputs.connector }}
run: |
./tools/integrations/manage.sh build airbyte-integrations/${{ github.event.inputs.connector }}
id: build
env:
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
# Oracle expects this variable to be set. Although usually present, this is not set by default on GitHub virtual runners.
TZ: UTC
- name: Test ${{ github.event.inputs.connector }}
if: github.event.inputs.run-tests == 'true'
- name: Set Name and Version Environment Vars
if: startsWith(github.event.inputs.connector, 'connectors')
run: |
./tools/integrations/manage.sh test airbyte-integrations/${{ github.event.inputs.connector }}
source tools/lib/lib.sh
DOCKERFILE=airbyte-integrations/${{ github.event.inputs.connector }}/Dockerfile
echo "IMAGE_NAME=$(echo ${{ github.event.inputs.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV
echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV
- name: Prepare Sentry
if: startsWith(github.event.inputs.connector, 'connectors')
run: |
curl -sL https://sentry.io/get-cli/ | bash
- name: Create Sentry Release
if: startsWith(github.event.inputs.connector, 'connectors') && steps.bump-version.outputs.bumped == 'true'
if: startsWith(github.event.inputs.connector, 'connectors')
run: |
sentry-cli releases set-commits "${{ env.CONNECTOR_IMAGE_NAME }}@${{ steps.bump-version.outputs.bumped_version }}" --auto --ignore-missing
sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing
env:
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }}
SENTRY_ORG: airbyte-5j
SENTRY_PROJECT: airbyte-connectors
- name: Publish ${{ github.event.inputs.connector }}
run: |
echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD}
./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} --publish_spec_to_cache
./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache
id: publish
env:
DOCKER_USERNAME: airbytebot
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
# Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
TZ: UTC
- name: Process Build ${{ github.event.inputs.connector }}
run: |
export publish_spec_to_cache=true
./tools/integrations/manage.sh process_build airbyte-integrations/${{ github.event.inputs.connector }}
env:
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
- name: Commit and Push Bump ${{ github.event.inputs.connector }}
if: success() && steps.bump-version.outputs.bumped == 'true'
run: |
git config user.name 'Octavia Squidington III'
git config user.email '[email protected]'
if ! (git diff --quiet && git diff --staged --quiet); then
echo "Detected changed files and version bump"
git commit -a -m "auto-bump-version ${{github.event.inputs.connector}} ${{ steps.bump-version.outputs.master_version }} to ${{ steps.bump-version.outputs.bumped_version }}"
git pull origin ${{ github.event.inputs.gitref }}
git push origin ${{ github.event.inputs.gitref }}
fi
- name: Finalize Sentry release
if: startsWith(github.event.inputs.connector, 'connectors') && steps.bump-version.outputs.bumped == 'true'
if: startsWith(github.event.inputs.connector, 'connectors')
run: |
sentry-cli releases finalize "${{ env.CONNECTOR_IMAGE_NAME }}@${{ steps.bump-version.outputs.bumped_version }}"
sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}"
env:
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }}
SENTRY_ORG: airbyte-5j
Expand Down Expand Up @@ -250,6 +193,57 @@ jobs:
SLACK_COLOR: DC143C
SLACK_TITLE: "Failed to publish connector ${{ github.event.inputs.connector }} from branch ${{ github.ref }}"
SLACK_FOOTER: ""
- name: Check if connector in definitions yaml
if: github.event.inputs.auto-bump-version == 'true' && success()
run: |
connector="airbyte/${{ env.IMAGE_NAME }}"
definitionpath=./airbyte-config/init/src/main/resources/seed/
sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml)
destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml)
if [[ (-z "$sourcecheck" && -z "$destcheck") ]]
then exit 1
fi
- name: Bump version in definitions yaml
if: github.event.inputs.auto-bump-version == 'true' && success()
run: |
connector="airbyte/${{ env.IMAGE_NAME }}"
definitionpath=./airbyte-config/init/src/main/resources/seed/
sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml)
destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml)
if [ -z "$sourcename" ]
then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml
else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml
fi
- name: Run gradle process changes
if: github.event.inputs.auto-bump-version == 'true' && success()
run: |
./gradlew :airbyte-config:init:processResources
- name: git config
if: github.event.inputs.auto-bump-version == 'true' && success()
run: |
git config user.name 'Octavia Squidington III'
git config user.email '[email protected]'
- name: git commit and push
if: github.event.inputs.auto-bump-version == 'true' && success()
run: |
git add -u
git commit -m "auto-bump connector version"
git pull origin ${{ github.event.inputs.gitref }}
git push origin ${{ github.event.inputs.gitref }}
- name: Add Version Bump Success Comment
if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success()
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ github.event.inputs.comment-id }}
body: |
> :rocket: Auto-bumped version for ${{github.event.inputs.connector}}
- name: Add Version Bump Failure Comment
if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success()
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ github.event.inputs.comment-id }}
body: |
> :x: Couldn't auto-bump version for ${{github.event.inputs.connector}}
- name: Add Final Success Comment
if: github.event.inputs.comment-id && success()
uses: peter-evans/create-or-update-comment@v1
Expand Down

0 comments on commit 3bff0cf

Please sign in to comment.