diff --git a/.github/workflows/build-connector-command.yml b/.github/workflows/build-connector-command.yml new file mode 100644 index 000000000000..ff39163501be --- /dev/null +++ b/.github/workflows/build-connector-command.yml @@ -0,0 +1,260 @@ +name: Bump, Build, Test Connectors [EXPERIMENTAL] +on: + workflow_dispatch: + inputs: + repo: + description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos." + required: false + default: "airbytehq/airbyte" + gitref: + description: "The git ref to check out from the specified repository." + required: false + default: master + connector: + description: "Airbyte Connector" + required: true + bump-version: + description: "Set to major, minor, or patch to automatically bump connectors version in Dockerfile, definitions.yaml and generate seed spec. You can also do this manually" + required: false + default: "false" + run-tests: + description: "Should run tests" + required: false + default: "true" + comment-id: + description: "The comment-id of the slash command. Used to update the comment with the status." + required: false + +jobs: + find_valid_pat: + name: "Find a PAT with room for actions" + timeout-minutes: 10 + runs-on: ubuntu-latest + outputs: + pat: ${{ steps.variables.outputs.pat }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + - name: Check PAT rate limits + id: variables + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} + ## Gradle Build + # In case of self-hosted EC2 errors, remove this block. + start-bump-build-test-connector-runner: + name: Start Build EC2 Runner + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + # 80 gb disk + ec2-image-id: ami-0d648081937c75a73 + bump-build-test-connector: + name: Bump, Build, Test Connector + needs: start-bump-build-test-connector-runner + runs-on: ${{ needs.start-bump-build-test-connector-runner.outputs.label }} + environment: more-secrets + steps: + ############################ + ## SET UP ## + ############################ + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} + export_default_credentials: true + - name: Search for valid connector name format + id: regex + uses: AsasInnab/regex-action@v1 + with: + regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" + regex_flags: "i" # required to be set for this plugin + search_string: ${{ github.event.inputs.connector }} + - name: Validate input workflow format + if: steps.regex.outputs.first_match != github.event.inputs.connector + run: echo "The connector provided has an invalid format!" && exit 1 + - name: Link comment to workflow run + if: github.event.inputs.comment-id + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + token: ${{ secrets.OCTAVIA_PAT }} + - name: Install Java + uses: actions/setup-java@v1 + with: + java-version: "17" + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: "3.9" + - name: Install Pyenv and Tox + run: | + python3 -m pip install --quiet virtualenv==16.7.9 --user + python3 -m virtualenv venv + source venv/bin/activate + pip install --quiet tox==3.24.4 + - name: Install yq + if: github.event.inputs.bump-version != 'false' && success() + run: | + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 + sudo add-apt-repository ppa:rmescandon/yq + sudo apt update + sudo apt install yq -y + - name: Test and install CI scripts + # all CI python packages have the prefix "ci_" + run: | + source venv/bin/activate + tox -r -c ./tools/tox_ci.ini + pip install --quiet -e ./tools/ci_* + - name: Get Credentials for ${{ github.event.inputs.connector }} + run: | + source venv/bin/activate + ci_credentials ${{ github.event.inputs.connector }} + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + # TODO: seems like this should run in post-merge workflow + # - name: Prepare Sentry + # if: startsWith(github.event.inputs.connector, 'connectors') + # run: | + # curl -sL https://sentry.io/get-cli/ | bash + # - name: Create Sentry Release + # if: startsWith(github.event.inputs.connector, 'connectors') + # run: | + # sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing + # env: + # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + # SENTRY_ORG: airbyte-5j + # SENTRY_PROJECT: airbyte-connectors + ############################ + ## BUMP ## + ############################ + - name: Bump Connector Version + if: github.event.inputs.bump-version != 'false' && success() + run: ./tools/integrations/manage.sh bump_version airbyte-integrations/${{ github.event.inputs.connector }} + - name: Commit and Push Version Bump + if: github.event.inputs.bump-version != 'false' && success() + run: | + git config user.name 'Octavia Squidington III' + git config user.email 'octavia-squidington-iii@users.noreply.github.com' + git add -u + git commit -m "bump-version ${{github.event.inputs.connector}}" + git push origin ${{ github.event.inputs.gitref }} + - name: Add Version Bump Success Comment + if: github.event.inputs.comment-id && github.event.inputs.bump-version != 'false' && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :rocket: Bumped version for ${{github.event.inputs.connector}} + - name: Add Version Bump Failure Comment + if: github.event.inputs.comment-id && github.event.inputs.bump-version != 'false' && !success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :x: Couldn't bump version for ${{github.event.inputs.connector}} + ############################ + ## BUILD AND TEST ## + ############################ + - name: Build ${{ github.event.inputs.connector }} + run: ./tools/integrations/manage.sh build_experiment airbyte-integrations/${{ github.event.inputs.connector }} + id: build + env: + PR_NUMBER: ${{ github.event.number }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. + TZ: UTC + # - name: Test ${{ github.event.inputs.connector }} + # if: github.event.inputs.run-tests == 'true' + # run: ./tools/integrations/manage.sh test airbyte-integrations/${{ github.event.inputs.connector }} + # - name: Finalize Sentry release + # if: startsWith(github.event.inputs.connector, 'connectors') + # run: | + # sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" + # env: + # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + # SENTRY_ORG: airbyte-5j + # SENTRY_PROJECT: airbyte-connectors + # - name: Build and Test Success Comment + # if: github.event.inputs.comment-id && success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :rocket: Successfully built and tested ${{github.event.inputs.connector}} + # - name: Build and Test Failure Comment + # if: github.event.inputs.comment-id && !success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :x: Failed to build and test ${{github.event.inputs.connector}} + # - name: Slack Notification - Failure + # if: failure() + # uses: rtCamp/action-slack-notify@master + # env: + # SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }} + # SLACK_USERNAME: Buildozer + # SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg + # SLACK_COLOR: DC143C + # SLACK_TITLE: "Failed to build and test connector ${{ github.event.inputs.connector }} from branch ${{ github.ref }}" + # SLACK_FOOTER: "" + # - name: Add Final Success Comment + # if: github.event.inputs.comment-id && success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + # - name: Set publish label + # if: success() + # run: | + # echo "set some label on PR" + # In case of self-hosted EC2 errors, remove this block. + stop-bump-build-test-connector-runner: + name: Stop Build EC2 Runner + needs: + - start-bump-build-test-connector-runner # required to get output from the start-runner job + - bump-build-test-connector # required to wait when the main job is done + - find_valid_pat + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Stop EC2 runner + uses: supertopher/ec2-github-runner@base64v1.0.10 + with: + mode: stop + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ needs.start-bump-build-test-connector-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-bump-build-test-connector-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/publish-connector-command.yml b/.github/workflows/publish-connector-command.yml new file mode 100644 index 000000000000..4aac116c8dbf --- /dev/null +++ b/.github/workflows/publish-connector-command.yml @@ -0,0 +1,210 @@ +name: Publish Connector [EXPERIMENTAL] +on: + workflow_dispatch: + inputs: + repo: + description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos." + required: false + default: "airbytehq/airbyte" + gitref: + description: "The git ref to check out from the specified repository." + required: false + default: master + connector: + description: "Airbyte Connector" + required: true + bump-version: + description: "Set to major, minor, or patch to automatically bump connectors version in Dockerfile, definitions.yaml and generate seed spec. You can also do this manually" + required: false + default: "false" + run-tests: + description: "Should run tests" + required: false + default: "true" + comment-id: + description: "The comment-id of the slash command. Used to update the comment with the status." + required: false + +jobs: + find_valid_pat: + name: "Find a PAT with room for actions" + timeout-minutes: 10 + runs-on: ubuntu-latest + outputs: + pat: ${{ steps.variables.outputs.pat }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + - name: Check PAT rate limits + id: variables + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} + ## Gradle Build + # In case of self-hosted EC2 errors, remove this block. + +# start-bump-build-test-connector-runner: +# name: Start Build EC2 Runner +# runs-on: ubuntu-latest +# needs: find_valid_pat +# outputs: +# label: ${{ steps.start-ec2-runner.outputs.label }} +# ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} +# steps: +# - name: Checkout Airbyte +# uses: actions/checkout@v2 +# with: +# repository: ${{ github.event.inputs.repo }} +# ref: ${{ github.event.inputs.gitref }} +# - name: Start AWS Runner +# id: start-ec2-runner +# uses: ./.github/actions/start-aws-runner +# with: +# aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} +# github-token: ${{ needs.find_valid_pat.outputs.pat }} +# # 80 gb disk +# ec2-image-id: ami-0d648081937c75a73 +# bump-build-test-connector: +# needs: start-bump-build-test-connector-runner +# runs-on: ${{ needs.start-bump-build-test-connector-runner.outputs.label }} +# environment: more-secrets +# steps: +# ############################ +# ## SET UP ## +# ############################ +# - name: Set up Cloud SDK +# uses: google-github-actions/setup-gcloud@v0 +# with: +# service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} +# export_default_credentials: true +# - name: Search for valid connector name format +# id: regex +# uses: AsasInnab/regex-action@v1 +# with: +# regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" +# regex_flags: "i" # required to be set for this plugin +# search_string: ${{ github.event.inputs.connector }} +# - name: Validate input workflow format +# if: steps.regex.outputs.first_match != github.event.inputs.connector +# run: echo "The connector provided has an invalid format!" && exit 1 +# - name: Link comment to workflow run +# if: github.event.inputs.comment-id +# uses: peter-evans/create-or-update-comment@v1 +# with: +# comment-id: ${{ github.event.inputs.comment-id }} +# body: | +# > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} +# - name: Checkout Airbyte +# uses: actions/checkout@v2 +# with: +# repository: ${{ github.event.inputs.repo }} +# ref: ${{ github.event.inputs.gitref }} +# token: ${{ secrets.OCTAVIA_PAT }} +# - name: Install Java +# uses: actions/setup-java@v1 +# with: +# java-version: "17" +# - name: Install Python +# uses: actions/setup-python@v2 +# with: +# python-version: "3.9" +# - name: Install Pyenv and Tox +# run: | +# python3 -m pip install --quiet virtualenv==16.7.9 --user +# python3 -m virtualenv venv +# source venv/bin/activate +# pip install --quiet tox==3.24.4 +# - name: Install yq +# if: github.event.inputs.bump-version != 'false' && success() +# run: | +# sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 +# sudo add-apt-repository ppa:rmescandon/yq +# sudo apt update +# sudo apt install yq -y +# - name: Test and install CI scripts +# # all CI python packages have the prefix "ci_" +# run: | +# source venv/bin/activate +# tox -r -c ./tools/tox_ci.ini +# pip install --quiet -e ./tools/ci_* +# - name: Get Credentials for ${{ github.event.inputs.connector }} +# run: | +# source venv/bin/activate +# ci_credentials ${{ github.event.inputs.connector }} +# env: +# GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} +# # TODO: seems like this should run in post-merge workflow +# # - name: Prepare Sentry +# # if: startsWith(github.event.inputs.connector, 'connectors') +# # run: | +# # curl -sL https://sentry.io/get-cli/ | bash +# # - name: Create Sentry Release +# # if: startsWith(github.event.inputs.connector, 'connectors') +# # run: | +# # sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing +# # env: +# # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} +# # SENTRY_ORG: airbyte-5j +# # SENTRY_PROJECT: airbyte-connectors +# # - name: Build and Test Success Comment +# # if: github.event.inputs.comment-id && success() +# # uses: peter-evans/create-or-update-comment@v1 +# # with: +# # comment-id: ${{ github.event.inputs.comment-id }} +# # body: | +# # > :rocket: Successfully built and tested ${{github.event.inputs.connector}} +# # - name: Build and Test Failure Comment +# # if: github.event.inputs.comment-id && !success() +# # uses: peter-evans/create-or-update-comment@v1 +# # with: +# # comment-id: ${{ github.event.inputs.comment-id }} +# # body: | +# # > :x: Failed to build and test ${{github.event.inputs.connector}} +# # - name: Slack Notification - Failure +# # if: failure() +# # uses: rtCamp/action-slack-notify@master +# # env: +# # SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }} +# # SLACK_USERNAME: Buildozer +# # SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg +# # SLACK_COLOR: DC143C +# # SLACK_TITLE: "Failed to build and test connector ${{ github.event.inputs.connector }} from branch ${{ github.ref }}" +# # SLACK_FOOTER: "" +# # - name: Add Final Success Comment +# # if: github.event.inputs.comment-id && success() +# # uses: peter-evans/create-or-update-comment@v1 +# # with: +# # comment-id: ${{ github.event.inputs.comment-id }} +# # body: | +# # > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} +# # - name: Set publish label +# # if: success() +# # run: | +# # echo "set some label on PR" +# # In case of self-hosted EC2 errors, remove this block. +# stop-bump-build-test-connector-runner: +# name: Stop Build EC2 Runner +# needs: +# - start-bump-build-test-connector-runner # required to get output from the start-runner job +# - bump-build-test-connector # required to wait when the main job is done +# - find_valid_pat +# runs-on: ubuntu-latest +# if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs +# steps: +# - name: Configure AWS credentials +# uses: aws-actions/configure-aws-credentials@v1 +# with: +# aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} +# aws-region: us-east-2 +# - name: Stop EC2 runner +# uses: supertopher/ec2-github-runner@base64v1.0.10 +# with: +# mode: stop +# github-token: ${{ needs.find_valid_pat.outputs.pat }} +# label: ${{ needs.start-bump-build-test-connector-runner.outputs.label }} +# ec2-instance-id: ${{ needs.start-bump-build-test-connector-runner.outputs.ec2-instance-id }} diff --git a/tools/integrations/manage.sh b/tools/integrations/manage.sh index fdb40bb8b3b2..719bfd7ef78d 100755 --- a/tools/integrations/manage.sh +++ b/tools/integrations/manage.sh @@ -10,6 +10,7 @@ Usage: $(basename "$0") For publish, if you want to push the spec to the spec cache, provide a path to a service account key file that can write to the cache. Available commands: scaffold + test build [] publish [] [--publish_spec_to_cache] [--publish_spec_to_cache_with_key_file ] publish_external @@ -51,6 +52,117 @@ cmd_build() { fi } +# Experimental version of the above for a new way to build/tag images +cmd_build_experiment() { + local path=$1; shift || error "Missing target (root path of integration) $USAGE" + [ -d "$path" ] || error "Path must be the root path of the integration" + + echo "Building $path" + ./gradlew --no-daemon "$(_to_gradle_path "$path" clean)" + ./gradlew --no-daemon "$(_to_gradle_path "$path" build)" + + # After this happens this image should exist: "image_name:dev" + # Re-tag with CI candidate label + local image_name; image_name=$(_get_docker_image_name "$path/Dockerfile") + local image_version; image_version=$(_get_docker_image_version "$path/Dockerfile") + local image_candidate_tag; image_candidate_tag="$image_version-candidate-$PR_NUMBER" + + # If running via the bump-build-test-connector job, re-tag gradle built image following candidate image pattern + if [[ "$GITHUB_JOB" == "bump-build-test-connector" ]]; then + docker tag "$image_name:dev" "$image_name:$image_candidate_tag" + # TODO: docker push "$image_name:$image_candidate_tag" + fi +} + +cmd_test() { + local path=$1; shift || error "Missing target (root path of integration) $USAGE" + [ -d "$path" ] || error "Path must be the root path of the integration" + + # TODO: needs to know to use alternate image tag from cmd_build_experiment + echo "Running integration tests..." + ./gradlew --no-daemon "$(_to_gradle_path "$path" integrationTest)" +} + +# Bumps connector version in Dockerfile, definitions.yaml file, and updates seeds with gradle. +# This does not build or test, it solely manages the versions of connectors to be +1'd. +# +# NOTE: this does NOT update changelogs because the changelog markdown files do not have a reliable machine-readable +# format to automatically handle this. Someday it could though: https://github.com/airbytehq/airbyte/issues/12031 +cmd_bump_version() { + # Take params + local connector_path + local bump_version + connector_path="$1" # Should look like airbyte-integrations/connectors/source-X + bump_version="$2" || bump_version="patch" + + # Set local constants + connector=${connector_path#airbyte-integrations/connectors/} + if [[ "$connector" =~ "source-" ]]; then + connector_type="source" + elif [[ "$connector" =~ "destination-" ]]; then + connector_type="destination" + else + echo "Invalid connector_type from $connector" + exit 1 + fi + definitions_path="./airbyte-config/init/src/main/resources/seed/${connector_type}_definitions.yaml" + dockerfile="$connector_path/Dockerfile" + master_dockerfile="/tmp/master_${connector}_dockerfile" + # This allows getting the contents of a file without checking it out + git --no-pager show "origin/master:$dockerfile" > "$master_dockerfile" + + # Current version always comes from master, this way we can always bump correctly relative to master + # verses a potentially stale local branch + current_version=$(_get_docker_image_version "$master_dockerfile") + local image_name; image_name=$(_get_docker_image_name "$dockerfile") + rm "$master_dockerfile" + + ## Create bumped version + IFS=. read -r major_version minor_version patch_version <<<"${current_version##*-}" + case "$bump_version" in + "major") + ((major_version++)) + minor_version=0 + patch_version=0 + ;; + "minor") + ((minor_version++)) + patch_version=0 + ;; + "patch") + ((patch_version++)) + ;; + *) + echo "Invalid bump_version option: $bump_version. Valid options are major, minor, patch" + exit 1 + esac + + bumped_version="$major_version.$minor_version.$patch_version" + # This image should not already exist, if it does, something weird happened + _error_if_tag_exists "$image_name:$bumped_version" + echo "$connector:$current_version will be bumped to $connector:$bumped_version" + + ## Write new version to files + # 1) Dockerfile + sed -i "s/$current_version/$bumped_version/g" "$dockerfile" + + # 2) Definitions YAML file + definitions_check=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitions_path") + + if [[ (-z "$definitions_check") ]]; then + echo "Could not find $connector in $definitions_path, exiting 1" + exit 1 + fi + + connector_name=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitions_path") + yq e "(.[] | select(.name == \"$connector_name\").dockerImageTag)|=\"$bumped_version\"" -i "$definitions_path" + + # 3) Seed files + ./gradlew :airbyte-config:init:processResources + + echo "Woohoo! Successfully bumped $connector:$current_version to $connector:$bumped_version" +} + cmd_publish() { local path=$1; shift || error "Missing target (root path of integration) $USAGE" [ -d "$path" ] || error "Path must be the root path of the integration" @@ -149,7 +261,7 @@ cmd_publish() { echo "Using environment gcloud" fi - gsutil cp "$tmp_spec_file" gs://io-airbyte-cloud-spec-cache/specs/"$image_name"/"$image_version"/spec.json + gsutil cp "$tmp_spec_file" "gs://io-airbyte-cloud-spec-cache/specs/$image_name/$image_version/spec.json" else echo "Publishing without writing to spec cache." fi @@ -176,7 +288,7 @@ cmd_publish_external() { echo "Using environment gcloud" - gsutil cp "$tmp_spec_file" gs://io-airbyte-cloud-spec-cache/specs/"$image_name"/"$image_version"/spec.json + gsutil cp "$tmp_spec_file" "gs://io-airbyte-cloud-spec-cache/specs/$image_name/$image_version/spec.json" } main() {