diff --git a/.github/actions/cancel-workflow-runs b/.github/actions/cancel-workflow-runs new file mode 160000 index 0000000000000..953e057dc81d3 --- /dev/null +++ b/.github/actions/cancel-workflow-runs @@ -0,0 +1 @@ +Subproject commit 953e057dc81d3458935a18d1184c386b0f6b5738 diff --git a/.github/actions/checks-action b/.github/actions/checks-action new file mode 160000 index 0000000000000..9f02872da71b6 --- /dev/null +++ b/.github/actions/checks-action @@ -0,0 +1 @@ +Subproject commit 9f02872da71b6f558c6a6f190f925dde5e4d8798 diff --git a/.github/actions/codecov-action b/.github/actions/codecov-action new file mode 160000 index 0000000000000..1fc7722ded470 --- /dev/null +++ b/.github/actions/codecov-action @@ -0,0 +1 @@ +Subproject commit 1fc7722ded4708880a5aea49f2bfafb9336f0c8d diff --git a/.github/actions/configure-aws-credentials b/.github/actions/configure-aws-credentials new file mode 160000 index 0000000000000..e97d7fbc8e0e5 --- /dev/null +++ b/.github/actions/configure-aws-credentials @@ -0,0 +1 @@ +Subproject commit e97d7fbc8e0e5af69631c13daa0f4b5a8d88165b diff --git a/.github/actions/get-workflow-origin b/.github/actions/get-workflow-origin new file mode 160000 index 0000000000000..588cc14f9f1cd --- /dev/null +++ b/.github/actions/get-workflow-origin @@ -0,0 +1 @@ +Subproject commit 588cc14f9f1cdf1b8be3db816855e96422204fec diff --git a/.github/actions/github-push-action b/.github/actions/github-push-action new file mode 160000 index 0000000000000..40bf560936a80 --- /dev/null +++ b/.github/actions/github-push-action @@ -0,0 +1 @@ +Subproject commit 40bf560936a8022e68a3c00e7d2abefaf01305a6 diff --git a/.github/actions/label-when-approved-action b/.github/actions/label-when-approved-action new file mode 160000 index 0000000000000..4c5190fec5661 --- /dev/null +++ b/.github/actions/label-when-approved-action @@ -0,0 +1 @@ +Subproject commit 4c5190fec5661e98d83f50bbd4ef9ebb48bd1194 diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml index fc3d5a1ee96e1..03b5835b252d1 100644 --- a/.github/workflows/build-images-workflow-run.yml +++ b/.github/workflows/build-images-workflow-run.yml @@ -57,14 +57,19 @@ jobs: cacheDirective: ${{ steps.cache-directive.outputs.docker-cache }} buildImages: ${{ steps.build-images.outputs.buildImages }} steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive - name: "Get information about the original trigger of the run" - uses: apache/airflow-get-workflow-origin@588cc14f9f1cdf1b8be3db816855e96422204fec # v1_3 + uses: ./.github/actions/get-workflow-origin id: source-run-info with: token: ${{ secrets.GITHUB_TOKEN }} sourceRunId: ${{ github.event.workflow_run.id }} - name: "Cancel duplicated 'CI Build' runs" - uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 + uses: ./.github/actions/cancel-workflow-runs with: token: ${{ secrets.GITHUB_TOKEN }} cancelMode: allDuplicates @@ -81,7 +86,7 @@ jobs: # trick ¯\_(ツ)_/¯. We name the build-info job appropriately # and then we try to find and cancel all the jobs with the same Event + Repo + Branch as the # current Event/Repo/Branch combination. - uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 + uses: ./.github/actions/cancel-workflow-runs with: cancelMode: namedJobs token: ${{ secrets.GITHUB_TOKEN }} @@ -97,7 +102,7 @@ jobs: # We also produce list of canceled "CI Build' runs as output, so that we # can cancel all the matching "Build Images" workflow runs in the two following steps. # Yeah. Adding to the complexity ¯\_(ツ)_/¯. - uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 + uses: ./.github/actions/cancel-workflow-runs id: cancel-failed with: token: ${{ secrets.GITHUB_TOKEN }} @@ -130,14 +135,14 @@ jobs: # it to cancel any jobs that have matching names containing Source Run Id: # followed by one of the run ids. Yes I know it's super complex ¯\_(ツ)_/¯. if: env.BUILD_IMAGES == 'true' && steps.cancel-failed.outputs.cancelledRuns != '[]' - uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 + uses: ./.github/actions/cancel-workflow-runs with: cancelMode: namedJobs token: ${{ secrets.GITHUB_TOKEN }} notifyPRCancel: true jobNameRegexps: ${{ steps.extract-cancelled-failed-runs.outputs.matching-regexp }} - name: "Cancel duplicated 'CodeQL' runs" - uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 + uses: ./.github/actions/cancel-workflow-runs id: cancel with: token: ${{ secrets.GITHUB_TOKEN }} @@ -157,7 +162,7 @@ jobs: # trick ¯\_(ツ)_/¯. We name the build-info job appropriately and then we try to match # all the jobs with the same Event + Repo + Branch match and cancel all the duplicates for those # This might cancel own run, so this is the last step in the job - uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 + uses: ./.github/actions/cancel-workflow-runs with: cancelMode: allDuplicatedNamedJobs token: ${{ secrets.GITHUB_TOKEN }} @@ -297,8 +302,19 @@ jobs: else echo "::set-output name=proceed::true" fi + - name: > + Checkout "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" branch to 'main-airflow' folder + to use ci/scripts from there. + uses: actions/checkout@v2 + with: + path: "main-airflow" + ref: "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" + persist-credentials: false + submodules: recursive + if: steps.defaults.outputs.proceed == 'true' - name: Initiate GitHub Checks for Building image - uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798 # v1.1.0 + # Use the submodule from main, not the PR branch + uses: ./main-airflow/.github/actions/checks-action id: build-image-check with: token: ${{ secrets.GITHUB_TOKEN }} @@ -312,15 +328,6 @@ jobs: [Image Build](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details" } if: steps.defaults.outputs.proceed == 'true' - - name: > - Checkout "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" branch to 'main-airflow' folder - to use ci/scripts from there. - uses: actions/checkout@v2 - with: - path: "main-airflow" - ref: "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" - persist-credentials: false - if: steps.defaults.outputs.proceed == 'true' - name: "Setup python" uses: actions/setup-python@v2 with: @@ -474,7 +481,7 @@ jobs: run: ./scripts/ci/images/ci_push_production_images.sh if: steps.defaults.outputs.proceed == 'true' - name: Update GitHub Checks for Building image with status - uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798 # v1.1.0 + uses: ./main-airflow/.github/actions/checks-action if: always() && steps.defaults.outputs.proceed == 'true' with: token: ${{ secrets.GITHUB_TOKEN }} @@ -495,8 +502,13 @@ jobs: if: cancelled() needs: [build-ci-images, build-prod-images] steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive - name: "Canceling the 'CI Build' source workflow in case of failure!" - uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 + uses: ./.github/actions/cancel-workflow-runs with: token: ${{ secrets.GITHUB_TOKEN }} cancelMode: self @@ -510,8 +522,13 @@ jobs: if: failure() needs: [build-ci-images, build-prod-images] steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive - name: "Canceling the 'CI Build' source workflow in case of failure!" - uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 + uses: ./.github/actions/cancel-workflow-runs with: token: ${{ secrets.GITHUB_TOKEN }} cancelMode: self diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a20ed741ab2e..132f702ac3bb9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,15 +100,16 @@ jobs: pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }} pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }} steps: - - name: "Get information about the PR" - uses: apache/airflow-get-workflow-origin@588cc14f9f1cdf1b8be3db816855e96422204fec # v1_3 - id: source-run-info - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v2 with: persist-credentials: false + submodules: recursive + - name: "Get information about the PR" + uses: ./.github/actions/get-workflow-origin + id: source-run-info + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: > Event: ${{ github.event_name }} Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }} @@ -346,6 +347,7 @@ jobs: uses: actions/checkout@v2 with: persist-credentials: false + submodules: recursive - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Build docs" @@ -358,7 +360,7 @@ jobs: path: "./files/documentation" retention-days: 7 - name: Configure AWS credentials - uses: apache/airflow-configure-aws-credentials@e97d7fbc8e0e5af69631c13daa0f4b5a8d88165b + uses: ./.github/actions/configure-aws-credentials if: > github.ref == 'refs/heads/master' && github.repository == 'apache/airflow' && github.event_name == 'push' @@ -823,6 +825,11 @@ jobs: - tests-mysql - tests-quarantined steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive - name: "Download all artifacts from the current build" uses: actions/download-artifact@v2 with: @@ -830,7 +837,7 @@ jobs: - name: "Removes unnecessary artifacts" run: ls ./coverage-files | grep -v coverage | xargs rm -rf - name: "Upload all coverage reports to codecov" - uses: apache/airflow-codecov-action@1fc7722ded4708880a5aea49f2bfafb9336f0c8d # v1.1.1 + uses: ./.github/actions/codecov-action with: directory: "./coverage-files" @@ -1118,6 +1125,7 @@ jobs: uses: actions/checkout@v2 with: persist-credentials: false + submodules: recursive - name: "Set constraints branch name" id: constraints-branch run: ./scripts/ci/constraints/ci_branch_constraints.sh @@ -1134,7 +1142,7 @@ jobs: - name: "Commit changed constraint files for ${{needs.build-info.outputs.pythonVersions}}" run: ./scripts/ci/constraints/ci_commit_constraints.sh - name: "Push changes" - uses: apache/airflow-github-push-action@40bf560936a8022e68a3c00e7d2abefaf01305a6 # v0.6.0 + uses: ./.github/actions/github-push-action with: github_token: ${{ secrets.GITHUB_TOKEN }} branch: ${{ steps.constraints-branch.outputs.branch }} @@ -1160,6 +1168,7 @@ jobs: uses: actions/checkout@v2 with: persist-credentials: false + submodules: recursive - name: "Free space" run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Tag commit" @@ -1168,7 +1177,7 @@ jobs: echo "Tagging ${BRANCH_NAME}" git tag -f nightly-${BRANCH_NAME} HEAD - name: "Push tags" - uses: apache/airflow-github-push-action@40bf560936a8022e68a3c00e7d2abefaf01305a6 # v0.6.0 + uses: ./.github/actions/github-push-action with: github_token: ${{ secrets.GITHUB_TOKEN }} tags: true diff --git a/.github/workflows/label_when_reviewed_workflow_run.yml b/.github/workflows/label_when_reviewed_workflow_run.yml index 4400bae86cd44..1ed50dd95f10e 100644 --- a/.github/workflows/label_when_reviewed_workflow_run.yml +++ b/.github/workflows/label_when_reviewed_workflow_run.yml @@ -29,14 +29,19 @@ jobs: outputs: labelSet: ${{ steps.label-when-reviewed.outputs.labelSet }} steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive - name: "Get information about the original trigger of the run" - uses: apache/airflow-get-workflow-origin@588cc14f9f1cdf1b8be3db816855e96422204fec # v1_3 + uses: ./.github/actions/get-workflow-origin id: source-run-info with: token: ${{ secrets.GITHUB_TOKEN }} sourceRunId: ${{ github.event.workflow_run.id }} - name: Initiate Selective Build check - uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798 # v1.1.0 + uses: ./.github/actions/checks-action id: selective-build-check with: token: ${{ secrets.GITHUB_TOKEN }} @@ -49,10 +54,6 @@ jobs: "Checking selective status of the build in [the run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) "} - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - name: > Event: ${{ steps.source-run-info.outputs.sourceEvent }} Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }} @@ -76,6 +77,7 @@ jobs: uses: actions/checkout@v2 with: persist-credentials: false + submodules: recursive - name: Selective checks id: selective-checks env: @@ -91,7 +93,7 @@ jobs: ./scripts/ci/selective_ci_checks.sh fi - name: "Label when approved by committers for PRs that require full tests" - uses: apache/airflow-label-when-approved@4c5190fec5661e98d83f50bbd4ef9ebb48bd1194 # v1.3 + uses: ./.github/actions/label-when-approved-action id: label-full-test-prs-when-approved-by-commiters if: > steps.selective-checks.outputs.run-tests == 'true' && @@ -107,7 +109,7 @@ jobs: If they don't merge it quickly - please rebase it to the latest master at your convenience, or amend the last commit of the PR, and push it with --force-with-lease. - name: "Initiate GitHub Check forcing rerun of SH ${{ github.event.pull_request.head.sha }}" - uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798 # v1.1.0 + uses: ./.github/actions/checks-action id: full-test-check if: steps.label-full-test-prs-when-approved-by-commiters.outputs.labelSet == 'true' with: @@ -122,7 +124,7 @@ jobs: [the run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) "} - name: "Label when approved by committers for PRs that do not require full tests" - uses: apache/airflow-label-when-approved@4c5190fec5661e98d83f50bbd4ef9ebb48bd1194 # v1.3 + uses: ./.github/actions/label-when-approved-action id: label-simple-test-prs-when-approved-by-commiters if: > steps.selective-checks.outputs.run-tests == 'true' && @@ -139,7 +141,7 @@ jobs: 'full tests needed'. Then you should rebase to the latest master or amend the last commit of the PR, and push it with --force-with-lease. - name: "Label when approved by committers for PRs that do not require tests at all" - uses: apache/airflow-label-when-approved@4c5190fec5661e98d83f50bbd4ef9ebb48bd1194 # v1.3 + uses: ./.github/actions/label-when-approved-action id: label-no-test-prs-when-approved-by-commiters if: steps.selective-checks.outputs.run-tests != 'true' with: @@ -153,7 +155,7 @@ jobs: needed and add the 'full tests needed' label. Then you should rebase it to the latest master or amend the last commit of the PR, and push it with --force-with-lease. - name: Update Selective Build check - uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798 # v1.1.0 + uses: ./.github/actions/checks-action if: always() with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000000..1779dddb76551 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,21 @@ +[submodule ".github/actions/get-workflow-origin"] + path = .github/actions/get-workflow-origin + url = https://github.com/potiuk/get-workflow-origin +[submodule ".github/actions/cancel-workflow-runs"] + path = .github/actions/cancel-workflow-runs + url = https://github.com/potiuk/cancel-workflow-runs +[submodule ".github/actions/checks-action"] + path = .github/actions/checks-action + url = https://github.com/LouisBrunner/checks-action +[submodule ".github/actions/configure-aws-credentials"] + path = .github/actions/configure-aws-credentials + url = https://github.com/aws-actions/configure-aws-credentials +[submodule ".github/actions/codecov-action"] + path = .github/actions/codecov-action + url = https://github.com/codecov/codecov-action +[submodule ".github/actions/github-push-action"] + path = .github/actions/github-push-action + url = https://github.com/ad-m/github-push-action +[submodule ".github/actions/label-when-approved-action"] + path = .github/actions/label-when-approved-action + url = https://github.com/TobKed/label-when-approved-action diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 019ff45522ce5..13f8ac2e0a1bb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: rev: v1.1.9 hooks: - id: forbid-tabs - exclude: ^docs/Makefile$|^clients/gen/go.sh + exclude: ^docs/Makefile$|^clients/gen/go.sh|\.gitmodules$ - id: insert-license name: Add license for all SQL files files: \.sql$