Run "third party" github actions from submodules instead (apache#13514)

Rather than having to mirror all the repos we can instead use git submodules to pull in the third party actions we want to use - with recent(ish) changes in review for submodules on GitHub we still get the same "review/audit" visibility for changes, but this way we don't have to either "pollute" our repo with the actions code, nor do we have to maintain a fork of the third party action. (cherry picked from commit f115983) (cherry picked from commit eddd632)
astronomer · Apr 23, 2021 · cbcf44a · cbcf44a
1 parent 4a5751f
commit cbcf44a
Show file tree

Hide file tree

Showing 12 changed files with 97 additions and 41 deletions.
diff --git a/.github/actions/cancel-workflow-runs b/.github/actions/cancel-workflow-runs
diff --git a/.github/actions/checks-action b/.github/actions/checks-action
diff --git a/.github/actions/codecov-action b/.github/actions/codecov-action
diff --git a/.github/actions/configure-aws-credentials b/.github/actions/configure-aws-credentials
diff --git a/.github/actions/get-workflow-origin b/.github/actions/get-workflow-origin
diff --git a/.github/actions/github-push-action b/.github/actions/github-push-action
diff --git a/.github/actions/label-when-approved-action b/.github/actions/label-when-approved-action
diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml
@@ -57,14 +57,19 @@ jobs:
       cacheDirective: ${{ steps.cache-directive.outputs.docker-cache }}
       buildImages: ${{ steps.build-images.outputs.buildImages }}
     steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          submodules: recursive
       - name: "Get information about the original trigger of the run"
-        uses: apache/airflow-get-workflow-origin@588cc14f9f1cdf1b8be3db816855e96422204fec  # v1_3
+        uses: ./.github/actions/get-workflow-origin
         id: source-run-info
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           sourceRunId: ${{ github.event.workflow_run.id }}
       - name: "Cancel duplicated 'CI Build' runs"
-        uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738  # v4_7
+        uses: ./.github/actions/cancel-workflow-runs
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           cancelMode: allDuplicates
@@ -81,7 +86,7 @@ jobs:
         # trick ¯\_(ツ)_/¯. We name the build-info job appropriately
         # and then we try to find and cancel all the jobs with the same Event + Repo + Branch as the
         # current Event/Repo/Branch combination.
-        uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738  # v4_7
+        uses: ./.github/actions/cancel-workflow-runs
         with:
           cancelMode: namedJobs
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -97,7 +102,7 @@ jobs:
         # We also produce list of canceled "CI Build' runs as output, so that we
         # can cancel all the matching "Build Images" workflow runs in the two following steps.
         # Yeah. Adding to the complexity ¯\_(ツ)_/¯.
-        uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738  # v4_7
+        uses: ./.github/actions/cancel-workflow-runs
         id: cancel-failed
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -130,14 +135,14 @@ jobs:
         # it to cancel any jobs that have matching names containing Source Run Id:
         # followed by one of the run ids. Yes I know it's super complex ¯\_(ツ)_/¯.
         if: env.BUILD_IMAGES == 'true' && steps.cancel-failed.outputs.cancelledRuns != '[]'
-        uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738  # v4_7
+        uses: ./.github/actions/cancel-workflow-runs
         with:
           cancelMode: namedJobs
           token: ${{ secrets.GITHUB_TOKEN }}
           notifyPRCancel: true
           jobNameRegexps: ${{ steps.extract-cancelled-failed-runs.outputs.matching-regexp }}
       - name: "Cancel duplicated 'CodeQL' runs"
-        uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738  # v4_7
+        uses: ./.github/actions/cancel-workflow-runs
         id: cancel
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -157,7 +162,7 @@ jobs:
         # trick ¯\_(ツ)_/¯. We name the build-info job appropriately and then we try to match
         # all the jobs with the same Event + Repo + Branch match and cancel all the duplicates for those
         # This might cancel own run, so this is the last step in the job
-        uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738  # v4_7
+        uses: ./.github/actions/cancel-workflow-runs
         with:
           cancelMode: allDuplicatedNamedJobs
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -296,8 +301,19 @@ jobs:
           else
               echo "::set-output name=proceed::true"
           fi
+      - name: >
+          Checkout "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" branch to 'main-airflow' folder
+          to use ci/scripts from there.
+        uses: actions/checkout@v2
+        with:
+          path: "main-airflow"
+          ref: "${{ needs.cancel-workflow-runs.outputs.targetBranch }}"
+          persist-credentials: false
+          submodules: recursive
+        if: steps.defaults.outputs.proceed == 'true'
       - name: Initiate GitHub Checks for Building image
-        uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798  # v1.1.0
+        # Use the submodule from main, not the PR branch
+        uses: ./main-airflow/.github/actions/checks-action
         id: build-image-check
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -311,15 +327,6 @@ jobs:
             [Image Build](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
             for details" }
         if: steps.defaults.outputs.proceed == 'true'
-      - name: >
-          Checkout "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" branch to 'main-airflow' folder
-          to use ci/scripts from there.
-        uses: actions/checkout@v2
-        with:
-          path: "main-airflow"
-          ref: "${{ needs.cancel-workflow-runs.outputs.targetBranch }}"
-          persist-credentials: false
-        if: steps.defaults.outputs.proceed == 'true'
       - name: "Setup python"
         uses: actions/setup-python@v2
         with:
@@ -473,7 +480,7 @@ jobs:
         run: ./scripts/ci/images/ci_push_production_images.sh
         if: steps.defaults.outputs.proceed == 'true'
       - name: Update GitHub Checks for Building image with status
-        uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798  # v1.1.0
+        uses: ./main-airflow/.github/actions/checks-action
         if: always() && steps.defaults.outputs.proceed == 'true'
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -494,8 +501,13 @@ jobs:
     if: cancelled()
     needs: [build-ci-images, build-prod-images]
     steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          submodules: recursive
       - name: "Canceling the 'CI Build' source workflow in case of failure!"
-        uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738  # v4_7
+        uses: ./.github/actions/cancel-workflow-runs
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           cancelMode: self
@@ -509,8 +521,13 @@ jobs:
     if: failure()
     needs: [build-ci-images, build-prod-images]
     steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          submodules: recursive
       - name: "Canceling the 'CI Build' source workflow in case of failure!"
-        uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738  # v4_7
+        uses: ./.github/actions/cancel-workflow-runs
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           cancelMode: self

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -101,15 +101,16 @@ jobs:
       pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }}
       pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }}
     steps:
-      - name: "Get information about the PR"
-        uses: apache/airflow-get-workflow-origin@588cc14f9f1cdf1b8be3db816855e96422204fec  # v1_3
-        id: source-run-info
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v2
         with:
           persist-credentials: false
+          submodules: recursive
+      - name: "Get information about the PR"
+        uses: ./.github/actions/get-workflow-origin
+        id: source-run-info
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
       - name: >
           Event: ${{ github.event_name }}
           Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
@@ -347,6 +348,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           persist-credentials: false
+          submodules: recursive
       - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
         run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Build docs"
@@ -359,7 +361,7 @@ jobs:
           path: "./files/documentation"
           retention-days: 7
       - name: Configure AWS credentials
-        uses: apache/airflow-configure-aws-credentials@e97d7fbc8e0e5af69631c13daa0f4b5a8d88165b
+        uses: ./.github/actions/configure-aws-credentials
         if: >
           github.ref == 'refs/heads/master' && github.repository == 'apache/airflow' &&
           github.event_name == 'push'
@@ -785,14 +787,19 @@ jobs:
       - tests-mysql
       - tests-quarantined
     steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          submodules: recursive
       - name: "Download all artifacts from the current build"
         uses: actions/download-artifact@v2
         with:
           path: ./coverage-files
       - name: "Removes unnecessary artifacts"
         run: ls ./coverage-files | grep -v coverage | xargs rm -rf
       - name: "Upload all coverage reports to codecov"
-        uses: apache/airflow-codecov-action@1fc7722ded4708880a5aea49f2bfafb9336f0c8d  # v1.1.1
+        uses: ./.github/actions/codecov-action
         with:
           directory: "./coverage-files"
 
@@ -1073,6 +1080,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           persist-credentials: false
+          submodules: recursive
       - name: "Set constraints branch name"
         id: constraints-branch
         run: ./scripts/ci/constraints/ci_branch_constraints.sh
@@ -1089,7 +1097,7 @@ jobs:
       - name: "Commit changed constraint files for ${{needs.build-info.outputs.pythonVersions}}"
         run: ./scripts/ci/constraints/ci_commit_constraints.sh
       - name: "Push changes"
-        uses: apache/airflow-github-push-action@40bf560936a8022e68a3c00e7d2abefaf01305a6  # v0.6.0
+        uses: ./.github/actions/github-push-action
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           branch: ${{ steps.constraints-branch.outputs.branch }}
@@ -1115,6 +1123,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           persist-credentials: false
+          submodules: recursive
       - name: "Free space"
         run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Tag commit"
@@ -1123,7 +1132,7 @@ jobs:
           echo "Tagging ${BRANCH_NAME}"
           git tag -f nightly-${BRANCH_NAME} HEAD
       - name: "Push tags"
-        uses: apache/airflow-github-push-action@40bf560936a8022e68a3c00e7d2abefaf01305a6  # v0.6.0
+        uses: ./.github/actions/github-push-action
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           tags: true

diff --git a/.github/workflows/label_when_reviewed_workflow_run.yml b/.github/workflows/label_when_reviewed_workflow_run.yml
@@ -29,14 +29,19 @@ jobs:
     outputs:
       labelSet: ${{ steps.label-when-reviewed.outputs.labelSet }}
     steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          submodules: recursive
       - name: "Get information about the original trigger of the run"
-        uses: apache/airflow-get-workflow-origin@588cc14f9f1cdf1b8be3db816855e96422204fec  # v1_3
+        uses: ./.github/actions/get-workflow-origin
         id: source-run-info
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           sourceRunId: ${{ github.event.workflow_run.id }}
       - name: Initiate Selective Build check
-        uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798  # v1.1.0
+        uses: ./.github/actions/checks-action
         id: selective-build-check
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -49,10 +54,6 @@ jobs:
             "Checking selective status of the build in
             [the run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
             "}
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v2
-        with:
-          persist-credentials: false
       - name: >
           Event: ${{ steps.source-run-info.outputs.sourceEvent }}
           Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
@@ -76,6 +77,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           persist-credentials: false
+          submodules: recursive
       - name: Selective checks
         id: selective-checks
         env:
@@ -91,7 +93,7 @@ jobs:
             ./scripts/ci/selective_ci_checks.sh
           fi
       - name: "Label when approved by committers for PRs that require full tests"
-        uses: apache/airflow-label-when-approved@4c5190fec5661e98d83f50bbd4ef9ebb48bd1194  # v1.3
+        uses: ./.github/actions/label-when-approved-action
         id: label-full-test-prs-when-approved-by-commiters
         if: >
           steps.selective-checks.outputs.run-tests == 'true' &&
@@ -107,7 +109,7 @@ jobs:
             If they don't merge it quickly - please rebase it to the latest master at your convenience,
             or amend the last commit of the PR, and push it with --force-with-lease.
       - name: "Initiate GitHub Check forcing rerun of SH ${{ github.event.pull_request.head.sha }}"
-        uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798  # v1.1.0
+        uses: ./.github/actions/checks-action
         id: full-test-check
         if: steps.label-full-test-prs-when-approved-by-commiters.outputs.labelSet == 'true'
         with:
@@ -122,7 +124,7 @@ jobs:
             [the run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
             "}
       - name: "Label when approved by committers for PRs that do not require full tests"
-        uses: apache/airflow-label-when-approved@4c5190fec5661e98d83f50bbd4ef9ebb48bd1194  # v1.3
+        uses: ./.github/actions/label-when-approved-action
         id: label-simple-test-prs-when-approved-by-commiters
         if: >
           steps.selective-checks.outputs.run-tests == 'true' &&
@@ -139,7 +141,7 @@ jobs:
             'full tests needed'. Then you should rebase to the latest master or amend the last commit
             of the PR, and push it with --force-with-lease.
       - name: "Label when approved by committers for PRs that do not require tests at all"
-        uses: apache/airflow-label-when-approved@4c5190fec5661e98d83f50bbd4ef9ebb48bd1194  # v1.3
+        uses: ./.github/actions/label-when-approved-action
         id: label-no-test-prs-when-approved-by-commiters
         if: steps.selective-checks.outputs.run-tests != 'true'
         with:
@@ -153,7 +155,7 @@ jobs:
             needed and add the 'full tests needed' label. Then you should rebase it to the latest master
             or amend the last commit of the PR, and push it with --force-with-lease.
       - name: Update Selective Build check
-        uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798  # v1.1.0
+        uses: ./.github/actions/checks-action
         if: always()
         with:
           token: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,21 @@
+[submodule ".github/actions/get-workflow-origin"]
+	path = .github/actions/get-workflow-origin
+	url = https://github.com/potiuk/get-workflow-origin
+[submodule ".github/actions/cancel-workflow-runs"]
+	path = .github/actions/cancel-workflow-runs
+	url = https://github.com/potiuk/cancel-workflow-runs
+[submodule ".github/actions/checks-action"]
+	path = .github/actions/checks-action
+	url = https://github.com/LouisBrunner/checks-action
+[submodule ".github/actions/configure-aws-credentials"]
+	path = .github/actions/configure-aws-credentials
+	url = https://github.com/aws-actions/configure-aws-credentials
+[submodule ".github/actions/codecov-action"]
+	path = .github/actions/codecov-action
+	url = https://github.com/codecov/codecov-action
+[submodule ".github/actions/github-push-action"]
+	path = .github/actions/github-push-action
+	url = https://github.com/ad-m/github-push-action
+[submodule ".github/actions/label-when-approved-action"]
+	path = .github/actions/label-when-approved-action
+	url = https://github.com/TobKed/label-when-approved-action
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -29,7 +29,7 @@ repos:
     rev: v1.1.9
     hooks:
       - id: forbid-tabs
-        exclude: ^docs/Makefile$|^clients/gen/go.sh
+        exclude: ^docs/Makefile$|^clients/gen/go.sh|\.gitmodules$
       - id: insert-license
         name: Add license for all SQL files
         files: \.sql$