Use git submodules for (securely) using third party Github Actions.

.github/workflows/build-images-workflow-run.yml

@@ -57,14 +57,19 @@ jobs:
       cacheDirective: ${{ steps.cache-directive.outputs.docker-cache }}
       buildImages: ${{ steps.build-images.outputs.buildImages }}
     steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          submodules: recursive
       - name: "Get information about the original trigger of the run"
-        uses: apache/airflow-get-workflow-origin@588cc14f9f1cdf1b8be3db816855e96422204fec  # v1_3
+        uses: ./.github/actions/get-workflow-origin
         id: source-run-info
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           sourceRunId: ${{ github.event.workflow_run.id }}
       - name: "Cancel duplicated 'CI Build' runs"
-        uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738  # v4_7
+        uses: ./.github/actions/cancel-workflow-runs
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           cancelMode: allDuplicates
@@ -81,7 +86,7 @@ jobs:
         # trick ¯\_(ツ)_/¯. We name the build-info job appropriately
         # and then we try to find and cancel all the jobs with the same Event + Repo + Branch as the
         # current Event/Repo/Branch combination.
-        uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738  # v4_7
+        uses: ./.github/actions/cancel-workflow-runs
         with:
           cancelMode: namedJobs
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -97,7 +102,7 @@ jobs:
         # We also produce list of canceled "CI Build' runs as output, so that we
         # can cancel all the matching "Build Images" workflow runs in the two following steps.
         # Yeah. Adding to the complexity ¯\_(ツ)_/¯.
-        uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738  # v4_7
+        uses: ./.github/actions/cancel-workflow-runs
         id: cancel-failed
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -130,14 +135,14 @@ jobs:
         # it to cancel any jobs that have matching names containing Source Run Id:
         # followed by one of the run ids. Yes I know it's super complex ¯\_(ツ)_/¯.
         if: env.BUILD_IMAGES == 'true' && steps.cancel-failed.outputs.cancelledRuns != '[]'
-        uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738  # v4_7
+        uses: ./.github/actions/cancel-workflow-runs
         with:
           cancelMode: namedJobs
           token: ${{ secrets.GITHUB_TOKEN }}
           notifyPRCancel: true
           jobNameRegexps: ${{ steps.extract-cancelled-failed-runs.outputs.matching-regexp }}
       - name: "Cancel duplicated 'CodeQL' runs"
-        uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738  # v4_7
+        uses: ./.github/actions/cancel-workflow-runs
         id: cancel
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -157,7 +162,7 @@ jobs:
         # trick ¯\_(ツ)_/¯. We name the build-info job appropriately and then we try to match
         # all the jobs with the same Event + Repo + Branch match and cancel all the duplicates for those
         # This might cancel own run, so this is the last step in the job
-        uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738  # v4_7
+        uses: ./.github/actions/cancel-workflow-runs
         with:
           cancelMode: allDuplicatedNamedJobs
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -297,8 +302,19 @@ jobs:
           else
               echo "::set-output name=proceed::true"
           fi
+      - name: >
+          Checkout "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" branch to 'main-airflow' folder
+          to use ci/scripts from there.
+        uses: actions/checkout@v2
+        with:
+          path: "main-airflow"
+          ref: "${{ needs.cancel-workflow-runs.outputs.targetBranch }}"
+          persist-credentials: false
+          submodules: recursive
+        if: steps.defaults.outputs.proceed == 'true'
       - name: Initiate GitHub Checks for Building image
-        uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798  # v1.1.0
+        # Use the submodule from main, not the PR branch
+        uses: ./main-airflow/.github/actions/checks-action
         id: build-image-check
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -312,15 +328,6 @@ jobs:
             [Image Build](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
             for details" }
         if: steps.defaults.outputs.proceed == 'true'
-      - name: >
-          Checkout "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" branch to 'main-airflow' folder
-          to use ci/scripts from there.
-        uses: actions/checkout@v2
-        with:
-          path: "main-airflow"
-          ref: "${{ needs.cancel-workflow-runs.outputs.targetBranch }}"
-          persist-credentials: false
-        if: steps.defaults.outputs.proceed == 'true'
       - name: "Setup python"
         uses: actions/setup-python@v2
         with:
@@ -474,7 +481,7 @@ jobs:
         run: ./scripts/ci/images/ci_push_production_images.sh
         if: steps.defaults.outputs.proceed == 'true'
       - name: Update GitHub Checks for Building image with status
-        uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798  # v1.1.0
+        uses: ./main-airflow/.github/actions/checks-action
         if: always() && steps.defaults.outputs.proceed == 'true'
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -495,8 +502,13 @@ jobs:
     if: cancelled()
     needs: [build-ci-images, build-prod-images]
     steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          submodules: recursive
       - name: "Canceling the 'CI Build' source workflow in case of failure!"
-        uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738  # v4_7
+        uses: ./.github/actions/cancel-workflow-runs
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           cancelMode: self
@@ -510,8 +522,13 @@ jobs:
     if: failure()
     needs: [build-ci-images, build-prod-images]
     steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          submodules: recursive
       - name: "Canceling the 'CI Build' source workflow in case of failure!"
-        uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738  # v4_7
+        uses: ./.github/actions/cancel-workflow-runs
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           cancelMode: self

.github/workflows/ci.yml

@@ -100,15 +100,16 @@ jobs:
       pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }}
       pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }}
     steps:
-      - name: "Get information about the PR"
-        uses: apache/airflow-get-workflow-origin@588cc14f9f1cdf1b8be3db816855e96422204fec  # v1_3
-        id: source-run-info
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v2
         with:
           persist-credentials: false
+          submodules: recursive
+      - name: "Get information about the PR"
+        uses: ./.github/actions/get-workflow-origin
+        id: source-run-info
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
       - name: >
           Event: ${{ github.event_name }}
           Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
@@ -346,6 +347,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           persist-credentials: false
+          submodules: recursive
       - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
         run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Build docs"
@@ -358,7 +360,7 @@ jobs:
           path: "./files/documentation"
           retention-days: 7
       - name: Configure AWS credentials
-        uses: apache/airflow-configure-aws-credentials@e97d7fbc8e0e5af69631c13daa0f4b5a8d88165b
+        uses: ./.github/actions/configure-aws-credentials
         if: >
           github.ref == 'refs/heads/master' && github.repository == 'apache/airflow' &&
           github.event_name == 'push'
@@ -823,14 +825,19 @@ jobs:
       - tests-mysql
       - tests-quarantined
     steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          submodules: recursive
       - name: "Download all artifacts from the current build"
         uses: actions/download-artifact@v2
         with:
           path: ./coverage-files
       - name: "Removes unnecessary artifacts"
         run: ls ./coverage-files | grep -v coverage | xargs rm -rf
       - name: "Upload all coverage reports to codecov"
-        uses: apache/airflow-codecov-action@1fc7722ded4708880a5aea49f2bfafb9336f0c8d  # v1.1.1
+        uses: ./.github/actions/codecov-action
         with:
           directory: "./coverage-files"
 
@@ -1118,6 +1125,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           persist-credentials: false
+          submodules: recursive
       - name: "Set constraints branch name"
         id: constraints-branch
         run: ./scripts/ci/constraints/ci_branch_constraints.sh
@@ -1134,7 +1142,7 @@ jobs:
       - name: "Commit changed constraint files for ${{needs.build-info.outputs.pythonVersions}}"
         run: ./scripts/ci/constraints/ci_commit_constraints.sh
       - name: "Push changes"
-        uses: apache/airflow-github-push-action@40bf560936a8022e68a3c00e7d2abefaf01305a6  # v0.6.0
+        uses: ./.github/actions/github-push-action
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           branch: ${{ steps.constraints-branch.outputs.branch }}
@@ -1160,6 +1168,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           persist-credentials: false
+          submodules: recursive
       - name: "Free space"
         run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Tag commit"
@@ -1168,7 +1177,7 @@ jobs:
           echo "Tagging ${BRANCH_NAME}"
           git tag -f nightly-${BRANCH_NAME} HEAD
       - name: "Push tags"
-        uses: apache/airflow-github-push-action@40bf560936a8022e68a3c00e7d2abefaf01305a6  # v0.6.0
+        uses: ./.github/actions/github-push-action
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           tags: true

.github/workflows/label_when_reviewed_workflow_run.yml

@@ -29,14 +29,19 @@ jobs:
     outputs:
       labelSet: ${{ steps.label-when-reviewed.outputs.labelSet }}
     steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          submodules: recursive
       - name: "Get information about the original trigger of the run"
-        uses: apache/airflow-get-workflow-origin@588cc14f9f1cdf1b8be3db816855e96422204fec  # v1_3
+        uses: ./.github/actions/get-workflow-origin
         id: source-run-info
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           sourceRunId: ${{ github.event.workflow_run.id }}
       - name: Initiate Selective Build check
-        uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798  # v1.1.0
+        uses: ./.github/actions/checks-action
         id: selective-build-check
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -49,10 +54,6 @@ jobs:
             "Checking selective status of the build in
             [the run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
             "}
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v2
-        with:
-          persist-credentials: false
       - name: >
           Event: ${{ steps.source-run-info.outputs.sourceEvent }}
           Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
@@ -76,6 +77,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           persist-credentials: false
+          submodules: recursive
       - name: Selective checks
         id: selective-checks
         env:
@@ -91,7 +93,7 @@ jobs:
             ./scripts/ci/selective_ci_checks.sh
           fi
       - name: "Label when approved by committers for PRs that require full tests"
-        uses: apache/airflow-label-when-approved@4c5190fec5661e98d83f50bbd4ef9ebb48bd1194  # v1.3
+        uses: ./.github/actions/label-when-approved-action
         id: label-full-test-prs-when-approved-by-commiters
         if: >
           steps.selective-checks.outputs.run-tests == 'true' &&
@@ -107,7 +109,7 @@ jobs:
             If they don't merge it quickly - please rebase it to the latest master at your convenience,
             or amend the last commit of the PR, and push it with --force-with-lease.
       - name: "Initiate GitHub Check forcing rerun of SH ${{ github.event.pull_request.head.sha }}"
-        uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798  # v1.1.0
+        uses: ./.github/actions/checks-action
         id: full-test-check
         if: steps.label-full-test-prs-when-approved-by-commiters.outputs.labelSet == 'true'
         with:
@@ -122,7 +124,7 @@ jobs:
             [the run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
             "}
       - name: "Label when approved by committers for PRs that do not require full tests"
-        uses: apache/airflow-label-when-approved@4c5190fec5661e98d83f50bbd4ef9ebb48bd1194  # v1.3
+        uses: ./.github/actions/label-when-approved-action
         id: label-simple-test-prs-when-approved-by-commiters
         if: >
           steps.selective-checks.outputs.run-tests == 'true' &&
@@ -139,7 +141,7 @@ jobs:
             'full tests needed'. Then you should rebase to the latest master or amend the last commit
             of the PR, and push it with --force-with-lease.
       - name: "Label when approved by committers for PRs that do not require tests at all"
-        uses: apache/airflow-label-when-approved@4c5190fec5661e98d83f50bbd4ef9ebb48bd1194  # v1.3
+        uses: ./.github/actions/label-when-approved-action
         id: label-no-test-prs-when-approved-by-commiters
         if: steps.selective-checks.outputs.run-tests != 'true'
         with:
@@ -153,7 +155,7 @@ jobs:
             needed and add the 'full tests needed' label. Then you should rebase it to the latest master
             or amend the last commit of the PR, and push it with --force-with-lease.
       - name: Update Selective Build check
-        uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798  # v1.1.0
+        uses: ./.github/actions/checks-action
         if: always()
         with:
           token: ${{ secrets.GITHUB_TOKEN }}

.gitmodules

@@ -0,0 +1,21 @@
+[submodule ".github/actions/get-workflow-origin"]
+	path = .github/actions/get-workflow-origin
+	url = https://github.com/potiuk/get-workflow-origin
+[submodule ".github/actions/cancel-workflow-runs"]
+	path = .github/actions/cancel-workflow-runs
+	url = https://github.com/potiuk/cancel-workflow-runs
+[submodule ".github/actions/checks-action"]
+	path = .github/actions/checks-action
+	url = https://github.com/LouisBrunner/checks-action
+[submodule ".github/actions/configure-aws-credentials"]
+	path = .github/actions/configure-aws-credentials
+	url = https://github.com/aws-actions/configure-aws-credentials
+[submodule ".github/actions/codecov-action"]
+	path = .github/actions/codecov-action
+	url = https://github.com/codecov/codecov-action
+[submodule ".github/actions/github-push-action"]
+	path = .github/actions/github-push-action
+	url = https://github.com/ad-m/github-push-action
+[submodule ".github/actions/label-when-approved-action"]
+	path = .github/actions/label-when-approved-action
+	url = https://github.com/TobKed/label-when-approved-action

.pre-commit-config.yaml

@@ -29,7 +29,7 @@ repos:
     rev: v1.1.9
     hooks:
       - id: forbid-tabs
-        exclude: ^docs/Makefile$|^clients/gen/go.sh
+        exclude: ^docs/Makefile$|^clients/gen/go.sh|\.gitmodules$
       - id: insert-license
         name: Add license for all SQL files
         files: \.sql$