chore(ci): changing main pipeline to run on self hosted runners (#400)

* changing main pipeline to run on self hosted runners * removing unneeded steps in main pipeline * fallback to standalone execution if remote fails * adding debug step for creating cache folder if it doesn't exist * adding shell * adding mkdir * adding git * adding update and upgrade if needed * adding -y to all apt commands * fixing setup * fixing setup * adding gcc * updating vector sha * adding platforms * adding docker components * adding gpg keys * adding manifest sha * removing docker * adding echos * adjusting workspace_status.sh * adding clang and g++ * adding libssl * adding pkg-config * printing openssl version * installing correct openssl * fixing dir creating * adding sleep to debug * adding symbolic links * installing full docker * removing groupadd * removing group management * adding systemd * adding sudo for docker info * removing dry-running of tarballs * testing configured custom image * setting container for the job * changing ci disk cache and repo cache locations * testing new runner setup * adding chown step * fixing indentation * cleaning up pipeline * commenting out currently unknown part * removing obsolete shell scripts * Revert "removing obsolete shell scripts" This reverts commit 98e6faa. * adding checks to execute some code only on github hosted runners * setting different container image for our ci * adding debug setp * changing uid * adding check for uid and gid * patching bash scripts
dfinity · May 30, 2024 · 51fcd6b · 51fcd6b
1 parent dbd482b
commit 51fcd6b
Show file tree

Hide file tree

Showing 8 changed files with 85 additions and 12 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -79,7 +79,7 @@ test:precommit --build_tests_only
 # Run all tests once by default, run flaky tests up to 3 times in CI.
 test    --flaky_test_attempts=1
 test:ci --flaky_test_attempts=default
-build:ci --noremote_local_fallback
+build:ci --remote_local_fallback
 # So that developers can build in debug mode.
 build:dev --compilation_mode=fastbuild
 build:macos_ci --compilation_mode=fastbuild

diff --git a/.github/workflows/build/action.yaml b/.github/workflows/build/action.yaml
@@ -55,7 +55,11 @@ runs:
         branch: ${{ github.head_ref }}
         ssh: true
 
+    # TODO: right now we can't do it because our runners run in "kubernetes mode"
+    #       we need them in "dind mode". Not sure how it will affect other parts
+    #       of the runners.
     - name: Dry run tarballs
+      if: ${{ !startsWith(runner.name, 'dre-runner-custom') }}
       shell: bash
       run: |
         #!/bin/bash

diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -21,7 +21,10 @@ concurrency:
 
 jobs:
   bazel:
-    runs-on: ubuntu-20.04
+    runs-on: 
+      labels: dre-runner-custom
+    # This image is based on ubuntu:20.04
+    container: ghcr.io/dfinity/dre/actions-runner:0.2.1
     steps:
       - uses: actions/checkout@v4
         with:
@@ -131,5 +134,6 @@ jobs:
       ########################################
       - uses: ./.github/workflows/manage-runner-post
         name: "🪓 Tear down runner"
+        if: ${{ !startsWith(runner.name, 'dre-runner-custom') }}
         with:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/manage-runner-post/action.yaml b/.github/workflows/manage-runner-post/action.yaml
@@ -34,15 +34,11 @@ runs:
       if: ${{ github.ref == 'refs/heads/main' }}
       run: |
         set -eExou pipefail
-
         gh extension install actions/gh-actions-cache
-
         REPO=${{ github.repository }}
         BRANCH=${{ github.ref }}
-
         echo "Fetching list of cache key"
         cacheKeys=$(gh actions-cache list -R $REPO -B $BRANCH | cut -f 1 )
-
         ## Setting this to not fail the workflow while deleting cache keys.
         echo "Deleting caches..."
         for cacheKey in $cacheKeys

diff --git a/.github/workflows/manage-runner-pre/action.yaml b/.github/workflows/manage-runner-pre/action.yaml
@@ -12,23 +12,51 @@ inputs:
 runs:
   using: composite
   steps:
+    - name: "👮‍♀️ Setup runner"
+      if: ${{ startsWith(runner.name, 'dre-runner-custom') }}
+      shell: bash
+      run: |
+        set -exuo pipefail
+        whoami
+
+        path_to_check="/home/runner/.cache"
+
+        current_owner=$(stat -c "%u" "$path_to_check")
+        current_group=$(stat -c "%g" "$path_to_check")
+
+        desired_owner=1001
+        desired_group=1001
+
+        # Check if the current owner and group match the desired owner and group
+        if [ "$current_owner" -ne "$desired_owner" ] || [ "$current_group" -ne "$desired_group" ]; then
+            echo "Owner or group of cache does not match. Changing ownership..."
+            # Change the owner and group recursively
+            sudo chown -R $desired_owner:$desired_group "$path_to_check"
+            echo "Ownership changed to $desired_owner:$desired_group for $path_to_check"
+        else
+            echo "Ownership is already set to $desired_owner:$desired_group for $path_to_check"
+        fi
+
     ########################################
     # Setup
     ########################################
     - name: "🔧 Free Up Disk Space"
       uses: jlumbroso/[email protected]
+      if: ${{ !startsWith(runner.name, 'dre-runner-custom') }}
       with:
         # this might remove tools that are actually needed,
         # when set to "true" but frees about 6 GB
         tool-cache: true
         large-packages: true  # this is slow
+
     - uses: bazelbuild/setup-bazelisk@v2
 
     ########################################
     # Download and unpack cache
     ########################################
     - name: "☁️ ⬇️ Restore bazel cache"
       uses: actions/cache/restore@v4
+      if: ${{ !startsWith(runner.name, 'dre-runner-custom') }}
       with:
         path: "~/.cache/bazel"
         # Configure cache updates
@@ -37,14 +65,16 @@ runs:
         key: ${{ runner.os }}-bazel-${{ hashFiles('.bazelversion', '.bazelrc', 'WORKSPACE.bazel', 'Cargo.Bazel.lock', 'requirements.txt') }}
         restore-keys: |
             ${{ runner.os }}-bazel-
+    
     - name: "🧹 Clean bazel cache if we're preparing a new release"
-      if: ${{ startsWith(github.ref, 'refs/tags/v') }}
+      if: ${{ startsWith(github.ref, 'refs/tags/v') && !startsWith(runner.name, 'dre-runner-custom') }}
       # This is desirable to make sure bazel does not use stale pre-built binaries
       # Bazel actually keeps all intermediate objects so builds are still fast
       run: bazel clean
       shell: bash
 
     - name: "☁️ Setup S3 cache credentials"
+      if: ${{ !startsWith(runner.name, 'dre-runner-custom') }}
       id: bazel-remote-creds
       shell: bash
       run: |
@@ -55,13 +85,14 @@ runs:
         [wasabi]
         region = eu-central-2
         _EOF
-
         cat <<_EOF | install -m 0600 /dev/stdin ~/.aws/credentials
         [wasabi]
         aws_access_key_id = ${{ inputs.aws_access_key_id }}
         aws_secret_access_key = ${{ inputs.aws_secret_access_key }}
         _EOF
 
     - name: "💾 Start bazel-remote s3 cache"
+      if: ${{ !startsWith(runner.name, 'dre-runner-custom') }}
       run: ./bin/bazel-cache-s3-start.sh
       shell: bash
+
diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel
@@ -194,9 +194,9 @@ oci_pull(
 )
 
 oci_pull(
-    # tag = 0.34.1-debian
+    # tag = 0.38.0-debian
     name = "vector-debian",
-    digest = "sha256:6d8cf7cbafe8591d3e04d9f4aa9889c720ab36bb8712cad04979261d35b501cf",
+    digest = "sha256:f33f11c554d522fb9a3d1725c9212725da8f1eee70a3971e587e78d46e72a779",
     image = "index.docker.io/timberio/vector",
 )
 

diff --git a/bazel/workspace_status.sh b/bazel/workspace_status.sh
@@ -4,8 +4,12 @@ set -euo pipefail
 
 echo "BUILD_TIME \"$(TZ=UTC date --rfc-3339=seconds)\""
 
-GIT_REV=$(git tag --points-at HEAD)
-if [[ -z "${GITREV:-}" ]]; then GIT_REV=$(git describe --always --dirty); fi
+if [[ -n "${GITHUB_SHA:-}" ]]; then
+  GIT_REV="$GITHUB_SHA"
+else
+  GIT_REV=$(git tag --points-at HEAD)
+  if [[ -z "${GIT_REV:-}" ]]; then GIT_REV=$(git describe --always --dirty); fi
+fi
 
 echo "GIT_REV ${GIT_REV:-unset}"
 

diff --git a/docker/runner.Dockerfile b/docker/runner.Dockerfile
@@ -0,0 +1,34 @@
+FROM ubuntu:20.04
+
+LABEL org.opencontainers.image.source="https://github.com/dfinity/dre"
+
+ENV TZ=UTC
+ENV DEBIAN_FRONTEND=noninteractive
+ENV RUNNER_UID=1001
+
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install ca-certificates curl git-all gcc g++ clang pkg-config make sudo docker.io build-essential -y
+
+RUN mkdir -p openssl && \
+    curl -o openssl/openssl-1.1.1w.tar.gz -L https://www.openssl.org/source/old/1.1.1/openssl-1.1.1w.tar.gz && \
+    tar -xzvf openssl/openssl-1.1.1w.tar.gz -C openssl && \
+    cd openssl/openssl-1.1.1w && \
+    ./config && \
+    make && \
+    make install
+RUN ln -s /usr/local/lib/libssl.so.1.1 /usr/lib64/libssl.so.1.1 && \
+    ln -s /usr/local/lib/libssl.so.1.1 /usr/lib/libssl.so.1.1 && \
+    ln -s /usr/local/lib/libcrypto.so.1.1 /usr/lib64/libcrypto.so.1.1 && \
+    ln -s /usr/local/lib/libcrypto.so.1.1 /usr/lib/libcrypto.so.1.1 && \
+    rm -rf openssl
+
+# Runner user
+RUN adduser --disabled-password --gecos "" --uid $RUNNER_UID runner \
+    && usermod -aG sudo runner \
+    && usermod -aG docker runner \
+    && echo "%sudo   ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers \
+    && echo "Defaults env_keep += \"DEBIAN_FRONTEND\"" >> /etc/sudoers
+ENV HOME=/home/runner
+USER runner
+WORKDIR /home/runner