feat(ci): Bazel remote cache on s3 (#369)

* feat(ci): Bazel remote cache on s3

* fix

.bazelrc

@@ -36,9 +36,15 @@ build --@rules_rust//:extra_rustc_flag=-Dbindings_with_variant_name
 build --strip=never
 
 # build:dfinity --remote_cache=bazel-remote.idx.dfinity.network
-build --experimental_remote_cache_async
+build --remote_cache=grpc://localhost:9092
+build --remote_instance_name=default
+build --google_default_credentials=false
+build --remote_upload_local_results=false
+# build --experimental_remote_cache_async
+build --experimental_remote_cache_compression # compress/decompress cache blobs with zstd
 build --remote_timeout=30s # Default is 60s.
 build:ci --remote_timeout=5m # Default is 60s.
+build:ci --remote_upload_local_results=true
 
 # build:dfinity --experimental_remote_downloader=bazel-remote.idx.dfinity.network --experimental_remote_downloader_local_fallback
 build:local --experimental_remote_downloader=

.github/workflows/main.yaml

@@ -26,6 +26,21 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 2
+      - name: "☁️ Setup S3 cache credentials"
+        id: bazel-remote-creds
+        shell: bash
+        run: |
+          mkdir -p ~/.aws
+          cat <<_EOF | install -m 0600 /dev/stdin ~/.aws/config
+          [wasabi]
+          region = eu-central-2
+          _EOF
+
+          cat <<_EOF | install -m 0600 /dev/stdin ~/.aws/credentials
+          [wasabi]
+          aws_access_key_id = ${{ secrets.WASABI_ACCESS_KEY }}
+          aws_secret_access_key = ${{ secrets.WASABI_SECRET_KEY }}
+          _EOF
       - name: "☁️ Setup runner"
         uses: ./.github/workflows/manage-runner-pre
 

.github/workflows/manage-runner-post/action.yaml

@@ -9,6 +9,13 @@ inputs:
 runs:
   using: composite
   steps:
+    ########################################
+    # Stop bazel-remote cache
+    ########################################
+    - name: "💾 Stop bazel-remote s3 cache"
+      run: ./bin/bazel-cache-s3-stop.sh
+      shell: bash
+
     ########################################
     # Optimize bazel cache by hard-linking duplicate files
     ########################################

.github/workflows/manage-runner-pre/action.yaml

@@ -35,3 +35,7 @@ runs:
       # Bazel actually keeps all intermediate objects so builds are still fast
       run: bazel clean
       shell: bash
+
+    - name: "💾 Start bazel-remote s3 cache"
+      run: ./bin/bazel-cache-s3-start.sh
+      shell: bash

bin/bazel-cache-s3-start.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -eExuo pipefail
+
+mkdir -p ~/.cache/bazel-remote
+docker run --detach --name=bazel-remote -u $(id -u):$(id -g) \
+    -v ~/.cache/bazel-remote:/data -v $HOME/.aws:/aws-config \
+    -p 9090:8080 -p 9092:9092 \
+    quay.io/bazel-remote/bazel-remote \
+    --max_size 20 \
+    --s3.auth_method=aws_credentials_file --s3.aws_profile=wasabi --s3.aws_shared_credentials_file=/aws-config/credentials --s3.bucket=dre-ci-cache --s3.endpoint=s3.eu-central-2.wasabisys.com

bin/bazel-cache-s3-stop.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -eExuo pipefail
+
+docker logs bazel-remote
+docker stop bazel-remote
+docker rm bazel-remote