From 395a4a490caf4670115f7312828a3bd4bd6fded5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 25 Dec 2023 10:50:00 +0100 Subject: [PATCH] Cache CI Docker images in ghcr registry --- .github/workflows/ci.yml | 7 ++++ src/ci/docker/run.sh | 76 +++++++++++++----------------------- src/ci/github-actions/ci.yml | 5 +++ 3 files changed, 40 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 540e1eb157e2f..a4bd622dfc76d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,7 @@ jobs: CI_JOB_NAME: "${{ matrix.name }}" CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}" + DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}" SCCACHE_BUCKET: rust-lang-ci-sccache2 TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" CACHE_DOMAIN: ci-caches.rust-lang.org @@ -168,10 +169,13 @@ jobs: if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" auto: name: "auto - ${{ matrix.name }}" + permissions: + packages: write env: CI_JOB_NAME: "${{ matrix.name }}" CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}" + DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}" SCCACHE_BUCKET: rust-lang-ci-sccache2 DEPLOY_BUCKET: rust-lang-ci2 TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" @@ -561,11 +565,14 @@ jobs: if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" try: name: "try - ${{ matrix.name }}" + permissions: + packages: write env: DIST_TRY_BUILD: 1 CI_JOB_NAME: "${{ matrix.name }}" CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}" + DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}" SCCACHE_BUCKET: rust-lang-ci-sccache2 DEPLOY_BUCKET: rust-lang-ci2 TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index 55eed95492d3e..b11b765d989f6 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -74,25 +74,6 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then cksum=$(sha512sum $hash_key | \ awk '{print $1}') - - url="https://$CACHE_DOMAIN/docker/$cksum" - - echo "Attempting to download $url" - rm -f /tmp/rustci_docker_cache - set +e - retry curl --max-time 600 -y 30 -Y 10 --connect-timeout 30 -f -L -C - \ - -o /tmp/rustci_docker_cache "$url" - - docker_archive_hash=$(sha512sum /tmp/rustci_docker_cache | awk '{print $1}') - echo "Downloaded archive hash: ${docker_archive_hash}" - - echo "Loading images into docker" - # docker load sometimes hangs in the CI, so time out after 10 minutes with TERM, - # KILL after 12 minutes - loaded_images=$(/usr/bin/timeout -k 720 600 docker load -i /tmp/rustci_docker_cache \ - | sed 's/.* sha/sha/') - set -e - printf "Downloaded containers:\n$loaded_images\n" fi dockerfile="$docker_dir/$image/Dockerfile" @@ -103,39 +84,38 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then context="$script_dir" fi echo "::group::Building docker image for $image" - - # As of August 2023, Github Actions have updated Docker to 23.X, - # which uses the BuildKit by default. It currently throws aways all - # intermediate layers, which breaks our usage of S3 layer caching. - # Therefore we opt-in to the old build backend for now. - export DOCKER_BUILDKIT=0 - retry docker \ - build \ - --rm \ - -t rust-ci \ - -f "$dockerfile" \ - "$context" + echo "Image checksum ${cksum}" + + docker buildx create --use --driver docker-container + if [ "$PR_CI_JOB" -eq 1 ] + then + retry docker \ + buildx \ + build \ + --rm \ + -t rust-ci \ + -f "$dockerfile" \ + --cache-from type=registry,ref=ghcr.io/rust-lang-ci/rust-ci:${cksum} \ + --output=type=docker \ + "$context" + else + # Login to Docker registry + echo ${DOCKER_TOKEN} | docker login ghcr.io --username rust-lang-ci --password-stdin + retry docker \ + buildx \ + build \ + --rm \ + -t rust-ci \ + -f "$dockerfile" \ + --cache-from type=registry,ref=ghcr.io/rust-lang-ci/rust-ci:${cksum} \ + --cache-to type=registry,ref=ghcr.io/rust-lang-ci/rust-ci:${cksum},compression=zstd,mode=min \ + --output=type=docker \ + "$context" + fi echo "::endgroup::" if [ "$CI" != "" ]; then - s3url="s3://$SCCACHE_BUCKET/docker/$cksum" - upload="aws s3 cp - $s3url" digest=$(docker inspect rust-ci --format '{{.Id}}') - echo "Built container $digest" - if ! grep -q "$digest" <(echo "$loaded_images"); then - echo "Uploading finished image $digest to $url" - set +e - # Print image history for easier debugging of layer SHAs - docker history rust-ci - docker history -q rust-ci | \ - grep -v missing | \ - xargs docker save | \ - gzip | \ - $upload - set -e - else - echo "Looks like docker image is the same as before, not uploading" - fi # Record the container image for reuse, e.g. by rustup.rs builds info="$dist/image-$image.txt" mkdir -p "$dist" diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 3af370bf006ad..7745b94af66ad 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -34,6 +34,7 @@ x--expand-yaml-anchors--remove: CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs. HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }} - &public-variables SCCACHE_BUCKET: rust-lang-ci-sccache2 @@ -345,6 +346,8 @@ jobs: auto: <<: *base-ci-job name: auto - ${{ matrix.name }} + permissions: + packages: write env: <<: [*shared-ci-variables, *prod-variables] if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust' @@ -725,6 +728,8 @@ jobs: try: <<: *base-ci-job name: try - ${{ matrix.name }} + permissions: + packages: write env: DIST_TRY_BUILD: 1 <<: [*shared-ci-variables, *prod-variables]