From e17e218449ff17d3451c5f4836127cc81666e2c5 Mon Sep 17 00:00:00 2001 From: Josh Deprez Date: Thu, 10 Oct 2024 11:44:30 +1100 Subject: [PATCH] Parallelise container image uploads --- .buildkite/pipeline.release-experimental.yml | 29 ++++--- .buildkite/pipeline.release-stable.yml | 29 ++++--- .buildkite/pipeline.release-unstable.yml | 29 ++++--- .buildkite/steps/publish-docker-image.sh | 76 ++++++++++------- .buildkite/steps/publish-docker-images.sh | 85 +++++++++++--------- 5 files changed, 151 insertions(+), 97 deletions(-) diff --git a/.buildkite/pipeline.release-experimental.yml b/.buildkite/pipeline.release-experimental.yml index f2541c14c8..a551ce5e91 100644 --- a/.buildkite/pipeline.release-experimental.yml +++ b/.buildkite/pipeline.release-experimental.yml @@ -136,13 +136,22 @@ steps: - with: { pkg_arch: "SKIP_FAKE_ARCH" } skip: true - - name: ":docker: Publish Edge Docker Images" - command: ".buildkite/steps/publish-docker-images.sh" - env: - CODENAME: "experimental" - plugins: - - aws-assume-role-with-web-identity: - role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-agent-release-edge - - ecr#v2.7.0: - login: true - account-ids: "445615400570" + - group: ":docker: Publish Edge Docker Images" + steps: + - name: ":docker: Publish Edge Images to {{matrix.registry}}" + command: ".buildkite/steps/publish-docker-images.sh" + env: + CODENAME: "experimental" + REGISTRY: "{{matrix.registry}}" + plugins: + - aws-assume-role-with-web-identity: + role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-agent-release-edge + - ecr#v2.7.0: + login: true + account-ids: "445615400570" + matrix: + setup: + registry: + - docker.io + - ghcr.io + - packages.buildkite.com diff --git a/.buildkite/pipeline.release-stable.yml b/.buildkite/pipeline.release-stable.yml index 65b532c5c1..350abd102b 100644 --- a/.buildkite/pipeline.release-stable.yml +++ b/.buildkite/pipeline.release-stable.yml @@ -160,16 +160,25 @@ steps: - with: { pkg_arch: "SKIP_FAKE_ARCH" } skip: true - - name: ":docker: Publish Docker Images" - command: ".buildkite/steps/publish-docker-images.sh" - env: - CODENAME: "stable" - plugins: - - aws-assume-role-with-web-identity: - role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-agent-release-stable - - ecr#v2.7.0: - login: true - account-ids: "445615400570" + - group: ":docker: Publish Docker Images" + steps: + - name: ":docker: Publish Docker Images to {{matrix.registry}}" + command: ".buildkite/steps/publish-docker-images.sh" + env: + CODENAME: "stable" + REGISTRY: "{{matrix.registry}}" + plugins: + - aws-assume-role-with-web-identity: + role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-agent-release-stable + - ecr#v2.7.0: + login: true + account-ids: "445615400570" + matrix: + setup: + registry: + - docker.io + - ghcr.io + - packages.buildkite.com - wait diff --git a/.buildkite/pipeline.release-unstable.yml b/.buildkite/pipeline.release-unstable.yml index e5d6e6536b..4ce8523e21 100644 --- a/.buildkite/pipeline.release-unstable.yml +++ b/.buildkite/pipeline.release-unstable.yml @@ -160,16 +160,25 @@ steps: - with: { pkg_arch: "SKIP_FAKE_ARCH" } skip: true - - name: ":docker: Publish Unstable Docker Images" - command: ".buildkite/steps/publish-docker-images.sh" - env: - CODENAME: "unstable" - plugins: - - aws-assume-role-with-web-identity: - role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-agent-release-beta - - ecr#v2.7.0: - login: true - account-ids: "445615400570" + - group: ":docker: Publish Unstable Docker Images" + steps: + - name: ":docker: Publish Unstable Images to {{matrix.registry}}" + command: ".buildkite/steps/publish-docker-images.sh" + env: + CODENAME: "unstable" + REGISTRY: "{{matrix.registry}}" + plugins: + - aws-assume-role-with-web-identity: + role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-agent-release-beta + - ecr#v2.7.0: + login: true + account-ids: "445615400570" + matrix: + setup: + registry: + - docker.io + - ghcr.io + - packages.buildkite.com - wait diff --git a/.buildkite/steps/publish-docker-image.sh b/.buildkite/steps/publish-docker-image.sh index 9968b32b63..739a1cfbc9 100755 --- a/.buildkite/steps/publish-docker-image.sh +++ b/.buildkite/steps/publish-docker-image.sh @@ -1,10 +1,11 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail ## This script can be run locally like this: ## -## .buildkite/steps/publish-docker-image.sh (alpine|ubuntu) imagename (stable|experimental|unstable) -## .buildkite/steps/publish-docker-image.sh alpine buildkiteci/agent:lox-manual-build stable 3.1.1 +## .buildkite/steps/publish-docker-image.sh (docker.io|ghcr.io|packages.buildkite.com) (alpine|ubuntu) imagename (stable|experimental|unstable) +## e.g. +## .buildkite/steps/publish-docker-image.sh docker.io alpine buildkiteci/agent:lox-manual-build stable 3.1.1 dry_run() { if [[ "${DRY_RUN:-}" == "false" ]] ; then @@ -36,68 +37,81 @@ parse_version() { release_image() { local tag="$1" - echo "--- :docker: Copying ${target_image}:${tag} to Docker Hub" - dry_run skopeo copy --multi-arch all "docker://${source_image}" "docker://docker.io/buildkite/${target_image}:${tag}" - echo "--- :github: Copying ${target_image}:${tag} to GHCR" - dry_run skopeo copy --multi-arch all "docker://${source_image}" "docker://ghcr.io/buildkite/${target_image}:${tag}" - - # OIDC tokens only last 5 minutes, and issuing them is cheap, so log in as close as possible to the push - buildkite-agent oidc request-token \ - --audience "https://packages.buildkite.com/buildkite/agent-docker" \ - --lifetime 300 \ - | docker login packages.buildkite.com/buildkite/agent-docker --username=buildkite --password-stdin - - echo "--- :buildkite: Copying ${target_image}:${tag} to Buildkite Packages" - dry_run skopeo copy --multi-arch all "docker://${source_image}" "docker://packages.buildkite.com/buildkite/agent-docker/${target_image}:${tag}" + + case "${registry}" in + docker.io) + echo "--- :docker: Copying ${target_image}:${tag} to Docker Hub" + dry_run skopeo copy --multi-arch all "docker://${source_image}" "docker://docker.io/buildkite/${target_image}:${tag}" + ;; + ghcr.io) + echo "--- :github: Copying ${target_image}:${tag} to GHCR" + dry_run skopeo copy --multi-arch all "docker://${source_image}" "docker://ghcr.io/buildkite/${target_image}:${tag}" + ;; + packages.buildkite.com) + # OIDC tokens only last 5 minutes, and issuing them is cheap, so log in as close as possible to the push + buildkite-agent oidc request-token \ + --audience "https://packages.buildkite.com/buildkite/agent-docker" \ + --lifetime 300 \ + | docker login packages.buildkite.com/buildkite/agent-docker --username=buildkite --password-stdin + + echo "--- :buildkite: Copying ${target_image}:${tag} to Buildkite Packages" + dry_run skopeo copy --multi-arch all "docker://${source_image}" "docker://packages.buildkite.com/buildkite/agent-docker/${target_image}:${tag}" + ;; + *) + echo "+++ Registry '${registry}' is not supported\!" + exit 1 + ;; + esac } -variant="${1:-}" -source_image="${2:-}" -codename="${3:-}" -version="${4:-}" -build="${5:-dev}" +registry="${1:-}" +variant="${2:-}" +source_image="${3:-}" +codename="${4:-}" +version="${5:-}" +build="${6:-dev}" target_image="agent" variant_suffix="" -if [[ "$variant" != "alpine" ]] ; then - variant_suffix="-$variant" +if [[ "${variant}" != "alpine" ]] ; then + variant_suffix="-${variant}" fi echo "Tagging docker images for $variant/$codename (version $version build $build)" # variants of edge/experimental -if [[ "$codename" == "experimental" ]] ; then +if [[ "${codename}" == "experimental" ]] ; then release_image "edge-build-${build}${variant_suffix}" release_image "edge${variant_suffix}" fi # variants of stable - e.g 2.3.2 -if [[ "$codename" == "stable" ]] ; then - for tag in $(parse_version "$version") ; do +if [[ "${codename}" == "stable" ]] ; then + for tag in $(parse_version "${version}") ; do release_image "${tag}${variant_suffix}" done release_image "${variant}" # publish bare 'ubuntu' only from ubuntu-22.04 - if [[ "$variant" == "ubuntu-22.04" ]] ; then - for tag in $(parse_version "$version") ; do + if [[ "${variant}" == "ubuntu-22.04" ]] ; then + for tag in $(parse_version "${version}") ; do release_image "${tag}-ubuntu" done release_image "ubuntu" fi # publish latest and stable only from alpine - if [[ "$variant" == "alpine" ]] ; then + if [[ "${variant}" == "alpine" ]] ; then release_image "latest" release_image "stable" fi fi # variants of beta/unstable - e.g 3.0-beta.16 -if [[ "$codename" == "unstable" ]] ; then +if [[ "${codename}" == "unstable" ]] ; then release_image "beta${variant_suffix}" - if [[ "$version" =~ -(alpha|beta|rc)\.[0-9]+$ ]] ; then + if [[ "${version}" =~ -(alpha|beta|rc)\.[0-9]+$ ]] ; then release_image "${version}${variant_suffix}" fi fi diff --git a/.buildkite/steps/publish-docker-images.sh b/.buildkite/steps/publish-docker-images.sh index 1815ce6e25..adfa812835 100755 --- a/.buildkite/steps/publish-docker-images.sh +++ b/.buildkite/steps/publish-docker-images.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail dry_run() { @@ -9,51 +9,64 @@ dry_run() { fi } -if [[ "$CODENAME" == "" ]]; then +if [[ "${CODENAME:-}" == "" ]]; then echo "Error: Missing \$CODENAME (stable, experimental or unstable)" exit 1 fi -echo "--- docker login to Docker Hub" - -dockerhub_user="$(aws ssm get-parameter \ - --name /pipelines/agent/DOCKER_HUB_USER \ - --with-decryption \ - --output text \ - --query Parameter.Value \ - --region us-east-1\ -)" - -aws ssm get-parameter \ - --name /pipelines/agent/DOCKER_HUB_PASSWORD \ - --with-decryption \ - --output text \ - --query Parameter.Value \ - --region us-east-1 \ - | docker login --username="${dockerhub_user}" --password-stdin - +if [[ "${REGISTRY:-}" == "" ]]; then + echo "Error: Missing \$REGISTRY (docker.io, ghcr.io, or packages.buildkite.com)" + exit 1 +fi -echo "--- docker login to GitHub" +case "${REGISTRY}" in +docker.io) + echo "--- docker login to Docker Hub" + dockerhub_user="$(aws ssm get-parameter \ + --name /pipelines/agent/DOCKER_HUB_USER \ + --with-decryption \ + --output text \ + --query Parameter.Value \ + --region us-east-1\ + )" + aws ssm get-parameter \ + --name /pipelines/agent/DOCKER_HUB_PASSWORD \ + --with-decryption \ + --output text \ + --query Parameter.Value \ + --region us-east-1 \ + | docker login --username="${dockerhub_user}" --password-stdin + ;; +ghcr.io) + echo "--- docker login to GitHub" -ghcr_user=buildkite-agent-releaser -aws ssm get-parameter \ - --name /pipelines/agent/GITHUB_RELEASE_ACCESS_TOKEN \ - --with-decryption \ - --output text \ - --query Parameter.Value \ - --region us-east-1 \ - | docker login ghcr.io --username="${ghcr_user}" --password-stdin + ghcr_user=buildkite-agent-releaser + aws ssm get-parameter \ + --name /pipelines/agent/GITHUB_RELEASE_ACCESS_TOKEN \ + --with-decryption \ + --output text \ + --query Parameter.Value \ + --region us-east-1 \ + | docker login ghcr.io --username="${ghcr_user}" --password-stdin -echo "--- docker login to Buildkite Packages" + ;; +packages.buildkite.com) + echo "--- Uploading images to Buildkite Packages" + ;; +*) + echo "+++ Registry '${REGISTRY}' is not supported\!" + exit 1 + ;; +esac -version=$(buildkite-agent meta-data get "agent-version") -build=$(buildkite-agent meta-data get "agent-version-build") +version="$(buildkite-agent meta-data get "agent-version")" +build="$(buildkite-agent meta-data get "agent-version-build")" for variant in "alpine" "alpine-k8s" "ubuntu-18.04" "ubuntu-20.04" "ubuntu-22.04" "sidecar" ; do echo "--- Getting docker image tag for $variant from build meta data" - source_image=$(buildkite-agent meta-data get "agent-docker-image-$variant") - echo "Docker Image Tag for $variant: $source_image" + source_image="$(buildkite-agent meta-data get "agent-docker-image-${variant}")" + echo "Docker Image Tag for ${variant}: ${source_image}" - echo "--- :docker: Publishing images for $variant" - .buildkite/steps/publish-docker-image.sh "$variant" "$source_image" "$CODENAME" "$version" "$build" + echo "--- :docker: Publishing images for ${variant}" + .buildkite/steps/publish-docker-image.sh "${REGISTRY}" "${variant}" "${source_image}" "${CODENAME}" "${version}" "${build}" done