From 69034c34df803837720efc7ca2bf8e882b433b47 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 12 Dec 2024 22:20:41 -0800 Subject: [PATCH] test: allow testing frontends from different version This allows running Dockerfile tests so that the Dockerfile version and the BuildKit version are from different commits so that we can test that old Dockerfile releases remain compatible with the latest BuildKit. The tests are based on the commit of the Dockerfile frontend as we can't expect that new test would work on old frontends. In future we might consider doing it the other way as well but then we need a way to mark tests that can be ignored if they are not expected to pass because of a new feature dependency. Signed-off-by: Tonis Tiigi --- .github/workflows/.test.yml | 2 -- docker-bake.hcl | 32 +++++++++++++++++ .../cmd/dockerfile-frontend/hack/release | 2 ++ hack/images | 2 +- hack/test | 34 ++++++++----------- hack/util | 6 +--- 6 files changed, 51 insertions(+), 27 deletions(-) diff --git a/.github/workflows/.test.yml b/.github/workflows/.test.yml index 649ffbecf469..38f203272649 100644 --- a/.github/workflows/.test.yml +++ b/.github/workflows/.test.yml @@ -104,7 +104,6 @@ jobs: TESTFLAGS: "-v --parallel=6 --timeout=30m" GOTESTSUM_FORMAT: "standard-verbose" TEST_IMAGE_BUILD: "0" - TEST_IMAGE_ID: "buildkit-tests" strategy: fail-fast: false matrix: @@ -161,7 +160,6 @@ jobs: targets: integration-tests set: | *.cache-from=type=gha,scope=${{ inputs.cache_scope }} - *.output=type=docker,name=${{ env.TEST_IMAGE_ID }} env: BUILDKITD_TAGS: ${{ matrix.tags }} TEST_COVERAGE: 1 diff --git a/docker-bake.hcl b/docker-bake.hcl index 05e7a830e79d..67d9df39a09b 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -51,6 +51,24 @@ variable "TEST_COVERAGE" { default = null } +variable "TEST_IMAGE_NAME" { + default = "buildkit-tests" +} + +variable "TEST_CONTEXT" { + default = "." + description = "Context for building the test image" +} + +variable "TEST_BINARIES_CONTEXT" { + default = TEST_CONTEXT + description = "Context for building the buildkitd for test image" +} + +variable "BUILDKIT_SYNTAX" { + default = null +} + function "bindir" { params = [defaultdir] result = DESTDIR != "" ? DESTDIR : "./bin/${defaultdir}" @@ -117,12 +135,26 @@ target "integration-tests-base" { output = ["type=cacheonly"] } +target "integration-tests-binaries" { + inherits = ["_common"] + target = "binaries" + context = TEST_BINARIES_CONTEXT +} + target "integration-tests" { inherits = ["integration-tests-base"] target = "integration-tests" + context = TEST_CONTEXT + contexts = TEST_CONTEXT != TEST_BINARIES_CONTEXT ? { + "binaries" = "target:integration-tests-binaries" + } : null args = { GOBUILDFLAGS = TEST_COVERAGE == "1" ? "-cover" : null + BUILDKIT_SYNTAX = BUILDKIT_SYNTAX } + output = [ + "type=docker,name=${TEST_IMAGE_NAME}", + ] } group "validate" { diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/hack/release b/frontend/dockerfile/cmd/dockerfile-frontend/hack/release index ff28f9bf48b9..2d94fbd6eb20 100755 --- a/frontend/dockerfile/cmd/dockerfile-frontend/hack/release +++ b/frontend/dockerfile/cmd/dockerfile-frontend/hack/release @@ -104,6 +104,7 @@ case $TYP in buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags $nocacheFilterFlag $(buildAttestFlags) \ --build-arg "CHANNEL=$TAG" \ --build-arg "BUILDTAGS=$buildTags" \ + --build-arg "BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" \ --output "${outputFlag},name=$REPO:$pushTag" \ --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \ $currentcontext @@ -125,6 +126,7 @@ case $TYP in buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags $nocacheFilterFlag $(buildAttestFlags) \ --build-arg "CHANNEL=$TAG" \ --build-arg "BUILDTAGS=$buildTags" \ + --build-arg "BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" \ --output "$outputFlag" \ --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \ $currentcontext diff --git a/hack/images b/hack/images index 3de770f8f051..ce35ee226113 100755 --- a/hack/images +++ b/hack/images @@ -112,5 +112,5 @@ if [[ "$RELEASE" = "true" ]] && [[ "$GITHUB_ACTIONS" = "true" ]]; then nocacheFilterFlag="--no-cache-filter=buildkit-export,gobuild-base,rootless" fi -buildxCmd build --build-arg BUILDKIT_DEBUG $platformFlag $targetFlag $importCacheFlags $exportCacheFlags $tagFlags $outputFlag $nocacheFilterFlag $attestFlags \ +buildxCmd build --build-arg "BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" --build-arg BUILDKIT_DEBUG $platformFlag $targetFlag $importCacheFlags $exportCacheFlags $tagFlags $outputFlag $nocacheFilterFlag $attestFlags \ $currentcontext diff --git a/hack/test b/hack/test index fbc7759b0d0d..dbb02f3247ed 100755 --- a/hack/test +++ b/hack/test @@ -15,7 +15,7 @@ set -eu -o pipefail : "${TEST_COVERAGE=}" : "${TEST_IMAGE_BUILD=1}" -: "${TEST_IMAGE_ID=buildkit-tests}" +: "${TEST_IMAGE_NAME=buildkit-tests}" : "${TEST_INTEGRATION=}" : "${TEST_GATEWAY=}" : "${TEST_DOCKERFILE=}" @@ -23,6 +23,7 @@ set -eu -o pipefail : "${TEST_DOCKERD_BINARY=$(which dockerd)}" : "${TEST_REPORT_SUFFIX=}" : "${TEST_KEEP_CACHE=}" +: "${TEST_SUITE_CONTEXT=}" : "${TESTFLAGS=}" : "${DOCKERFILE_RELEASES=}" @@ -87,20 +88,15 @@ if [ "$TEST_COVERAGE" == "1" ]; then export GO_TEST_COVERPROFILE="/testreports/coverage-report$TEST_REPORT_SUFFIX.txt" fi +if [ -n "$TEST_SUITE_CONTEXT" ]; then + export TEST_BINARIES_CONTEXT=$currentcontext + # FIXME: something breaks with the syntax when using the context + export BUILDKIT_SYNTAX="docker/dockerfile:1.10.0" + currentcontext=$TEST_SUITE_CONTEXT +fi + if [ "$TEST_IMAGE_BUILD" = "1" ]; then - buildxCmd build $cacheFromFlags \ - --build-arg ALPINE_VERSION \ - --build-arg GO_VERSION \ - --build-arg BUILDKITD_TAGS \ - --build-arg HTTP_PROXY \ - --build-arg HTTPS_PROXY \ - --build-arg NO_PROXY \ - --build-arg GOBUILDFLAGS \ - --build-arg VERIFYFLAGS \ - --build-arg CGO_ENABLED \ - --target "integration-tests" \ - --output "type=docker,name=$TEST_IMAGE_ID" \ - $currentcontext + TEST_CONTEXT=$currentcontext buildxCmd bake integration-tests fi cacheVolume="buildkit-test-cache" @@ -131,7 +127,7 @@ if [ "$TEST_INTEGRATION" == 1 ]; then cid=$(dockerCmd create $baseCreateFlags \ ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} \ -e SKIP_INTEGRATION_TESTS \ - $TEST_IMAGE_ID \ + $TEST_IMAGE_NAME \ gotestsumandcover $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v}) if [ "$TEST_DOCKERD" = "1" ]; then dockerCmd cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd @@ -141,8 +137,8 @@ fi if [ "$TEST_GATEWAY" == 1 ]; then # Build-test "github.com/moby/buildkit/frontend/gateway/client", which isn't otherwise built by CI - # It really only needs buildkit-base. We have integration-tests in $TEST_IMAGE_ID, which is a direct child of buildkit-base. - cid=$(dockerCmd create --rm --volumes-from=$cacheVolume --entrypoint="" $TEST_IMAGE_ID go build -v ./frontend/gateway/client) + # It really only needs buildkit-base. We have integration-tests in $TEST_IMAGE_NAME, which is a direct child of buildkit-base. + cid=$(dockerCmd create --rm --volumes-from=$cacheVolume --entrypoint="" $TEST_IMAGE_NAME go build -v ./frontend/gateway/client) dockerCmd start -a $cid fi @@ -165,7 +161,7 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then tarout=$(mktemp -t dockerfile-frontend.XXXXXXXXXX) buildxCmd build $cacheFromFlags \ - --build-arg "BUILDTAGS=$buildtags" \ + --build-arg "BUILDTAGS=$buildtags" --build-arg "BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" \ --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \ --output "type=oci,dest=$tarout" \ $currentcontext @@ -176,7 +172,7 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then -e BUILDKIT_WORKER_RANDOM \ -e FRONTEND_GATEWAY_ONLY=local:/$release.tar \ -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend \ - $TEST_IMAGE_ID \ + $TEST_IMAGE_NAME \ gotestsumandcover $gotestsumArgs --packages=./frontend/dockerfile -- $gotestArgs --count=1 -tags "$buildtags" ${TESTFLAGS:--v}) dockerCmd cp $tarout $cid:/$release.tar if [ "$TEST_DOCKERD" = "1" ]; then diff --git a/hack/util b/hack/util index efd56b2d53e0..758353080d94 100755 --- a/hack/util +++ b/hack/util @@ -36,12 +36,11 @@ buildAttestFlags() { fi } -currentref="" currentcontext="." cacheFromFlags="" cacheToFlags="" if [ "$GITHUB_ACTIONS" = "true" ] && [ "$GITHUB_REPOSITORY" = "moby/buildkit" ]; then - currentref="https://github.com/$GITHUB_REPOSITORY.git#$GITHUB_REF" + currentcontext="https://github.com/$GITHUB_REPOSITORY.git#$GITHUB_REF" if [ -n "$CACHE_FROM" ]; then for cfrom in $CACHE_FROM; do if [[ $cfrom == *"type=gha"* ]]; then @@ -69,9 +68,6 @@ if [ "$GITHUB_ACTIONS" = "true" ] && [ "$GITHUB_REPOSITORY" = "moby/buildkit" ]; done fi fi -if [ -n "$currentref" ]; then - currentcontext="--build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=1 $currentref" -fi if [ -n "$CONTEXT" ]; then currentcontext=$CONTEXT fi