Multi-platform caching for buildx

examples/aws-container-registry/ts/index.ts

@@ -51,6 +51,12 @@ const buildxImage = new docker.buildx.Image("buildx", {
   exports: ["type=registry"],
   file: "app/Dockerfile",
   platforms: ["linux/arm64", "linux/amd64"],
+  cacheTo: [
+    pulumi.interpolate`type=registry,mode=max,image-manifest=true,oci-mediatypes=true,ref=${repo.repositoryUrl}:cache`,
+  ],
+  cacheFrom: [
+    pulumi.interpolate`type=registry,ref=${repo.repositoryUrl}:cache`,
+  ],
   context: "app",
   registries: [
     {

examples/docker-container-registry/ts/index.ts

@@ -33,6 +33,8 @@ const buildxImage = new docker.buildx.Image("my-buildx-image", {
   tags: [`${imageName}:buildx`],
   exports: ["type=registry"],
   platforms: ["linux/arm64", "linux/amd64"],
+  cacheFrom: ["type=gha", `type=registry,ref=docker.io/${imageName}`],
+  cacheTo: ["type=gha", `type=registry,ref=docker.io/${imageName},mode=max`],
   context: "app",
   file: "app/Dockerfile",
   registries: [

examples/examples_nodejs_test.go

@@ -19,10 +19,16 @@ package examples
 import (
 	"encoding/base64"
 	"encoding/json"
+	"fmt"
+	"math/rand"
 	"os"
 	"path"
+	"strings"
 	"testing"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ecr"
 	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
 	"github.com/stretchr/testify/assert"
 )
@@ -239,6 +245,116 @@ func TestLocalRepoDigestNode(t *testing.T) {
 	integration.ProgramTest(t, &test)
 }
 
+// TestBuildxCaching simulates a slow multi-platform build with --cache-to
+// enabled. We aren't able to directly detect cache hits, so we re-run the
+// update and confirm it took less time than the image originally took to
+// build.
+//
+// This is a moderately slow test because we need to "build" (i.e., sleep)
+// longer than it would take for cache layer uploads under slow network
+// conditions.
+func TestBuildxCaching(t *testing.T) {
+	t.Parallel()
+
+	sleep := 20.0 // seconds
+
+	// Provision ECR outside of our stack, because the cache needs to be shared
+	// across updates.
+	ecr, ecrOK := tmpEcr(t)
+
+	localCache := t.TempDir()
+
+	tests := []struct {
+		name string
+		skip bool
+
+		cacheTo   string
+		cacheFrom string
+		address   string
+		username  string
+		password  string
+	}{
+		{
+			name:      "local",
+			cacheTo:   fmt.Sprintf("type=local,mode=max,oci-mediatypes=true,dest=%s", localCache),
+			cacheFrom: fmt.Sprintf("type=local,src=%s", localCache),
+		},
+		{
+			name:      "gha",
+			skip:      os.Getenv("ACTIONS_CACHE_URL") == "",
+			cacheTo:   "type=gha,mode=max,scope=cache-test",
+			cacheFrom: "type=gha,scope=cache-test",
+		},
+		{
+			name:      "dockerhub",
+			skip:      os.Getenv("DOCKER_HUB_PASSWORD") == "",
+			cacheTo:   "type=registry,mode=max,ref=docker.io/pulumibot/myapp:cache",
+			cacheFrom: "type=registry,ref=docker.io/pulumibot/myapp:cache",
+			address:   "docker.io",
+			username:  "pulumibot",
+			password:  os.Getenv("DOCKER_HUB_PASSWORD"),
+		},
+		{
+			name:      "ecr",
+			skip:      !ecrOK,
+			cacheTo:   fmt.Sprintf("type=registry,mode=max,image-manifest=true,oci-mediatypes=true,ref=%s:cache", ecr.address),
+			cacheFrom: fmt.Sprintf("type=registry,ref=%s:cache", ecr.address),
+			address:   ecr.address,
+			username:  ecr.username,
+			password:  ecr.password,
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.skip {
+				t.Skip("Missing environment variables")
+			}
+
+			sleepFuzzed := sleep + rand.Float64() // Add some fuzz to bust any prior build caches.
+
+			test := getJsOptions(t).
+				With(integration.ProgramTestOptions{
+					Dir: path.Join(getCwd(t), "test-buildx", "caching", "ts"),
+					ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
+						duration, ok := stack.Outputs["durationSeconds"]
+						assert.True(t, ok)
+						assert.Greater(t, duration.(float64), sleepFuzzed)
+					},
+					Config: map[string]string{
+						"SLEEP_SECONDS": fmt.Sprint(sleepFuzzed),
+						"cacheTo":       tt.cacheTo,
+						"cacheFrom":     tt.cacheFrom,
+						"name":          tt.name,
+						"address":       tt.address,
+						"username":      tt.username,
+					},
+					Secrets: map[string]string{
+						"password": tt.password,
+					},
+					NoParallel:  true,
+					Quick:       true,
+					SkipPreview: true,
+					SkipRefresh: true,
+					Verbose:     true,
+				})
+
+			// First run should be un-cached.
+			integration.ProgramTest(t, &test)
+
+			// Now run again and confirm our build was faster due to a cache hit.
+			test.ExtraRuntimeValidation = func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
+				duration, ok := stack.Outputs["durationSeconds"]
+				assert.True(t, ok)
+				assert.Less(t, duration.(float64), sleepFuzzed)
+			}
+			test.Config["name"] += "-cached"
+			integration.ProgramTest(t, &test)
+		})
+	}
+}
+
 func getJsOptions(t *testing.T) integration.ProgramTestOptions {
 	base := getBaseOptions()
 	baseJs := base.With(integration.ProgramTestOptions{
@@ -249,3 +365,68 @@ func getJsOptions(t *testing.T) integration.ProgramTestOptions {
 
 	return baseJs
 }
+
+type ECR struct {
+	address  string
+	username string
+	password string
+}
+
+// tmpEcr creates a new ECR repo and cleans it up after the test concludes.
+func tmpEcr(t *testing.T) (ECR, bool) {
+	sess, err := session.NewSession(&aws.Config{
+		Region: aws.String("us-west-2"),
+	})
+	if err != nil {
+		return ECR{}, false
+	}
+
+	svc := ecr.New(sess)
+	name := strings.ToLower(t.Name())
+
+	// Always attempt to delete pre-existing repos, in case our cleanup didn't
+	// run.
+	_, _ = svc.DeleteRepository(&ecr.DeleteRepositoryInput{
+		Force:          aws.Bool(true),
+		RepositoryName: aws.String(name),
+	})
+
+	params := &ecr.CreateRepositoryInput{
+		RepositoryName: aws.String(name),
+	}
+	resp, err := svc.CreateRepository(params)
+	if err != nil {
+		return ECR{}, false
+	}
+	repo := resp.Repository
+	t.Cleanup(func() {
+		svc.DeleteRepository(&ecr.DeleteRepositoryInput{
+			Force:          aws.Bool(true),
+			RegistryId:     repo.RegistryId,
+			RepositoryName: repo.RepositoryName,
+		})
+	})
+
+	// Now grab auth for the repo.
+	auth, err := svc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
+	if err != nil {
+		return ECR{}, false
+	}
+	b64token := auth.AuthorizationData[0].AuthorizationToken
+	token, err := base64.StdEncoding.DecodeString(*b64token)
+	if err != nil {
+		return ECR{}, false
+	}
+	parts := strings.SplitN(string(token), ":", 2)
+	if len(parts) != 2 {
+		return ECR{}, false
+	}
+	username := parts[0]
+	password := parts[1]
+
+	return ECR{
+		address:  *repo.RepositoryUri,
+		username: username,
+		password: password,
+	}, true
+}

examples/go.mod

@@ -3,6 +3,7 @@ module github.com/pulumi/pulumi-docker/examples
 go 1.21
 
 require (
+	github.com/aws/aws-sdk-go v1.49.0
 	github.com/docker/docker v24.0.7+incompatible
 	github.com/pulumi/pulumi/pkg/v3 v3.107.0
 	github.com/stretchr/testify v1.8.4
@@ -34,7 +35,6 @@ require (
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/armon/go-radix v1.0.0 // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
-	github.com/aws/aws-sdk-go v1.49.0 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.24.0 // indirect
 	github.com/aws/aws-sdk-go-v2/config v1.26.1 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.16.12 // indirect
@@ -133,7 +133,7 @@ require (
 	github.com/natefinch/atomic v1.0.1 // indirect
 	github.com/oklog/run v1.1.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.2 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
 	github.com/opentracing/basictracer-go v1.1.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pgavlin/fx v0.1.6 // indirect

examples/go.sum

@@ -1693,8 +1693,9 @@ github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3I
 github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/image-spec v1.0.2-0.20211117181255-693428a734f5/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
 github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8=
+github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
 github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=

examples/test-buildx/caching/ts/Dockerfile

@@ -0,0 +1,7 @@
+FROM --platform=$BUILDPLATFORM golang:1.21.6-alpine3.18 as initial
+ARG SLEEP_SECONDS
+RUN sleep ${SLEEP_SECONDS} && echo ${SLEEP_SECONDS} > output
+
+FROM alpine:3.18 as final
+COPY --from=initial /go/output output
+RUN cat output

examples/test-buildx/caching/ts/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: test-buildx-caching
+runtime: nodejs
+description: A minimal TypeScript Pulumi program

examples/test-buildx/caching/ts/index.ts

@@ -0,0 +1,44 @@
+import * as docker from "@pulumi/docker";
+import * as pulumi from "@pulumi/pulumi";
+
+const config = new pulumi.Config();
+
+const start = new Date().getTime();
+
+// docker buildx build \
+//  -f Dockerfile \
+//  --cache-to type=local,dest=tmp,mode=max,oci-mediatypes=true \
+//  --cache-from type=local,src=tmp \
+//  --build-arg SLEEP-MS=$SLEEP_MS \
+//  -t not-pushed \
+//  -f Dockerfile \
+//  --platform linux/arm64 \
+//  --platform linux/amd64 \
+//  .
+const img = new docker.buildx.Image(`buildx-${config.require("name")}`, {
+  tags: ["not-pushed"],
+  file: "Dockerfile",
+  context: ".",
+  platforms: ["linux/arm64", "linux/amd64"],
+  buildArgs: {
+    SLEEP_SECONDS: config.require("SLEEP_SECONDS"),
+  },
+  cacheTo: [config.require("cacheTo")],
+  cacheFrom: [config.require("cacheFrom")],
+  // Set registry auth if it was provided.
+  registries: config.getSecret("username").apply((a) =>
+    a
+      ? [
+          {
+            address: config.getSecret("address"),
+            username: config.getSecret("username"),
+            password: config.getSecret("password"),
+          },
+        ]
+      : undefined
+  ),
+});
+
+export const durationSeconds = img.manifests.apply(
+  (_) => (new Date().getTime() - start) / 1000.0
+);

examples/test-buildx/caching/ts/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "test-buildx-caching",
+  "devDependencies": {
+    "@types/node": "^20.0.0"
+  },
+  "dependencies": {
+    "@pulumi/pulumi": "^3.0.0"
+  }
+}