From 4cb2c64fe6ab761fdf0cb130645b9780c9a0d770 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 22 Jan 2025 14:41:52 +0100 Subject: [PATCH] support additional_context reference to another service Signed-off-by: Nicolas De Loof --- go.mod | 2 +- go.sum | 4 +- pkg/compose/build.go | 27 +++++- pkg/compose/build_bake.go | 89 ++++++++++++------- pkg/e2e/build_test.go | 17 ++++ pkg/e2e/fixtures/build-dependent/Dockerfile.a | 2 + pkg/e2e/fixtures/build-dependent/Dockerfile.b | 2 + pkg/e2e/fixtures/build-dependent/compose.yaml | 15 ++++ 8 files changed, 120 insertions(+), 38 deletions(-) create mode 100644 pkg/e2e/fixtures/build-dependent/Dockerfile.a create mode 100644 pkg/e2e/fixtures/build-dependent/Dockerfile.b create mode 100644 pkg/e2e/fixtures/build-dependent/compose.yaml diff --git a/go.mod b/go.mod index 08203e87907..771e0f127f0 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Microsoft/go-winio v0.6.2 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/buger/goterm v1.0.4 - github.com/compose-spec/compose-go/v2 v2.4.7 + github.com/compose-spec/compose-go/v2 v2.4.8-0.20250122084341-25e1083beabb github.com/containerd/containerd v1.7.24 github.com/containerd/platforms v0.2.1 github.com/davecgh/go-spew v1.1.1 diff --git a/go.sum b/go.sum index 031598eabe2..e0e670403d1 100644 --- a/go.sum +++ b/go.sum @@ -79,8 +79,8 @@ github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004 h1:lkAMpLVBDaj17e github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= -github.com/compose-spec/compose-go/v2 v2.4.7 h1:WNpz5bIbKG+G+w9pfu72B1ZXr+Og9jez8TMEo8ecXPk= -github.com/compose-spec/compose-go/v2 v2.4.7/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= +github.com/compose-spec/compose-go/v2 v2.4.8-0.20250122084341-25e1083beabb h1:TDtkmu7WlsYNwapWsB6fV7pFB7rCCDLFbZQykZ+lO5A= +github.com/compose-spec/compose-go/v2 v2.4.8-0.20250122084341-25e1083beabb/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= diff --git a/pkg/compose/build.go b/pkg/compose/build.go index 8a8b04d2c45..b20e2928a61 100644 --- a/pkg/compose/build.go +++ b/pkg/compose/build.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "os" + "strings" "github.com/compose-spec/compose-go/v2/types" "github.com/containerd/platforms" @@ -71,7 +72,26 @@ func (s *composeService) build(ctx context.Context, project *types.Project, opti if options.Deps { policy = types.IncludeDependencies } - err := project.ForEachService(options.Services, func(serviceName string, service *types.ServiceConfig) error { + + project, err := project.WithServicesTransform(func(serviceName string, service types.ServiceConfig) (types.ServiceConfig, error) { + // drop runtime dependencies, unrelated to build ordering + service.DependsOn = types.DependsOnConfig{} + if service.Build != nil { + for _, c := range service.Build.AdditionalContexts { + if t, found := strings.CutPrefix(c, types.ServicePrefix); found { + service.DependsOn[t] = types.ServiceDependency{ + Condition: "build", // non-canonical, but will force dependency graph ordering + } + } + } + } + return service, nil + }) + if err != nil { + return imageIDs, err + } + + err = project.ForEachService(options.Services, func(serviceName string, service *types.ServiceConfig) error { if service.Build == nil { return nil } @@ -536,6 +556,11 @@ func getImageBuildLabels(project *types.Project, service types.ServiceConfig) ty func toBuildContexts(additionalContexts types.Mapping) map[string]build.NamedContext { namedContexts := map[string]build.NamedContext{} for name, contextPath := range additionalContexts { + if _, found := strings.CutPrefix(contextPath, types.ServicePrefix); found { + // image we depend on has been build previously, as we run in dependency order. + // this assumes use of docker engine builder, so that build can access local images + continue + } namedContexts[name] = build.NamedContext{Path: contextPath} } return namedContexts diff --git a/pkg/compose/build_bake.go b/pkg/compose/build_bake.go index 2ff8c6b29cb..3da900c3eef 100644 --- a/pkg/compose/build_bake.go +++ b/pkg/compose/build_bake.go @@ -93,19 +93,27 @@ type bakeGroup struct { } type bakeTarget struct { - Context string `json:"context,omitempty"` - Dockerfile string `json:"dockerfile,omitempty"` - Args map[string]string `json:"args,omitempty"` - Labels map[string]string `json:"labels,omitempty"` - Tags []string `json:"tags,omitempty"` - CacheFrom []string `json:"cache-from,omitempty"` - CacheTo []string `json:"cache-to,omitempty"` - Secrets []string `json:"secret,omitempty"` - SSH []string `json:"ssh,omitempty"` - Platforms []string `json:"platforms,omitempty"` - Target string `json:"target,omitempty"` - Pull bool `json:"pull,omitempty"` - NoCache bool `json:"no-cache,omitempty"` + Context string `json:"context,omitempty"` + Contexts map[string]string `json:"contexts,omitempty"` + Dockerfile string `json:"dockerfile,omitempty"` + DockerfileInline string `json:"dockerfile-inline,omitempty"` + Args map[string]string `json:"args,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Tags []string `json:"tags,omitempty"` + CacheFrom []string `json:"cache-from,omitempty"` + CacheTo []string `json:"cache-to,omitempty"` + Target string `json:"target,omitempty"` + Secrets []string `json:"secret,omitempty"` + SSH []string `json:"ssh,omitempty"` + Platforms []string `json:"platforms,omitempty"` + Pull bool `json:"pull,omitempty"` + NoCache bool `json:"no-cache,omitempty"` + NetworkMode string `json:"network,omitempty"` + NoCacheFilter []string `json:"no-cache-filter,omitempty"` + ShmSize types.UnitBytes `json:"shm-size,omitempty"` + Ulimits []string `json:"ulimits,omitempty"` + Call string `json:"call,omitempty"` + Entitlements []string `json:"entitlements,omitempty"` } type bakeMetadata map[string]buildStatus @@ -115,11 +123,6 @@ type buildStatus struct { } func (s *composeService) doBuildBake(ctx context.Context, project *types.Project, serviceToBeBuild types.Services, options api.BuildOptions) (map[string]string, error) { //nolint:gocyclo - cw := progress.ContextWriter(ctx) - for name := range serviceToBeBuild { - cw.Event(progress.BuildingEvent(name)) - } - eg := errgroup.Group{} ch := make(chan *client.SolveStatus) display, err := progressui.NewDisplay(os.Stdout, progressui.DisplayMode(options.Progress)) @@ -137,7 +140,7 @@ func (s *composeService) doBuildBake(ctx context.Context, project *types.Project } var group bakeGroup - for _, service := range serviceToBeBuild { + for name, service := range serviceToBeBuild { if service.Build == nil { continue } @@ -153,23 +156,29 @@ func (s *composeService) doBuildBake(ctx context.Context, project *types.Project image := api.GetImageNameOrDefault(service, project.Name) - cfg.Targets[image] = bakeTarget{ - Context: build.Context, - Dockerfile: dockerFilePath(build.Context, build.Dockerfile), - Args: args, - Labels: build.Labels, - Tags: append(build.Tags, image), - - CacheFrom: build.CacheFrom, + cfg.Targets[name] = bakeTarget{ + Context: build.Context, + Contexts: additionalContexts(build.AdditionalContexts), + Dockerfile: dockerFilePath(build.Context, build.Dockerfile), + DockerfileInline: build.DockerfileInline, + Args: args, + Labels: build.Labels, + Tags: append(build.Tags, image), + CacheFrom: build.CacheFrom, // CacheTo: TODO - Platforms: build.Platforms, - Target: build.Target, - Secrets: toBakeSecrets(project, build.Secrets), - SSH: toBakeSSH(build.SSH), - Pull: options.Pull, - NoCache: options.NoCache, + Target: build.Target, + Secrets: toBakeSecrets(project, build.Secrets), + SSH: toBakeSSH(build.SSH), + Platforms: build.Platforms, + Pull: options.Pull, + NoCache: options.NoCache, + NetworkMode: build.Network, + ShmSize: build.ShmSize, + // Ulimits: TODO + // Call: TODO + // Entitlements: TODO } - group.Targets = append(group.Targets, image) + group.Targets = append(group.Targets, name) } cfg.Groups["default"] = group @@ -250,6 +259,7 @@ func (s *composeService) doBuildBake(ctx context.Context, project *types.Project return nil, err } + cw := progress.ContextWriter(ctx) results := map[string]string{} for name, m := range md { results[name] = m.Digest @@ -258,6 +268,17 @@ func (s *composeService) doBuildBake(ctx context.Context, project *types.Project return results, nil } +func additionalContexts(contexts types.Mapping) map[string]string { + ac := map[string]string{} + for k, c := range contexts { + if target, found := strings.CutPrefix(c, types.ServicePrefix); found { + c = "target:" + target + } + ac[k] = c + } + return ac +} + func toBakeSSH(ssh types.SSHConfig) []string { var s []string for _, key := range ssh { diff --git a/pkg/e2e/build_test.go b/pkg/e2e/build_test.go index 3e19f2acb28..03650a523b5 100644 --- a/pkg/e2e/build_test.go +++ b/pkg/e2e/build_test.go @@ -498,3 +498,20 @@ func TestBuildEntitlements(t *testing.T) { } }) } + +func TestBuildDependent(t *testing.T) { + c := NewCLI(t) + + for _, env := range []string{"COMPOSE_BAKE=0", "COMPOSE-BAKE=1"} { + t.Run(env, func(t *testing.T) { + cmd := c.NewDockerComposeCmd(t, "-f", "fixtures/build-dependent/compose.yaml", "build") + res := icmd.RunCmd(cmd, func(cmd *icmd.Cmd) { + cmd.Env = append(cmd.Env, env) + }) + res.Assert(t, icmd.Success) + + c.RunDockerCmd(t, "rmi", "--force", "a") + c.RunDockerCmd(t, "rmi", "--force", "b") + }) + } +} diff --git a/pkg/e2e/fixtures/build-dependent/Dockerfile.a b/pkg/e2e/fixtures/build-dependent/Dockerfile.a new file mode 100644 index 00000000000..0260ffde977 --- /dev/null +++ b/pkg/e2e/fixtures/build-dependent/Dockerfile.a @@ -0,0 +1,2 @@ +FROM nginx +RUN echo hello > /hello.txt \ No newline at end of file diff --git a/pkg/e2e/fixtures/build-dependent/Dockerfile.b b/pkg/e2e/fixtures/build-dependent/Dockerfile.b new file mode 100644 index 00000000000..ddc59193ab0 --- /dev/null +++ b/pkg/e2e/fixtures/build-dependent/Dockerfile.b @@ -0,0 +1,2 @@ +FROM a +RUN echo hello > /hello2.txt \ No newline at end of file diff --git a/pkg/e2e/fixtures/build-dependent/compose.yaml b/pkg/e2e/fixtures/build-dependent/compose.yaml new file mode 100644 index 00000000000..340494ed033 --- /dev/null +++ b/pkg/e2e/fixtures/build-dependent/compose.yaml @@ -0,0 +1,15 @@ +services: + a: + image: a + build: + context: . + dockerfile: Dockerfile.a + + b: + image: b + build: + context: . + dockerfile: Dockerfile.b + additional_contexts: + a: service:a +