Merge pull request #2841 from tonistiigi/outline

dockerfile: support for outline requests

cmd/buildctl/build.go

@@ -4,15 +4,18 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"io"
 	"os"
+	"strings"
 
 	"github.com/containerd/continuity"
 	"github.com/docker/cli/cli/config"
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/cmd/buildctl/build"
 	bccommon "github.com/moby/buildkit/cmd/buildctl/common"
+	gateway "github.com/moby/buildkit/frontend/gateway/client"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/auth/authprovider"
 	"github.com/moby/buildkit/session/sshforward/sshprovider"
@@ -262,13 +265,40 @@ func buildAction(clicontext *cli.Context) error {
 		}
 	}
 
+	var subMetadata map[string][]byte
+
 	eg.Go(func() error {
 		defer func() {
 			for _, w := range writers {
 				close(w.Status())
 			}
 		}()
-		resp, err := c.Solve(ctx, def, solveOpt, progresswriter.ResetTime(mw.WithPrefix("", false)).Status())
+		sreq := gateway.SolveRequest{
+			Frontend:    solveOpt.Frontend,
+			FrontendOpt: solveOpt.FrontendAttrs,
+		}
+		if def != nil {
+			sreq.Definition = def.ToPB()
+		}
+		solveOpt.Frontend = ""
+		solveOpt.FrontendAttrs = nil
+
+		resp, err := c.Build(ctx, solveOpt, "buildctl", func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
+			_, isSubRequest := sreq.FrontendOpt["requestid"]
+			if isSubRequest {
+				if _, ok := sreq.FrontendOpt["frontend.caps"]; !ok {
+					sreq.FrontendOpt["frontend.caps"] = "moby.buildkit.frontend.subrequests"
+				}
+			}
+			res, err := c.Solve(ctx, sreq)
+			if err != nil {
+				return nil, err
+			}
+			if isSubRequest && res != nil {
+				subMetadata = res.Metadata
+			}
+			return res, err
+		}, progresswriter.ResetTime(mw.WithPrefix("", false)).Status())
 		if err != nil {
 			return err
 		}
@@ -291,7 +321,20 @@ func buildAction(clicontext *cli.Context) error {
 		return pw.Err()
 	})
 
-	return eg.Wait()
+	if err := eg.Wait(); err != nil {
+		return err
+	}
+
+	if txt, ok := subMetadata["result.txt"]; ok {
+		fmt.Print(string(txt))
+	} else {
+		for k, v := range subMetadata {
+			if strings.HasPrefix(k, "result.") {
+				fmt.Printf("%s\n%s\n", k, v)
+			}
+		}
+	}
+	return nil
 }
 
 func writeMetadataFile(filename string, exporterResponse map[string]string) error {

frontend/dockerfile/builder/build.go

@@ -24,6 +24,8 @@ import (
 	"github.com/moby/buildkit/frontend/dockerfile/parser"
 	"github.com/moby/buildkit/frontend/gateway/client"
 	gwpb "github.com/moby/buildkit/frontend/gateway/pb"
+	"github.com/moby/buildkit/frontend/subrequests/outline"
+	"github.com/moby/buildkit/frontend/subrequests/targets"
 	"github.com/moby/buildkit/solver/errdefs"
 	"github.com/moby/buildkit/solver/pb"
 	binfotypes "github.com/moby/buildkit/util/buildinfo/types"
@@ -60,6 +62,7 @@ const (
 	keyShmSize          = "shm-size"
 	keyTargetPlatform   = "platform"
 	keyUlimit           = "ulimit"
+	keyRequestID        = "requestid"
 
 	// Don't forget to update frontend documentation if you add
 	// a new build-arg: frontend/dockerfile/docs/reference.md
@@ -72,7 +75,7 @@ const (
 
 var httpPrefix = regexp.MustCompile(`^https?://`)
 
-func Build(ctx context.Context, c client.Client) (*client.Result, error) {
+func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) {
 	opts := c.BuildOpts().Opts
 	caps := c.BuildOpts().LLBCaps
 	gwcaps := c.BuildOpts().Caps
@@ -420,49 +423,72 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 		opts[keyHostname] = v
 	}
 
+	convertOpt := dockerfile2llb.ConvertOpt{
+		Target:           opts[keyTarget],
+		MetaResolver:     c,
+		BuildArgs:        filter(opts, buildArgPrefix),
+		Labels:           filter(opts, labelPrefix),
+		CacheIDNamespace: opts[keyCacheNSArg],
+		SessionID:        c.BuildOpts().SessionID,
+		BuildContext:     buildContext,
+		Excludes:         excludes,
+		IgnoreCache:      ignoreCache,
+		TargetPlatform:   targetPlatforms[0],
+		BuildPlatforms:   buildPlatforms,
+		ImageResolveMode: resolveMode,
+		PrefixPlatform:   exportMap,
+		ExtraHosts:       extraHosts,
+		ShmSize:          shmSize,
+		Ulimit:           ulimit,
+		CgroupParent:     opts[keyCgroupParent],
+		ForceNetMode:     defaultNetMode,
+		LLBCaps:          &caps,
+		SourceMap:        sourceMap,
+		Hostname:         opts[keyHostname],
+		Warn: func(msg, url string, detail [][]byte, location *parser.Range) {
+			c.Warn(ctx, defVtx, msg, warnOpts(sourceMap, location, detail, url))
+		},
+		ContextByName: contextByNameFunc(c, c.BuildOpts().SessionID, targetPlatforms[0]),
+	}
+
+	defer func() {
+		var el *parser.ErrorLocation
+		if errors.As(err, &el) {
+			err = wrapSource(err, sourceMap, el.Location)
+		}
+	}()
+
+	if req, ok := opts[keyRequestID]; ok {
+		switch req {
+		case outline.SubrequestsOutlineDefinition.Name:
+			o, err := dockerfile2llb.Dockefile2Outline(ctx, dtDockerfile, convertOpt)
+			if err != nil {
+				return nil, err
+			}
+			return o.ToResult()
+		case targets.SubrequestsTargetsDefinition.Name:
+			targets, err := dockerfile2llb.ListTargets(ctx, dtDockerfile)
+			if err != nil {
+				return nil, err
+			}
+			return targets.ToResult()
+		default:
+			return nil, errdefs.NewUnsupportedSubrequestError(req)
+		}
+	}
+
 	eg, ctx = errgroup.WithContext(ctx)
 
 	for i, tp := range targetPlatforms {
 		func(i int, tp *ocispecs.Platform) {
 			eg.Go(func() (err error) {
-				defer func() {
-					var el *parser.ErrorLocation
-					if errors.As(err, &el) {
-						err = wrapSource(err, sourceMap, el.Location)
-					}
-				}()
-
-				st, img, bi, err := dockerfile2llb.Dockerfile2LLB(ctx, dtDockerfile, dockerfile2llb.ConvertOpt{
-					Target:           opts[keyTarget],
-					MetaResolver:     c,
-					BuildArgs:        filter(opts, buildArgPrefix),
-					Labels:           filter(opts, labelPrefix),
-					CacheIDNamespace: opts[keyCacheNSArg],
-					SessionID:        c.BuildOpts().SessionID,
-					BuildContext:     buildContext,
-					Excludes:         excludes,
-					IgnoreCache:      ignoreCache,
-					TargetPlatform:   tp,
-					BuildPlatforms:   buildPlatforms,
-					ImageResolveMode: resolveMode,
-					PrefixPlatform:   exportMap,
-					ExtraHosts:       extraHosts,
-					ShmSize:          shmSize,
-					Ulimit:           ulimit,
-					CgroupParent:     opts[keyCgroupParent],
-					ForceNetMode:     defaultNetMode,
-					LLBCaps:          &caps,
-					SourceMap:        sourceMap,
-					Hostname:         opts[keyHostname],
-					Warn: func(msg, url string, detail [][]byte, location *parser.Range) {
-						if i != 0 {
-							return
-						}
-						c.Warn(ctx, defVtx, msg, warnOpts(sourceMap, location, detail, url))
-					},
-					ContextByName: contextByNameFunc(c, c.BuildOpts().SessionID, tp),
-				})
-
+				opt := convertOpt
+				opt.TargetPlatform = tp
+				if i != 0 {
+					opt.Warn = nil
+				}
+				opt.ContextByName = contextByNameFunc(c, c.BuildOpts().SessionID, tp)
+				st, img, bi, err := dockerfile2llb.Dockerfile2LLB(ctx, dtDockerfile, opt)
 				if err != nil {
 					return err
 				}

frontend/dockerfile/builder/subrequests.go

@@ -1,39 +1,54 @@
 package builder
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 
 	"github.com/moby/buildkit/frontend/gateway/client"
 	"github.com/moby/buildkit/frontend/subrequests"
+	"github.com/moby/buildkit/frontend/subrequests/outline"
+	"github.com/moby/buildkit/frontend/subrequests/targets"
 	"github.com/moby/buildkit/solver/errdefs"
 )
 
 func checkSubRequest(ctx context.Context, opts map[string]string) (*client.Result, bool, error) {
-	req, ok := opts["requestid"]
+	req, ok := opts[keyRequestID]
 	if !ok {
 		return nil, false, nil
 	}
 	switch req {
 	case subrequests.RequestSubrequestsDescribe:
 		res, err := describe()
 		return res, true, err
+	case outline.RequestSubrequestsOutline, targets.RequestTargets: // handled later
+		return nil, false, nil
 	default:
 		return nil, true, errdefs.NewUnsupportedSubrequestError(req)
 	}
 }
 
 func describe() (*client.Result, error) {
 	all := []subrequests.Request{
+		outline.SubrequestsOutlineDefinition,
+		targets.SubrequestsTargetsDefinition,
 		subrequests.SubrequestsDescribeDefinition,
 	}
-	dt, err := json.MarshalIndent(all, "  ", "")
+	dt, err := json.MarshalIndent(all, "", "  ")
 	if err != nil {
 		return nil, err
 	}
+
+	b := bytes.NewBuffer(nil)
+	if err := subrequests.PrintDescribe(dt, b); err != nil {
+		return nil, err
+	}
+
 	res := client.NewResult()
 	res.Metadata = map[string][]byte{
 		"result.json": dt,
+		"result.txt":  b.Bytes(),
+		"version":     []byte(subrequests.SubrequestsDescribeDefinition.Version),
 	}
 	return res, nil
 }