Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dockerfile: support for outline requests #2841

Merged
merged 10 commits into from
Aug 9, 2022
Merged

dockerfile: support for outline requests #2841

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ec9a3ac
dockerfile: support for outline requests
tonistiigi Feb 11, 2022
b1a2aa7
buildctl: print subrequest metadata result to stdout
tonistiigi May 2, 2022
3b1fd05
dockerfile: add secret and ssh information to outline
tonistiigi May 3, 2022
f950ba7
dockerfile: add request for listing targets
tonistiigi May 4, 2022
e7c8b22
dockerfile: add target stage information to outline
tonistiigi May 4, 2022
b88e435
subrequests: add base and platform to targets request
tonistiigi Aug 5, 2022
9924649
subrequest: always return version with subrequest result
tonistiigi Aug 5, 2022
f7bbe2f
subrequests: add text formatting to results
tonistiigi Aug 5, 2022
d1b0d8a
subrequests: normalize json fields to lowercase
tonistiigi Aug 5, 2022
28a156b
dockerfile: remove filtering out empty targets
tonistiigi Aug 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 45 additions & 2 deletions cmd/buildctl/build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,18 @@ import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"os"
"strings"

"github.com/containerd/continuity"
"github.com/docker/cli/cli/config"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/cmd/buildctl/build"
bccommon "github.com/moby/buildkit/cmd/buildctl/common"
gateway "github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/auth/authprovider"
"github.com/moby/buildkit/session/sshforward/sshprovider"
Expand Down Expand Up @@ -262,13 +265,40 @@ func buildAction(clicontext *cli.Context) error {
}
}

var subMetadata map[string][]byte

eg.Go(func() error {
defer func() {
for _, w := range writers {
close(w.Status())
}
}()
resp, err := c.Solve(ctx, def, solveOpt, progresswriter.ResetTime(mw.WithPrefix("", false)).Status())
sreq := gateway.SolveRequest{
Frontend: solveOpt.Frontend,
FrontendOpt: solveOpt.FrontendAttrs,
}
if def != nil {
sreq.Definition = def.ToPB()
}
solveOpt.Frontend = ""
solveOpt.FrontendAttrs = nil

resp, err := c.Build(ctx, solveOpt, "buildctl", func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
_, isSubRequest := sreq.FrontendOpt["requestid"]
if isSubRequest {
if _, ok := sreq.FrontendOpt["frontend.caps"]; !ok {
sreq.FrontendOpt["frontend.caps"] = "moby.buildkit.frontend.subrequests"
}
}
res, err := c.Solve(ctx, sreq)
if err != nil {
return nil, err
}
if isSubRequest && res != nil {
subMetadata = res.Metadata
}
return res, err
}, progresswriter.ResetTime(mw.WithPrefix("", false)).Status())
if err != nil {
return err
}
Expand All @@ -291,7 +321,20 @@ func buildAction(clicontext *cli.Context) error {
return pw.Err()
})

return eg.Wait()
if err := eg.Wait(); err != nil {
return err
}

if txt, ok := subMetadata["result.txt"]; ok {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would we want a case for result.json as well? Feels like we could format it here, in case the frontend hasn't done it.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We would need a specific printer for this and I don't want buildctl to have too much hardcoded info about specific requests.

Or are you saying that if there is a result.json we should only print that directly and not the full result like it is in current commit.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess we have an exception for result.txt, I'm just wondering why we shouldn't also have one for result.json. I think at the least, we should print directly and not the whole result if we find a result.json similar to result.txt.

Although maybe for buildctl, since it's a dev tool, we should assume no knowledge of the response, and always print out everything?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

and always print out everything?

That what it does atm. There is a special case for result.txt, so if it exists then the expectation is that frontend has already formatted the result and we can just print that output. Otherwise we just print whatever frontend provided(including result.json). If it makes more sense I can make it so that it only prints result.json if it exists and skips the rest.

fmt.Print(string(txt))
} else {
for k, v := range subMetadata {
if strings.HasPrefix(k, "result.") {
fmt.Printf("%s\n%s\n", k, v)
}
}
}
return nil
}

func writeMetadataFile(filename string, exporterResponse map[string]string) error {
Expand Down
104 changes: 65 additions & 39 deletions frontend/dockerfile/builder/build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import (
"github.com/moby/buildkit/frontend/dockerfile/parser"
"github.com/moby/buildkit/frontend/gateway/client"
gwpb "github.com/moby/buildkit/frontend/gateway/pb"
"github.com/moby/buildkit/frontend/subrequests/outline"
"github.com/moby/buildkit/frontend/subrequests/targets"
"github.com/moby/buildkit/solver/errdefs"
"github.com/moby/buildkit/solver/pb"
binfotypes "github.com/moby/buildkit/util/buildinfo/types"
Expand Down Expand Up @@ -60,6 +62,7 @@ const (
keyShmSize = "shm-size"
keyTargetPlatform = "platform"
keyUlimit = "ulimit"
keyRequestID = "requestid"

// Don't forget to update frontend documentation if you add
// a new build-arg: frontend/dockerfile/docs/reference.md
Expand All @@ -72,7 +75,7 @@ const (

var httpPrefix = regexp.MustCompile(`^https?://`)

func Build(ctx context.Context, c client.Client) (*client.Result, error) {
func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) {
opts := c.BuildOpts().Opts
caps := c.BuildOpts().LLBCaps
gwcaps := c.BuildOpts().Caps
Expand Down Expand Up @@ -420,49 +423,72 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
opts[keyHostname] = v
}

convertOpt := dockerfile2llb.ConvertOpt{
Target: opts[keyTarget],
MetaResolver: c,
BuildArgs: filter(opts, buildArgPrefix),
Labels: filter(opts, labelPrefix),
CacheIDNamespace: opts[keyCacheNSArg],
SessionID: c.BuildOpts().SessionID,
BuildContext: buildContext,
Excludes: excludes,
IgnoreCache: ignoreCache,
TargetPlatform: targetPlatforms[0],
BuildPlatforms: buildPlatforms,
ImageResolveMode: resolveMode,
PrefixPlatform: exportMap,
ExtraHosts: extraHosts,
ShmSize: shmSize,
Ulimit: ulimit,
CgroupParent: opts[keyCgroupParent],
ForceNetMode: defaultNetMode,
LLBCaps: &caps,
SourceMap: sourceMap,
Hostname: opts[keyHostname],
Warn: func(msg, url string, detail [][]byte, location *parser.Range) {
c.Warn(ctx, defVtx, msg, warnOpts(sourceMap, location, detail, url))
},
ContextByName: contextByNameFunc(c, c.BuildOpts().SessionID, targetPlatforms[0]),
}

defer func() {
var el *parser.ErrorLocation
if errors.As(err, &el) {
err = wrapSource(err, sourceMap, el.Location)
}
}()

if req, ok := opts[keyRequestID]; ok {
switch req {
case outline.SubrequestsOutlineDefinition.Name:
o, err := dockerfile2llb.Dockefile2Outline(ctx, dtDockerfile, convertOpt)
if err != nil {
return nil, err
}
return o.ToResult()
case targets.SubrequestsTargetsDefinition.Name:
targets, err := dockerfile2llb.ListTargets(ctx, dtDockerfile)
if err != nil {
return nil, err
}
return targets.ToResult()
default:
return nil, errdefs.NewUnsupportedSubrequestError(req)
}
}

eg, ctx = errgroup.WithContext(ctx)

for i, tp := range targetPlatforms {
func(i int, tp *ocispecs.Platform) {
eg.Go(func() (err error) {
defer func() {
var el *parser.ErrorLocation
if errors.As(err, &el) {
err = wrapSource(err, sourceMap, el.Location)
}
}()

st, img, bi, err := dockerfile2llb.Dockerfile2LLB(ctx, dtDockerfile, dockerfile2llb.ConvertOpt{
Target: opts[keyTarget],
MetaResolver: c,
BuildArgs: filter(opts, buildArgPrefix),
Labels: filter(opts, labelPrefix),
CacheIDNamespace: opts[keyCacheNSArg],
SessionID: c.BuildOpts().SessionID,
BuildContext: buildContext,
Excludes: excludes,
IgnoreCache: ignoreCache,
TargetPlatform: tp,
BuildPlatforms: buildPlatforms,
ImageResolveMode: resolveMode,
PrefixPlatform: exportMap,
ExtraHosts: extraHosts,
ShmSize: shmSize,
Ulimit: ulimit,
CgroupParent: opts[keyCgroupParent],
ForceNetMode: defaultNetMode,
LLBCaps: &caps,
SourceMap: sourceMap,
Hostname: opts[keyHostname],
Warn: func(msg, url string, detail [][]byte, location *parser.Range) {
if i != 0 {
return
}
c.Warn(ctx, defVtx, msg, warnOpts(sourceMap, location, detail, url))
},
ContextByName: contextByNameFunc(c, c.BuildOpts().SessionID, tp),
})

opt := convertOpt
opt.TargetPlatform = tp
if i != 0 {
opt.Warn = nil
}
opt.ContextByName = contextByNameFunc(c, c.BuildOpts().SessionID, tp)
st, img, bi, err := dockerfile2llb.Dockerfile2LLB(ctx, dtDockerfile, opt)
if err != nil {
return err
}
Expand Down
19 changes: 17 additions & 2 deletions frontend/dockerfile/builder/subrequests.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,39 +1,54 @@
package builder

import (
"bytes"
"context"
"encoding/json"

"github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/frontend/subrequests"
"github.com/moby/buildkit/frontend/subrequests/outline"
"github.com/moby/buildkit/frontend/subrequests/targets"
"github.com/moby/buildkit/solver/errdefs"
)

func checkSubRequest(ctx context.Context, opts map[string]string) (*client.Result, bool, error) {
req, ok := opts["requestid"]
req, ok := opts[keyRequestID]
if !ok {
return nil, false, nil
}
switch req {
case subrequests.RequestSubrequestsDescribe:
res, err := describe()
return res, true, err
case outline.RequestSubrequestsOutline, targets.RequestTargets: // handled later
return nil, false, nil
default:
return nil, true, errdefs.NewUnsupportedSubrequestError(req)
}
}

func describe() (*client.Result, error) {
all := []subrequests.Request{
outline.SubrequestsOutlineDefinition,
targets.SubrequestsTargetsDefinition,
subrequests.SubrequestsDescribeDefinition,
}
dt, err := json.MarshalIndent(all, " ", "")
dt, err := json.MarshalIndent(all, "", " ")
if err != nil {
return nil, err
}

b := bytes.NewBuffer(nil)
if err := subrequests.PrintDescribe(dt, b); err != nil {
return nil, err
}

res := client.NewResult()
res.Metadata = map[string][]byte{
"result.json": dt,
"result.txt": b.Bytes(),
"version": []byte(subrequests.SubrequestsDescribeDefinition.Version),
}
return res, nil
}
Loading