diff --git a/client/client_test.go b/client/client_test.go index 137c20544cd8..e5c2ba897fda 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -96,6 +96,7 @@ func TestIntegration(t *testing.T) { testResolveAndHosts, testUser, testOCIExporter, + testOCIExporterContentStore, testWhiteoutParentDir, testFrontendImageNaming, testDuplicateWhiteouts, @@ -2355,6 +2356,99 @@ func testOCIExporter(t *testing.T, sb integration.Sandbox) { checkAllReleasable(t, c, sb, true) } +func testOCIExporterContentStore(t *testing.T, sb integration.Sandbox) { + integration.SkipIfDockerd(t, sb, "oci exporter") + requiresLinux(t) + c, err := New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + busybox := llb.Image("busybox:latest") + st := llb.Scratch() + + run := func(cmd string) { + st = busybox.Run(llb.Shlex(cmd), llb.Dir("/wd")).AddMount("/wd", st) + } + + run(`sh -c "echo -n first > foo"`) + run(`sh -c "echo -n second > bar"`) + + def, err := st.Marshal(sb.Context()) + require.NoError(t, err) + + for _, exp := range []string{ExporterOCI, ExporterDocker} { + destDir := t.TempDir() + target := "example.com/buildkit/testoci:latest" + + outTar := filepath.Join(destDir, "out.tar") + outW, err := os.Create(outTar) + require.NoError(t, err) + attrs := map[string]string{} + if exp == ExporterDocker { + attrs["name"] = target + } + _, err = c.Solve(sb.Context(), def, SolveOpt{ + Exports: []ExportEntry{ + { + Type: exp, + Attrs: attrs, + Output: fixedWriteCloser(outW), + }, + }, + }, nil) + require.NoError(t, err) + + outDir := filepath.Join(destDir, "out.d") + attrs = map[string]string{ + "tar": "false", + } + if exp == ExporterDocker { + attrs["name"] = target + } + _, err = c.Solve(sb.Context(), def, SolveOpt{ + Exports: []ExportEntry{ + { + Type: exp, + Attrs: attrs, + OutputDir: outDir, + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := os.ReadFile(outTar) + require.NoError(t, err) + m, err := testutil.ReadTarToMap(dt, false) + require.NoError(t, err) + + filepath.Walk(outDir, func(filename string, fi os.FileInfo, err error) error { + filename = strings.TrimPrefix(filename, outDir) + filename = strings.Trim(filename, "/") + if filename == "" || filename == "ingest" { + return nil + } + + if fi.IsDir() { + require.Contains(t, m, filename+"/") + } else { + require.Contains(t, m, filename) + if filename == "index.json" { + // this file has a timestamp in it, so we can't compare + return nil + } + f, err := os.Open(path.Join(outDir, filename)) + require.NoError(t, err) + data, err := io.ReadAll(f) + require.NoError(t, err) + require.Equal(t, m[filename].Data, data) + } + return nil + }) + } + + checkAllReleasable(t, c, sb, true) +} + func testSourceDateEpochLayerTimestamps(t *testing.T, sb integration.Sandbox) { integration.SkipIfDockerd(t, sb, "oci exporter") requiresLinux(t) @@ -2692,7 +2786,6 @@ func testSourceDateEpochTarExporter(t *testing.T, sb integration.Sandbox) { checkAllReleasable(t, c, sb, true) } - func testFrontendMetadataReturn(t *testing.T, sb integration.Sandbox) { requiresLinux(t) c, err := New(sb.Context(), sb.Address()) diff --git a/client/solve.go b/client/solve.go index 71c51ba29608..e5bc1ccb7e63 100644 --- a/client/solve.go +++ b/client/solve.go @@ -2,6 +2,7 @@ package client import ( "context" + "encoding/base64" "encoding/json" "io" "os" @@ -14,6 +15,7 @@ import ( controlapi "github.com/moby/buildkit/api/services/control" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/client/ociindex" + "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/session" sessioncontent "github.com/moby/buildkit/session/content" @@ -124,6 +126,8 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG ex = opt.Exports[0] } + indicesToUpdate := []string{} + if !opt.SessionPreInitialized { if len(syncedDirs) > 0 { s.Allow(filesync.NewFSSyncProvider(syncedDirs)) @@ -133,49 +137,64 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG s.Allow(a) } + contentStores := map[string]content.Store{} + for key, store := range cacheOpt.contentStores { + contentStores[key] = store + } + for key, store := range opt.OCIStores { + key2 := "oci:" + key + if _, ok := contentStores[key2]; ok { + return nil, errors.Errorf("oci store key %q already exists", key) + } + contentStores[key2] = store + } + + var supportFile bool + var supportDir bool switch ex.Type { case ExporterLocal: - if ex.Output != nil { - return nil, errors.New("output file writer is not supported by local exporter") - } - if ex.OutputDir == "" { - return nil, errors.New("output directory is required for local exporter") - } - s.Allow(filesync.NewFSSyncTargetDir(ex.OutputDir)) - case ExporterOCI, ExporterDocker, ExporterTar: - if ex.OutputDir != "" { - return nil, errors.Errorf("output directory %s is not supported by %s exporter", ex.OutputDir, ex.Type) - } + supportDir = true + case ExporterTar: + supportFile = true + case ExporterOCI, ExporterDocker: + supportDir = ex.OutputDir != "" + supportFile = ex.Output != nil + } + + if supportFile && supportDir { + return nil, errors.Errorf("both file and directory output is not support by %s exporter", ex.Type) + } + if !supportFile && ex.Output != nil { + return nil, errors.Errorf("output file writer is not supported by %s exporter", ex.Type) + } + if !supportDir && ex.OutputDir != "" { + return nil, errors.Errorf("output directory is not supported by %s exporter", ex.Type) + } + + if supportFile { if ex.Output == nil { return nil, errors.Errorf("output file writer is required for %s exporter", ex.Type) } s.Allow(filesync.NewFSSyncTarget(ex.Output)) - default: - if ex.Output != nil { - return nil, errors.Errorf("output file writer is not supported by %s exporter", ex.Type) - } - if ex.OutputDir != "" { - return nil, errors.Errorf("output directory %s is not supported by %s exporter", ex.OutputDir, ex.Type) - } } - - // this is a new map that contains both cacheOpt stores and OCILayout stores - contentStores := make(map[string]content.Store, len(cacheOpt.contentStores)+len(opt.OCIStores)) - // copy over the stores references from cacheOpt - for key, store := range cacheOpt.contentStores { - contentStores[key] = store - } - // copy over the stores references from ociLayout opts - for key, store := range opt.OCIStores { - // conflicts are not allowed - if _, ok := contentStores[key]; ok { - // we probably should check if the store is identical, but given that - // https://pkg.go.dev/github.com/containerd/containerd/content#Store - // is just an interface, composing 4 others, that is rather hard to do. - // For a future iteration. - return nil, errors.Errorf("contentStore key %s exists in both cache and OCI layouts", key) + if supportDir { + if ex.OutputDir == "" { + return nil, errors.Errorf("output directory is required for %s exporter", ex.Type) + } + switch ex.Type { + case ExporterOCI, ExporterDocker: + if err := os.MkdirAll(ex.OutputDir, 0755); err != nil { + return nil, err + } + cs, err := contentlocal.NewStore(ex.OutputDir) + if err != nil { + return nil, err + } + contentStores["export"] = cs + indicesToUpdate = append(indicesToUpdate, filepath.Join(ex.OutputDir, "index.json")) + default: + s.Allow(filesync.NewFSSyncTargetDir(ex.OutputDir)) } - contentStores[key] = store } if len(contentStores) > 0 { @@ -352,6 +371,25 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG } } } + if manifestDescDt := res.ExporterResponse[exptypes.ExporterImageDescriptorKey]; manifestDescDt != "" { + manifestDescDt, err := base64.StdEncoding.DecodeString(manifestDescDt) + if err != nil { + return nil, err + } + var manifestDesc ocispecs.Descriptor + if err = json.Unmarshal([]byte(manifestDescDt), &manifestDesc); err != nil { + return nil, err + } + for _, indexJSONPath := range indicesToUpdate { + tag := "latest" + if t, ok := res.ExporterResponse["image.name"]; ok { + tag = t + } + if err = ociindex.PutDescToIndexJSONFileLocked(indexJSONPath, manifestDesc, tag); err != nil { + return nil, err + } + } + } return res, nil } diff --git a/cmd/buildctl/build/output.go b/cmd/buildctl/build/output.go index 6c8a78a09f62..abdd508b833f 100644 --- a/cmd/buildctl/build/output.go +++ b/cmd/buildctl/build/output.go @@ -4,6 +4,7 @@ import ( "encoding/csv" "io" "os" + "strconv" "strings" "github.com/containerd/console" @@ -41,7 +42,7 @@ func parseOutputCSV(s string) (client.ExportEntry, error) { if v, ok := ex.Attrs["output"]; ok { return ex, errors.Errorf("output=%s not supported for --output, you meant dest=%s?", v, v) } - ex.Output, ex.OutputDir, err = resolveExporterDest(ex.Type, ex.Attrs["dest"]) + ex.Output, ex.OutputDir, err = resolveExporterDest(ex.Type, ex.Attrs["dest"], ex.Attrs) if err != nil { return ex, errors.Wrap(err, "invalid output option: output") } @@ -65,19 +66,35 @@ func ParseOutput(exports []string) ([]client.ExportEntry, error) { } // resolveExporterDest returns at most either one of io.WriteCloser (single file) or a string (directory path). -func resolveExporterDest(exporter, dest string) (func(map[string]string) (io.WriteCloser, error), string, error) { +func resolveExporterDest(exporter, dest string, attrs map[string]string) (func(map[string]string) (io.WriteCloser, error), string, error) { wrapWriter := func(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) { return func(m map[string]string) (io.WriteCloser, error) { return wc, nil } } + + var supportFile bool + var supportDir bool switch exporter { case client.ExporterLocal: + supportDir = true + case client.ExporterTar: + supportFile = true + case client.ExporterOCI, client.ExporterDocker: + tar, err := strconv.ParseBool(attrs["tar"]) + if err != nil { + tar = true + } + supportFile = tar + supportDir = !tar + } + + if supportDir { if dest == "" { - return nil, "", errors.New("output directory is required for local exporter") + return nil, "", errors.Errorf("output directory is required for %s exporter", exporter) } return nil, dest, nil - case client.ExporterOCI, client.ExporterDocker, client.ExporterTar: + } else if supportFile { if dest != "" && dest != "-" { fi, err := os.Stat(dest) if err != nil && !errors.Is(err, os.ErrNotExist) { @@ -94,7 +111,8 @@ func resolveExporterDest(exporter, dest string) (func(map[string]string) (io.Wri return nil, "", errors.Errorf("output file is required for %s exporter. refusing to write to console", exporter) } return wrapWriter(os.Stdout), "", nil - default: // e.g. client.ExporterImage + } else { + // e.g. client.ExporterImage if dest != "" { return nil, "", errors.Errorf("output %s is not supported by %s exporter", dest, exporter) } diff --git a/exporter/local/export.go b/exporter/local/export.go index f09eba60f9c6..3972f08f7ccb 100644 --- a/exporter/local/export.go +++ b/exporter/local/export.go @@ -165,7 +165,7 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source } } - progress := newProgressHandler(ctx, lbl) + progress := NewProgressHandler(ctx, lbl) if err := filesync.CopyToCaller(ctx, fs, caller, progress); err != nil { return err } @@ -193,7 +193,7 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source return nil, nil } -func newProgressHandler(ctx context.Context, id string) func(int, bool) { +func NewProgressHandler(ctx context.Context, id string) func(int, bool) { limiter := rate.NewLimiter(rate.Every(100*time.Millisecond), 1) pw, _, _ := progress.NewFromContext(ctx) now := time.Now() diff --git a/exporter/oci/export.go b/exporter/oci/export.go index ab8a65d82a38..5cefafdcf817 100644 --- a/exporter/oci/export.go +++ b/exporter/oci/export.go @@ -5,18 +5,22 @@ import ( "encoding/base64" "encoding/json" "fmt" + "strconv" "strings" "time" archiveexporter "github.com/containerd/containerd/images/archive" "github.com/containerd/containerd/leases" + "github.com/containerd/containerd/remotes" "github.com/docker/distribution/reference" + intoto "github.com/in-toto/in-toto-golang/in_toto" "github.com/moby/buildkit/cache" cacheconfig "github.com/moby/buildkit/cache/config" "github.com/moby/buildkit/exporter" "github.com/moby/buildkit/exporter/containerimage" "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/session" + sessioncontent "github.com/moby/buildkit/session/content" "github.com/moby/buildkit/session/filesync" "github.com/moby/buildkit/util/compression" "github.com/moby/buildkit/util/contentutil" @@ -35,6 +39,10 @@ const ( VariantDocker = "docker" ) +const ( + keyTar = "tar" +) + type Opt struct { SessionManager *session.Manager ImageWriter *containerimage.ImageWriter @@ -54,6 +62,7 @@ func New(opt Opt) (exporter.Exporter, error) { func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exporter.ExporterInstance, error) { i := &imageExporterInstance{ imageExporter: e, + tar: true, opts: containerimage.ImageCommitOpts{ RefCfg: cacheconfig.RefConfig{ Compression: compression.New(compression.Default), @@ -69,10 +78,23 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp } for k, v := range opt { - if i.meta == nil { - i.meta = make(map[string][]byte) + switch k { + case keyTar: + if v == "" { + i.tar = true + continue + } + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "non-bool value specified for %s", k) + } + i.tar = b + default: + if i.meta == nil { + i.meta = make(map[string][]byte) + } + i.meta[k] = []byte(v) } - i.meta[k] = []byte(v) } return i, nil } @@ -80,6 +102,7 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp type imageExporterInstance struct { *imageExporter opts containerimage.ImageCommitOpts + tar bool meta map[string][]byte } @@ -179,11 +202,6 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source return nil, err } - w, err := filesync.CopyFileWriter(ctx, resp, caller) - if err != nil { - return nil, err - } - mprovider := contentutil.NewMultiProvider(e.opt.ImageWriter.ContentStore()) if src.Ref != nil { remotes, err := src.Ref.GetRemotes(ctx, false, e.opts.RefCfg, false, session.NewGroup(sessionID)) @@ -220,19 +238,41 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source } } - report := progress.OneOff(ctx, "sending tarball") - if err := archiveexporter.Export(ctx, mprovider, w, expOpts...); err != nil { - w.Close() + if e.tar { + w, err := filesync.CopyFileWriter(ctx, resp, caller) + if err != nil { + return nil, err + } + + report := progress.OneOff(ctx, "sending tarball") + if err := archiveexporter.Export(ctx, mprovider, w, expOpts...); err != nil { + w.Close() + if grpcerrors.Code(err) == codes.AlreadyExists { + return resp, report(nil) + } + return nil, report(err) + } + err = w.Close() if grpcerrors.Code(err) == codes.AlreadyExists { return resp, report(nil) } - return nil, report(err) - } - err = w.Close() - if grpcerrors.Code(err) == codes.AlreadyExists { - return resp, report(nil) + if err != nil { + return nil, report(err) + } + report(nil) + } else { + ctx = remotes.WithMediaTypeKeyPrefix(ctx, intoto.PayloadType, "intoto") + store := sessioncontent.NewCallerStore(caller, "export") + if err != nil { + return nil, err + } + err := contentutil.CopyChain(ctx, store, mprovider, *desc) + if err != nil { + return nil, err + } } - return resp, report(err) + + return resp, nil } func normalizedNames(name string) ([]string, error) { diff --git a/source/containerimage/ocilayout.go b/source/containerimage/ocilayout.go index 4c9326ef4000..86b0d9111211 100644 --- a/source/containerimage/ocilayout.go +++ b/source/containerimage/ocilayout.go @@ -47,7 +47,7 @@ func (r *ociLayoutResolver) Fetcher(ctx context.Context, ref string) (remotes.Fe func (r *ociLayoutResolver) Fetch(ctx context.Context, desc ocispecs.Descriptor) (io.ReadCloser, error) { var rc io.ReadCloser err := r.withCaller(ctx, func(ctx context.Context, caller session.Caller) error { - store := sessioncontent.NewCallerStore(caller, r.storeID) + store := sessioncontent.NewCallerStore(caller, "oci:"+r.storeID) readerAt, err := store.ReaderAt(ctx, desc) if err != nil { return err @@ -103,7 +103,7 @@ func (r *ociLayoutResolver) Resolve(ctx context.Context, refString string) (stri func (r *ociLayoutResolver) info(ctx context.Context, ref reference.Spec) (content.Info, error) { var info *content.Info err := r.withCaller(ctx, func(ctx context.Context, caller session.Caller) error { - store := sessioncontent.NewCallerStore(caller, r.storeID) + store := sessioncontent.NewCallerStore(caller, "oci:"+r.storeID) _, dgst := reference.SplitObject(ref.Object) if dgst == "" {