oci exporter: support exporting directories

client/client_test.go

@@ -96,6 +96,7 @@ func TestIntegration(t *testing.T) {
 		testResolveAndHosts,
 		testUser,
 		testOCIExporter,
+		testOCIExporterContentStore,
 		testWhiteoutParentDir,
 		testFrontendImageNaming,
 		testDuplicateWhiteouts,
@@ -2355,6 +2356,99 @@ func testOCIExporter(t *testing.T, sb integration.Sandbox) {
 	checkAllReleasable(t, c, sb, true)
 }
 
+func testOCIExporterContentStore(t *testing.T, sb integration.Sandbox) {
+	integration.SkipIfDockerd(t, sb, "oci exporter")
+	requiresLinux(t)
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	busybox := llb.Image("busybox:latest")
+	st := llb.Scratch()
+
+	run := func(cmd string) {
+		st = busybox.Run(llb.Shlex(cmd), llb.Dir("/wd")).AddMount("/wd", st)
+	}
+
+	run(`sh -c "echo -n first > foo"`)
+	run(`sh -c "echo -n second > bar"`)
+
+	def, err := st.Marshal(sb.Context())
+	require.NoError(t, err)
+
+	for _, exp := range []string{ExporterOCI, ExporterDocker} {
+		destDir := t.TempDir()
+		target := "example.com/buildkit/testoci:latest"
+
+		outTar := filepath.Join(destDir, "out.tar")
+		outW, err := os.Create(outTar)
+		require.NoError(t, err)
+		attrs := map[string]string{}
+		if exp == ExporterDocker {
+			attrs["name"] = target
+		}
+		_, err = c.Solve(sb.Context(), def, SolveOpt{
+			Exports: []ExportEntry{
+				{
+					Type:   exp,
+					Attrs:  attrs,
+					Output: fixedWriteCloser(outW),
+				},
+			},
+		}, nil)
+		require.NoError(t, err)
+
+		outDir := filepath.Join(destDir, "out.d")
+		attrs = map[string]string{
+			"tar": "false",
+		}
+		if exp == ExporterDocker {
+			attrs["name"] = target
+		}
+		_, err = c.Solve(sb.Context(), def, SolveOpt{
+			Exports: []ExportEntry{
+				{
+					Type:      exp,
+					Attrs:     attrs,
+					OutputDir: outDir,
+				},
+			},
+		}, nil)
+		require.NoError(t, err)
+
+		dt, err := os.ReadFile(outTar)
+		require.NoError(t, err)
+		m, err := testutil.ReadTarToMap(dt, false)
+		require.NoError(t, err)
+
+		filepath.Walk(outDir, func(filename string, fi os.FileInfo, err error) error {
+			filename = strings.TrimPrefix(filename, outDir)
+			filename = strings.Trim(filename, "/")
+			if filename == "" || filename == "ingest" {
+				return nil
+			}
+
+			if fi.IsDir() {
+				require.Contains(t, m, filename+"/")
+			} else {
+				require.Contains(t, m, filename)
+				if filename == "index.json" {
+					// this file has a timestamp in it, so we can't compare
+					return nil
+				}
+				f, err := os.Open(path.Join(outDir, filename))
+				require.NoError(t, err)
+				data, err := io.ReadAll(f)
+				require.NoError(t, err)
+				require.Equal(t, m[filename].Data, data)
+			}
+			return nil
+		})
+	}
+
+	checkAllReleasable(t, c, sb, true)
+}
+
 func testSourceDateEpochLayerTimestamps(t *testing.T, sb integration.Sandbox) {
 	integration.SkipIfDockerd(t, sb, "oci exporter")
 	requiresLinux(t)
@@ -2692,7 +2786,6 @@ func testSourceDateEpochTarExporter(t *testing.T, sb integration.Sandbox) {
 
 	checkAllReleasable(t, c, sb, true)
 }
-
 func testFrontendMetadataReturn(t *testing.T, sb integration.Sandbox) {
 	requiresLinux(t)
 	c, err := New(sb.Context(), sb.Address())

client/solve.go

@@ -2,6 +2,7 @@ package client
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"io"
 	"os"
@@ -14,6 +15,7 @@ import (
 	controlapi "github.com/moby/buildkit/api/services/control"
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/client/ociindex"
+	"github.com/moby/buildkit/exporter/containerimage/exptypes"
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/session"
 	sessioncontent "github.com/moby/buildkit/session/content"
@@ -124,6 +126,8 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 		ex = opt.Exports[0]
 	}
 
+	indicesToUpdate := []string{}
+
 	if !opt.SessionPreInitialized {
 		if len(syncedDirs) > 0 {
 			s.Allow(filesync.NewFSSyncProvider(syncedDirs))
@@ -133,49 +137,64 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 			s.Allow(a)
 		}
 
+		contentStores := map[string]content.Store{}
+		for key, store := range cacheOpt.contentStores {
+			contentStores[key] = store
+		}
+		for key, store := range opt.OCIStores {
+			key2 := "oci:" + key
+			if _, ok := contentStores[key2]; ok {
+				return nil, errors.Errorf("oci store key %q already exists", key)
+			}
+			contentStores[key2] = store
+		}
+
+		var supportFile bool
+		var supportDir bool
 		switch ex.Type {
 		case ExporterLocal:
-			if ex.Output != nil {
-				return nil, errors.New("output file writer is not supported by local exporter")
-			}
-			if ex.OutputDir == "" {
-				return nil, errors.New("output directory is required for local exporter")
-			}
-			s.Allow(filesync.NewFSSyncTargetDir(ex.OutputDir))
-		case ExporterOCI, ExporterDocker, ExporterTar:
-			if ex.OutputDir != "" {
-				return nil, errors.Errorf("output directory %s is not supported by %s exporter", ex.OutputDir, ex.Type)
-			}
+			supportDir = true
+		case ExporterTar:
+			supportFile = true
+		case ExporterOCI, ExporterDocker:
+			supportDir = ex.OutputDir != ""
+			supportFile = ex.Output != nil
+		}
+
+		if supportFile && supportDir {
+			return nil, errors.Errorf("both file and directory output is not support by %s exporter", ex.Type)
+		}
+		if !supportFile && ex.Output != nil {
+			return nil, errors.Errorf("output file writer is not supported by %s exporter", ex.Type)
+		}
+		if !supportDir && ex.OutputDir != "" {
+			return nil, errors.Errorf("output directory is not supported by %s exporter", ex.Type)
+		}
+
+		if supportFile {
 			if ex.Output == nil {
 				return nil, errors.Errorf("output file writer is required for %s exporter", ex.Type)
 			}
 			s.Allow(filesync.NewFSSyncTarget(ex.Output))
-		default:
-			if ex.Output != nil {
-				return nil, errors.Errorf("output file writer is not supported by %s exporter", ex.Type)
-			}
-			if ex.OutputDir != "" {
-				return nil, errors.Errorf("output directory %s is not supported by %s exporter", ex.OutputDir, ex.Type)
-			}
 		}
-
-		// this is a new map that contains both cacheOpt stores and OCILayout stores
-		contentStores := make(map[string]content.Store, len(cacheOpt.contentStores)+len(opt.OCIStores))
-		// copy over the stores references from cacheOpt
-		for key, store := range cacheOpt.contentStores {
-			contentStores[key] = store
-		}
-		// copy over the stores references from ociLayout opts
-		for key, store := range opt.OCIStores {
-			// conflicts are not allowed
-			if _, ok := contentStores[key]; ok {
-				// we probably should check if the store is identical, but given that
-				// https://pkg.go.dev/github.com/containerd/containerd/content#Store
-				// is just an interface, composing 4 others, that is rather hard to do.
-				// For a future iteration.
-				return nil, errors.Errorf("contentStore key %s exists in both cache and OCI layouts", key)
+		if supportDir {
+			if ex.OutputDir == "" {
+				return nil, errors.Errorf("output directory is required for %s exporter", ex.Type)
+			}
+			switch ex.Type {
+			case ExporterOCI, ExporterDocker:
+				if err := os.MkdirAll(ex.OutputDir, 0755); err != nil {
+					return nil, err
+				}
+				cs, err := contentlocal.NewStore(ex.OutputDir)
+				if err != nil {
+					return nil, err
+				}
+				contentStores["export"] = cs
+				indicesToUpdate = append(indicesToUpdate, filepath.Join(ex.OutputDir, "index.json"))
+			default:
+				s.Allow(filesync.NewFSSyncTargetDir(ex.OutputDir))
 			}
-			contentStores[key] = store
 		}
 
 		if len(contentStores) > 0 {
@@ -352,6 +371,25 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 			}
 		}
 	}
+	if manifestDescDt := res.ExporterResponse[exptypes.ExporterImageDescriptorKey]; manifestDescDt != "" {
+		manifestDescDt, err := base64.StdEncoding.DecodeString(manifestDescDt)
+		if err != nil {
+			return nil, err
+		}
+		var manifestDesc ocispecs.Descriptor
+		if err = json.Unmarshal([]byte(manifestDescDt), &manifestDesc); err != nil {
+			return nil, err
+		}
+		for _, indexJSONPath := range indicesToUpdate {
+			tag := "latest"
+			if t, ok := res.ExporterResponse["image.name"]; ok {
+				tag = t
+			}
+			if err = ociindex.PutDescToIndexJSONFileLocked(indexJSONPath, manifestDesc, tag); err != nil {
+				return nil, err
+			}
+		}
+	}
 	return res, nil
 }
 

cmd/buildctl/build/output.go

@@ -4,6 +4,7 @@ import (
 	"encoding/csv"
 	"io"
 	"os"
+	"strconv"
 	"strings"
 
 	"github.com/containerd/console"
@@ -41,7 +42,7 @@ func parseOutputCSV(s string) (client.ExportEntry, error) {
 	if v, ok := ex.Attrs["output"]; ok {
 		return ex, errors.Errorf("output=%s not supported for --output, you meant dest=%s?", v, v)
 	}
-	ex.Output, ex.OutputDir, err = resolveExporterDest(ex.Type, ex.Attrs["dest"])
+	ex.Output, ex.OutputDir, err = resolveExporterDest(ex.Type, ex.Attrs["dest"], ex.Attrs)
 	if err != nil {
 		return ex, errors.Wrap(err, "invalid output option: output")
 	}
@@ -65,19 +66,35 @@ func ParseOutput(exports []string) ([]client.ExportEntry, error) {
 }
 
 // resolveExporterDest returns at most either one of io.WriteCloser (single file) or a string (directory path).
-func resolveExporterDest(exporter, dest string) (func(map[string]string) (io.WriteCloser, error), string, error) {
+func resolveExporterDest(exporter, dest string, attrs map[string]string) (func(map[string]string) (io.WriteCloser, error), string, error) {
 	wrapWriter := func(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
 		return func(m map[string]string) (io.WriteCloser, error) {
 			return wc, nil
 		}
 	}
+
+	var supportFile bool
+	var supportDir bool
 	switch exporter {
 	case client.ExporterLocal:
+		supportDir = true
+	case client.ExporterTar:
+		supportFile = true
+	case client.ExporterOCI, client.ExporterDocker:
+		tar, err := strconv.ParseBool(attrs["tar"])
+		if err != nil {
+			tar = true
+		}
+		supportFile = tar
+		supportDir = !tar
+	}
+
+	if supportDir {
 		if dest == "" {
-			return nil, "", errors.New("output directory is required for local exporter")
+			return nil, "", errors.Errorf("output directory is required for %s exporter", exporter)
 		}
 		return nil, dest, nil
-	case client.ExporterOCI, client.ExporterDocker, client.ExporterTar:
+	} else if supportFile {
 		if dest != "" && dest != "-" {
 			fi, err := os.Stat(dest)
 			if err != nil && !errors.Is(err, os.ErrNotExist) {
@@ -94,7 +111,8 @@ func resolveExporterDest(exporter, dest string) (func(map[string]string) (io.Wri
 			return nil, "", errors.Errorf("output file is required for %s exporter. refusing to write to console", exporter)
 		}
 		return wrapWriter(os.Stdout), "", nil
-	default: // e.g. client.ExporterImage
+	} else {
+		// e.g. client.ExporterImage
 		if dest != "" {
 			return nil, "", errors.Errorf("output %s is not supported by %s exporter", dest, exporter)
 		}

exporter/local/export.go

@@ -165,7 +165,7 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source
 				}
 			}
 
-			progress := newProgressHandler(ctx, lbl)
+			progress := NewProgressHandler(ctx, lbl)
 			if err := filesync.CopyToCaller(ctx, fs, caller, progress); err != nil {
 				return err
 			}
@@ -193,7 +193,7 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source
 	return nil, nil
 }
 
-func newProgressHandler(ctx context.Context, id string) func(int, bool) {
+func NewProgressHandler(ctx context.Context, id string) func(int, bool) {
 	limiter := rate.NewLimiter(rate.Every(100*time.Millisecond), 1)
 	pw, _, _ := progress.NewFromContext(ctx)
 	now := time.Now()