Merge pull request #1551 from tonistiigi/session-group

session: track sessions with a group construct

cache/remotecache/export.go

@@ -10,6 +10,7 @@ import (
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/images"
 	v1 "github.com/moby/buildkit/cache/remotecache/v1"
+	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/util/contentutil"
 	"github.com/moby/buildkit/util/progress"
@@ -19,7 +20,7 @@ import (
 	"github.com/pkg/errors"
 )
 
-type ResolveCacheExporterFunc func(ctx context.Context, attrs map[string]string) (Exporter, error)
+type ResolveCacheExporterFunc func(ctx context.Context, g session.Group, attrs map[string]string) (Exporter, error)
 
 func oneOffProgress(ctx context.Context, id string) func(err error) error {
 	pw, _, _ := progress.FromContext(ctx)

cache/remotecache/import.go

@@ -10,6 +10,7 @@ import (
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/images"
 	v1 "github.com/moby/buildkit/cache/remotecache/v1"
+	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/util/imageutil"
 	"github.com/moby/buildkit/worker"
@@ -21,7 +22,7 @@ import (
 )
 
 // ResolveCacheImporterFunc returns importer and descriptor.
-type ResolveCacheImporterFunc func(ctx context.Context, attrs map[string]string) (Importer, ocispec.Descriptor, error)
+type ResolveCacheImporterFunc func(ctx context.Context, g session.Group, attrs map[string]string) (Importer, ocispec.Descriptor, error)
 
 type Importer interface {
 	Resolve(ctx context.Context, desc ocispec.Descriptor, id string, w worker.Worker) (solver.CacheManager, error)

cache/remotecache/inline/inline.go

@@ -6,13 +6,14 @@ import (
 
 	"github.com/moby/buildkit/cache/remotecache"
 	v1 "github.com/moby/buildkit/cache/remotecache/v1"
+	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/solver"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/sirupsen/logrus"
 )
 
 func ResolveCacheExporterFunc() remotecache.ResolveCacheExporterFunc {
-	return func(ctx context.Context, _ map[string]string) (remotecache.Exporter, error) {
+	return func(ctx context.Context, _ session.Group, _ map[string]string) (remotecache.Exporter, error) {
 		return NewExporter(), nil
 	}
 }

cache/remotecache/local/local.go

@@ -22,13 +22,13 @@ const (
 
 // ResolveCacheExporterFunc for "local" cache exporter.
 func ResolveCacheExporterFunc(sm *session.Manager) remotecache.ResolveCacheExporterFunc {
-	return func(ctx context.Context, attrs map[string]string) (remotecache.Exporter, error) {
+	return func(ctx context.Context, g session.Group, attrs map[string]string) (remotecache.Exporter, error) {
 		store := attrs[attrDest]
 		if store == "" {
 			return nil, errors.New("local cache exporter requires dest")
 		}
 		csID := contentStoreIDPrefix + store
-		cs, err := getContentStore(ctx, sm, csID)
+		cs, err := getContentStore(ctx, sm, g, csID)
 		if err != nil {
 			return nil, err
 		}
@@ -38,7 +38,7 @@ func ResolveCacheExporterFunc(sm *session.Manager) remotecache.ResolveCacheExpor
 
 // ResolveCacheImporterFunc for "local" cache importer.
 func ResolveCacheImporterFunc(sm *session.Manager) remotecache.ResolveCacheImporterFunc {
-	return func(ctx context.Context, attrs map[string]string) (remotecache.Importer, specs.Descriptor, error) {
+	return func(ctx context.Context, g session.Group, attrs map[string]string) (remotecache.Importer, specs.Descriptor, error) {
 		dgstStr := attrs[attrDigest]
 		if dgstStr == "" {
 			return nil, specs.Descriptor{}, errors.New("local cache importer requires explicit digest")
@@ -49,7 +49,7 @@ func ResolveCacheImporterFunc(sm *session.Manager) remotecache.ResolveCacheImpor
 			return nil, specs.Descriptor{}, errors.New("local cache importer requires src")
 		}
 		csID := contentStoreIDPrefix + store
-		cs, err := getContentStore(ctx, sm, csID)
+		cs, err := getContentStore(ctx, sm, g, csID)
 		if err != nil {
 			return nil, specs.Descriptor{}, err
 		}
@@ -67,8 +67,9 @@ func ResolveCacheImporterFunc(sm *session.Manager) remotecache.ResolveCacheImpor
 	}
 }
 
-func getContentStore(ctx context.Context, sm *session.Manager, storeID string) (content.Store, error) {
-	sessionID := session.FromContext(ctx)
+func getContentStore(ctx context.Context, sm *session.Manager, g session.Group, storeID string) (content.Store, error) {
+	// TODO: to ensure correct session is detected, new api for finding if storeID is supported is needed
+	sessionID := g.SessionIterator().NextSession()
 	if sessionID == "" {
 		return nil, errors.New("local cache exporter/importer requires session")
 	}

cache/remotecache/registry/registry.go

@@ -32,12 +32,12 @@ const (
 )
 
 func ResolveCacheExporterFunc(sm *session.Manager, hosts docker.RegistryHosts) remotecache.ResolveCacheExporterFunc {
-	return func(ctx context.Context, attrs map[string]string) (remotecache.Exporter, error) {
+	return func(ctx context.Context, g session.Group, attrs map[string]string) (remotecache.Exporter, error) {
 		ref, err := canonicalizeRef(attrs[attrRef])
 		if err != nil {
 			return nil, err
 		}
-		remote := resolver.New(ctx, hosts, sm)
+		remote := resolver.New(hosts, resolver.NewSessionAuthenticator(sm, g))
 		pusher, err := remote.Pusher(ctx, ref)
 		if err != nil {
 			return nil, err
@@ -47,12 +47,12 @@ func ResolveCacheExporterFunc(sm *session.Manager, hosts docker.RegistryHosts) r
 }
 
 func ResolveCacheImporterFunc(sm *session.Manager, cs content.Store, hosts docker.RegistryHosts) remotecache.ResolveCacheImporterFunc {
-	return func(ctx context.Context, attrs map[string]string) (remotecache.Importer, specs.Descriptor, error) {
+	return func(ctx context.Context, g session.Group, attrs map[string]string) (remotecache.Importer, specs.Descriptor, error) {
 		ref, err := canonicalizeRef(attrs[attrRef])
 		if err != nil {
 			return nil, specs.Descriptor{}, err
 		}
-		remote := resolver.New(ctx, hosts, sm)
+		remote := resolver.New(hosts, resolver.NewSessionAuthenticator(sm, g))
 		xref, desc, err := remote.Resolve(ctx, ref)
 		if err != nil {
 			return nil, specs.Descriptor{}, err

control/control.go

@@ -220,7 +220,6 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
 	if err := translateLegacySolveRequest(req); err != nil {
 		return nil, err
 	}
-	ctx = session.NewContext(ctx, req.Session)
 
 	defer func() {
 		time.AfterFunc(time.Second, c.throttledGC)
@@ -260,7 +259,7 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
 		if !ok {
 			return nil, errors.Errorf("unknown cache exporter: %q", e.Type)
 		}
-		cacheExporter, err = cacheExporterFunc(ctx, e.Attrs)
+		cacheExporter, err = cacheExporterFunc(ctx, session.NewGroup(req.Session), e.Attrs)
 		if err != nil {
 			return nil, err
 		}
@@ -273,7 +272,7 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
 		})
 	}
 
-	resp, err := c.solver.Solve(ctx, req.Ref, frontend.SolveRequest{
+	resp, err := c.solver.Solve(ctx, req.Ref, req.Session, frontend.SolveRequest{
 		Frontend:       req.Frontend,
 		Definition:     req.Definition,
 		FrontendOpt:    req.FrontendAttrs,

exporter/containerimage/export.go

@@ -168,7 +168,7 @@ func (e *imageExporterInstance) Name() string {
 	return "exporting to image"
 }
 
-func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source) (map[string]string, error) {
+func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source, sessionID string) (map[string]string, error) {
 	if src.Metadata == nil {
 		src.Metadata = make(map[string][]byte)
 	}
@@ -237,7 +237,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source)
 				}
 			}
 			if e.push {
-				if err := push.Push(ctx, e.opt.SessionManager, e.opt.ImageWriter.ContentStore(), desc.Digest, targetName, e.insecure, e.opt.RegistryHosts, e.pushByDigest); err != nil {
+				if err := push.Push(ctx, e.opt.SessionManager, sessionID, e.opt.ImageWriter.ContentStore(), desc.Digest, targetName, e.insecure, e.opt.RegistryHosts, e.pushByDigest); err != nil {
 					return nil, err
 				}
 			}

exporter/exporter.go

@@ -12,7 +12,7 @@ type Exporter interface {
 
 type ExporterInstance interface {
 	Name() string
-	Export(context.Context, Source) (map[string]string, error)
+	Export(ctx context.Context, src Source, sessionID string) (map[string]string, error)
 }
 
 type Source struct {

exporter/local/export.go

@@ -14,7 +14,6 @@ import (
 	"github.com/moby/buildkit/session/filesync"
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/util/progress"
-	"github.com/pkg/errors"
 	"github.com/tonistiigi/fsutil"
 	fstypes "github.com/tonistiigi/fsutil/types"
 	"golang.org/x/sync/errgroup"
@@ -36,33 +35,27 @@ func New(opt Opt) (exporter.Exporter, error) {
 }
 
 func (e *localExporter) Resolve(ctx context.Context, opt map[string]string) (exporter.ExporterInstance, error) {
-	id := session.FromContext(ctx)
-	if id == "" {
-		return nil, errors.New("could not access local files without session")
-	}
-
-	timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
-	defer cancel()
-
-	caller, err := e.opt.SessionManager.Get(timeoutCtx, id)
-	if err != nil {
-		return nil, err
-	}
-
-	li := &localExporterInstance{localExporter: e, caller: caller}
-	return li, nil
+	return &localExporterInstance{localExporter: e}, nil
 }
 
 type localExporterInstance struct {
 	*localExporter
-	caller session.Caller
 }
 
 func (e *localExporterInstance) Name() string {
 	return "exporting to client"
 }
 
-func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source) (map[string]string, error) {
+func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source, sessionID string) (map[string]string, error) {
+
+	timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+
+	caller, err := e.opt.SessionManager.Get(timeoutCtx, sessionID)
+	if err != nil {
+		return nil, err
+	}
+
 	isMap := len(inp.Refs) > 0
 
 	export := func(ctx context.Context, k string, ref cache.ImmutableRef) func() error {
@@ -125,7 +118,7 @@ func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source)
 			}
 
 			progress := newProgressHandler(ctx, lbl)
-			if err := filesync.CopyToCaller(ctx, fs, e.caller, progress); err != nil {
+			if err := filesync.CopyToCaller(ctx, fs, caller, progress); err != nil {
 				return err
 			}
 			return nil

exporter/oci/export.go

@@ -49,23 +49,9 @@ func New(opt Opt) (exporter.Exporter, error) {
 }
 
 func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exporter.ExporterInstance, error) {
-	id := session.FromContext(ctx)
-	if id == "" {
-		return nil, errors.New("could not access local files without session")
-	}
-
-	timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
-	defer cancel()
-
-	caller, err := e.opt.SessionManager.Get(timeoutCtx, id)
-	if err != nil {
-		return nil, err
-	}
-
 	var ot *bool
 	i := &imageExporterInstance{
 		imageExporter:    e,
-		caller:           caller,
 		layerCompression: blobs.DefaultCompression,
 	}
 	for k, v := range opt {
@@ -110,7 +96,6 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
 type imageExporterInstance struct {
 	*imageExporter
 	meta             map[string][]byte
-	caller           session.Caller
 	name             string
 	ociTypes         bool
 	layerCompression blobs.CompressionType
@@ -120,7 +105,7 @@ func (e *imageExporterInstance) Name() string {
 	return "exporting to oci image format"
 }
 
-func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source) (map[string]string, error) {
+func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source, sessionID string) (map[string]string, error) {
 	if e.opt.Variant == VariantDocker && len(src.Refs) > 0 {
 		return nil, errors.Errorf("docker exporter does not currently support exporting manifest lists")
 	}
@@ -175,7 +160,15 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source)
 		return nil, errors.Errorf("invalid variant %q", e.opt.Variant)
 	}
 
-	w, err := filesync.CopyFileWriter(ctx, resp, e.caller)
+	timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+
+	caller, err := e.opt.SessionManager.Get(timeoutCtx, sessionID)
+	if err != nil {
+		return nil, err
+	}
+
+	w, err := filesync.CopyFileWriter(ctx, resp, caller)
 	if err != nil {
 		return nil, err
 	}

exporter/tar/export.go

@@ -14,7 +14,6 @@ import (
 	"github.com/moby/buildkit/session/filesync"
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/util/progress"
-	"github.com/pkg/errors"
 	"github.com/tonistiigi/fsutil"
 	fstypes "github.com/tonistiigi/fsutil/types"
 )
@@ -34,33 +33,19 @@ func New(opt Opt) (exporter.Exporter, error) {
 }
 
 func (e *localExporter) Resolve(ctx context.Context, opt map[string]string) (exporter.ExporterInstance, error) {
-	id := session.FromContext(ctx)
-	if id == "" {
-		return nil, errors.New("could not access local files without session")
-	}
-
-	timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
-	defer cancel()
-
-	caller, err := e.opt.SessionManager.Get(timeoutCtx, id)
-	if err != nil {
-		return nil, err
-	}
-
-	li := &localExporterInstance{localExporter: e, caller: caller}
+	li := &localExporterInstance{localExporter: e}
 	return li, nil
 }
 
 type localExporterInstance struct {
 	*localExporter
-	caller session.Caller
 }
 
 func (e *localExporterInstance) Name() string {
 	return "exporting to client"
 }
 
-func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source) (map[string]string, error) {
+func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source, sessionID string) (map[string]string, error) {
 	var defers []func()
 
 	defer func() {
@@ -147,7 +132,15 @@ func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source)
 		fs = d.FS
 	}
 
-	w, err := filesync.CopyFileWriter(ctx, nil, e.caller)
+	timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+
+	caller, err := e.opt.SessionManager.Get(timeoutCtx, sessionID)
+	if err != nil {
+		return nil, err
+	}
+
+	w, err := filesync.CopyFileWriter(ctx, nil, caller)
 	if err != nil {
 		return nil, err
 	}

frontend/frontend.go

@@ -14,11 +14,11 @@ import (
 )
 
 type Frontend interface {
-	Solve(ctx context.Context, llb FrontendLLBBridge, opt map[string]string, inputs map[string]*pb.Definition) (*Result, error)
+	Solve(ctx context.Context, llb FrontendLLBBridge, opt map[string]string, inputs map[string]*pb.Definition, sid string) (*Result, error)
 }
 
 type FrontendLLBBridge interface {
-	Solve(ctx context.Context, req SolveRequest) (*Result, error)
+	Solve(ctx context.Context, req SolveRequest, sid string) (*Result, error)
 	ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt) (digest.Digest, []byte, error)
 	Exec(ctx context.Context, meta executor.Meta, rootfs cache.ImmutableRef, stdin io.ReadCloser, stdout, stderr io.WriteCloser) error
 }