Skip to content

Commit

Permalink
replace resolveimageconfig with generic sourcemetaresolver
Browse files Browse the repository at this point in the history
This is more versatile function that works for any source,
not just images.

It can be used together with a policy that switches
between input and output source as well as for adding
additional metadata for other sources in the future.

Signed-off-by: Tonis Tiigi <[email protected]>
  • Loading branch information
tonistiigi committed Feb 9, 2024
1 parent effe19a commit 9e14155
Show file tree
Hide file tree
Showing 36 changed files with 1,747 additions and 536 deletions.
5 changes: 5 additions & 0 deletions client/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,11 @@ func (g *gatewayClientForBuild) ResolveImageConfig(ctx context.Context, in *gate
return g.gateway.ResolveImageConfig(ctx, in, opts...)
}

func (g *gatewayClientForBuild) ResolveSourceMeta(ctx context.Context, in *gatewayapi.ResolveSourceMetaRequest, opts ...grpc.CallOption) (*gatewayapi.ResolveSourceMetaResponse, error) {
ctx = buildid.AppendToOutgoingContext(ctx, g.buildID)
return g.gateway.ResolveSourceMeta(ctx, in, opts...)
}

func (g *gatewayClientForBuild) Solve(ctx context.Context, in *gatewayapi.SolveRequest, opts ...grpc.CallOption) (*gatewayapi.SolveResponse, error) {
ctx = buildid.AppendToOutgoingContext(ctx, g.buildID)
return g.gateway.Solve(ctx, in, opts...)
Expand Down
5 changes: 3 additions & 2 deletions client/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import (
intoto "github.com/in-toto/in-toto-golang/in_toto"
controlapi "github.com/moby/buildkit/api/services/control"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
gateway "github.com/moby/buildkit/frontend/gateway/client"
gatewaypb "github.com/moby/buildkit/frontend/gateway/pb"
Expand Down Expand Up @@ -3061,7 +3062,7 @@ func testSourceDateEpochClamp(t *testing.T, sb integration.Sandbox) {

var bboxConfig []byte
_, err = c.Build(sb.Context(), SolveOpt{}, "", func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
_, _, bboxConfig, err = c.ResolveImageConfig(ctx, "docker.io/library/busybox:latest", llb.ResolveImageConfigOpt{})
_, _, bboxConfig, err = c.ResolveImageConfig(ctx, "docker.io/library/busybox:latest", sourceresolver.Opt{})
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -10059,7 +10060,7 @@ func testSourcePolicy(t *testing.T, sb integration.Sandbox) {
},
}

ref, dgst, _, err := c.ResolveImageConfig(ctx, origRef, llb.ResolveImageConfigOpt{
ref, dgst, _, err := c.ResolveImageConfig(ctx, origRef, sourceresolver.Opt{
SourcePolicies: pol,
})
if err != nil {
Expand Down
16 changes: 8 additions & 8 deletions client/llb/imagemetaresolver/resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/util/contentutil"
"github.com/moby/buildkit/util/imageutil"
"github.com/moby/buildkit/version"
Expand Down Expand Up @@ -70,32 +71,31 @@ type imageMetaResolver struct {
}

type resolveResult struct {
ref string
config []byte
dgst digest.Digest
}

func (imr *imageMetaResolver) ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt) (string, digest.Digest, []byte, error) {
func (imr *imageMetaResolver) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) {
imr.locker.Lock(ref)
defer imr.locker.Unlock(ref)

platform := opt.Platform
if platform == nil {
platform = imr.platform
platform := imr.platform
if opt.Platform != nil {
platform = opt.Platform
}

k := imr.key(ref, platform)

if res, ok := imr.cache[k]; ok {
return res.ref, res.dgst, res.config, nil
return ref, res.dgst, res.config, nil
}

ref, dgst, config, err := imageutil.Config(ctx, ref, imr.resolver, imr.buffer, nil, platform, opt.SourcePolicies)
dgst, config, err := imageutil.Config(ctx, ref, imr.resolver, imr.buffer, nil, platform)
if err != nil {
return "", "", nil, err
}

imr.cache[k] = resolveResult{dgst: dgst, config: config, ref: ref}
imr.cache[k] = resolveResult{dgst: dgst, config: config}
return ref, dgst, config, nil
}

Expand Down
34 changes: 2 additions & 32 deletions client/llb/resolver.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
package llb

import (
"context"

spb "github.com/moby/buildkit/sourcepolicy/pb"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/moby/buildkit/client/llb/sourceresolver"
)

// WithMetaResolver adds a metadata resolver to an image
Expand All @@ -31,30 +27,4 @@ func WithLayerLimit(l int) ImageOption {
}

// ImageMetaResolver can resolve image config metadata from a reference
type ImageMetaResolver interface {
ResolveImageConfig(ctx context.Context, ref string, opt ResolveImageConfigOpt) (string, digest.Digest, []byte, error)
}

type ResolverType int

const (
ResolverTypeRegistry ResolverType = iota
ResolverTypeOCILayout
)

type ResolveImageConfigOpt struct {
ResolverType

Platform *ocispecs.Platform
ResolveMode string
LogName string

Store ResolveImageConfigOptStore

SourcePolicies []*spb.Policy
}

type ResolveImageConfigOptStore struct {
SessionID string
StoreID string
}
type ImageMetaResolver = sourceresolver.ImageMetaResolver
3 changes: 2 additions & 1 deletion client/llb/resolver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"testing"

"github.com/containerd/containerd/platforms"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/solver/pb"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
Expand Down Expand Up @@ -74,7 +75,7 @@ type testResolver struct {
platform string
}

func (r *testResolver) ResolveImageConfig(ctx context.Context, ref string, opt ResolveImageConfigOpt) (string, digest.Digest, []byte, error) {
func (r *testResolver) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) {
var img struct {
Config struct {
Env []string `json:"Env,omitempty"`
Expand Down
19 changes: 11 additions & 8 deletions client/llb/source.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"strings"

"github.com/distribution/reference"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/apicaps"
"github.com/moby/buildkit/util/gitutil"
Expand Down Expand Up @@ -136,10 +137,11 @@ func Image(ref string, opts ...ImageOption) State {
if p == nil {
p = c.Platform
}
_, _, dt, err := info.metaResolver.ResolveImageConfig(ctx, ref, ResolveImageConfigOpt{
Platform: p,
ResolveMode: info.resolveMode.String(),
ResolverType: ResolverTypeRegistry,
_, _, dt, err := info.metaResolver.ResolveImageConfig(ctx, ref, sourceresolver.Opt{
Platform: p,
ImageOpt: &sourceresolver.ResolveImageOpt{
ResolveMode: info.resolveMode.String(),
},
})
if err != nil {
return State{}, err
Expand All @@ -152,10 +154,11 @@ func Image(ref string, opts ...ImageOption) State {
if p == nil {
p = c.Platform
}
ref, dgst, dt, err := info.metaResolver.ResolveImageConfig(context.TODO(), ref, ResolveImageConfigOpt{
Platform: p,
ResolveMode: info.resolveMode.String(),
ResolverType: ResolverTypeRegistry,
ref, dgst, dt, err := info.metaResolver.ResolveImageConfig(context.TODO(), ref, sourceresolver.Opt{
Platform: p,
ImageOpt: &sourceresolver.ResolveImageOpt{
ResolveMode: info.resolveMode.String(),
},
})
if err != nil {
return State{}, err
Expand Down
59 changes: 59 additions & 0 deletions client/llb/sourceresolver/imageresolver.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package sourceresolver

import (
"context"
"strings"

"github.com/distribution/reference"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/imageutil"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)

type ImageMetaResolver interface {
ResolveImageConfig(ctx context.Context, ref string, opt Opt) (string, digest.Digest, []byte, error)
}

type imageMetaResolver struct {
mr MetaResolver
}

var _ ImageMetaResolver = &imageMetaResolver{}

func NewImageMetaResolver(mr MetaResolver) ImageMetaResolver {
return &imageMetaResolver{
mr: mr,
}
}

func (imr *imageMetaResolver) ResolveImageConfig(ctx context.Context, ref string, opt Opt) (string, digest.Digest, []byte, error) {
parsed, err := reference.ParseNormalizedNamed(ref)
if err != nil {
return "", "", nil, errors.Wrapf(err, "could not parse reference %q", ref)
}
ref = parsed.String()
op := &pb.SourceOp{
Identifier: "docker-image://" + ref,
}
if opt := opt.OCILayoutOpt; opt != nil {
op.Identifier = "oci-layout://" + ref
op.Attrs = map[string]string{}
if opt.Store.SessionID != "" {
op.Attrs[pb.AttrOCILayoutSessionID] = opt.Store.SessionID
}
if opt.Store.StoreID != "" {
op.Attrs[pb.AttrOCILayoutStoreID] = opt.Store.StoreID
}
}
res, err := imr.mr.ResolveSourceMetadata(ctx, op, opt)
if err != nil {
return "", "", nil, errors.Wrapf(err, "failed to resolve source metadata for %s", ref)
}
if res.Image == nil {
return "", "", nil, &imageutil.ResolveToNonImageError{Ref: ref, Updated: res.Op.Identifier}
}
ref = strings.TrimPrefix(res.Op.Identifier, "docker-image://")
ref = strings.TrimPrefix(ref, "oci-layout://")
return ref, res.Image.Digest, res.Image.Config, nil
}
54 changes: 54 additions & 0 deletions client/llb/sourceresolver/types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package sourceresolver

import (
"context"

"github.com/moby/buildkit/solver/pb"
spb "github.com/moby/buildkit/sourcepolicy/pb"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
)

type ResolverType int

const (
ResolverTypeRegistry ResolverType = iota
ResolverTypeOCILayout
)

type MetaResolver interface {
ResolveSourceMetadata(ctx context.Context, op *pb.SourceOp, opt Opt) (*MetaResponse, error)
}

type Opt struct {
LogName string
SourcePolicies []*spb.Policy
Platform *ocispecs.Platform

ImageOpt *ResolveImageOpt
OCILayoutOpt *ResolveOCILayoutOpt
}

type MetaResponse struct {
Op *pb.SourceOp

Image *ResolveImageResponse
}

type ResolveImageOpt struct {
ResolveMode string
}

type ResolveImageResponse struct {
Digest digest.Digest
Config []byte
}

type ResolveOCILayoutOpt struct {
Store ResolveImageConfigOptStore
}

type ResolveImageConfigOptStore struct {
SessionID string
StoreID string
}
9 changes: 9 additions & 0 deletions control/gateway/gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,15 @@ func (gwf *GatewayForwarder) ResolveImageConfig(ctx context.Context, req *gwapi.
return fwd.ResolveImageConfig(ctx, req)
}

func (gwf *GatewayForwarder) ResolveSourceMeta(ctx context.Context, req *gwapi.ResolveSourceMetaRequest) (*gwapi.ResolveSourceMetaResponse, error) {
fwd, err := gwf.lookupForwarder(ctx)
if err != nil {
return nil, errors.Wrap(err, "forwarding ResolveSourceMeta")
}

return fwd.ResolveSourceMeta(ctx, req)
}

func (gwf *GatewayForwarder) Solve(ctx context.Context, req *gwapi.SolveRequest) (*gwapi.SolveResponse, error) {
fwd, err := gwf.lookupForwarder(ctx)
if err != nil {
Expand Down
6 changes: 4 additions & 2 deletions frontend/attestations/sbom/sbom.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

intoto "github.com/in-toto/in-toto-golang/in_toto"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
gatewaypb "github.com/moby/buildkit/frontend/gateway/pb"
"github.com/moby/buildkit/solver/result"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
Expand All @@ -33,12 +34,13 @@ const (
// attestation.
type Scanner func(ctx context.Context, name string, ref llb.State, extras map[string]llb.State, opts ...llb.ConstraintsOpt) (result.Attestation[*llb.State], error)

func CreateSBOMScanner(ctx context.Context, resolver llb.ImageMetaResolver, scanner string, resolveOpt llb.ResolveImageConfigOpt) (Scanner, error) {
func CreateSBOMScanner(ctx context.Context, resolver sourceresolver.MetaResolver, scanner string, resolveOpt sourceresolver.Opt) (Scanner, error) {
if scanner == "" {
return nil, nil
}

scanner, _, dt, err := resolver.ResolveImageConfig(ctx, scanner, resolveOpt)
imr := sourceresolver.NewImageMetaResolver(resolver)
scanner, _, dt, err := imr.ResolveImageConfig(ctx, scanner, resolveOpt)
if err != nil {
return nil, err
}
Expand Down
8 changes: 6 additions & 2 deletions frontend/dockerfile/builder/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

"github.com/containerd/containerd/platforms"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/exporter/containerimage/image"
"github.com/moby/buildkit/frontend"
"github.com/moby/buildkit/frontend/attestations/sbom"
Expand Down Expand Up @@ -101,8 +102,11 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) {

var scanner sbom.Scanner
if bc.SBOM != nil {
scanner, err = sbom.CreateSBOMScanner(ctx, c, bc.SBOM.Generator, llb.ResolveImageConfigOpt{
ResolveMode: opts["image-resolve-mode"],
// TODO: scanner should pass policy
scanner, err = sbom.CreateSBOMScanner(ctx, c, bc.SBOM.Generator, sourceresolver.Opt{
ImageOpt: &sourceresolver.ResolveImageOpt{
ResolveMode: opts["image-resolve-mode"],
},
})
if err != nil {
return nil, err
Expand Down
13 changes: 7 additions & 6 deletions frontend/dockerfile/dockerfile2llb/convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/docker/go-connections/nat"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/imagemetaresolver"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/exporter/containerimage/image"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/moby/buildkit/frontend/dockerfile/parser"
Expand Down Expand Up @@ -423,12 +424,12 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
prefix += platforms.Format(*platform) + " "
}
prefix += "internal]"
mutRef, dgst, dt, err := metaResolver.ResolveImageConfig(ctx, d.stage.BaseName, llb.ResolveImageConfigOpt{
Platform: platform,
ResolveMode: opt.ImageResolveMode.String(),
LogName: fmt.Sprintf("%s load metadata for %s", prefix, d.stage.BaseName),
ResolverType: llb.ResolverTypeRegistry,
SourcePolicies: nil,
mutRef, dgst, dt, err := metaResolver.ResolveImageConfig(ctx, d.stage.BaseName, sourceresolver.Opt{
LogName: fmt.Sprintf("%s load metadata for %s", prefix, d.stage.BaseName),
Platform: platform,
ImageOpt: &sourceresolver.ResolveImageOpt{
ResolveMode: opt.ImageResolveMode.String(),
},
})
if err != nil {
return suggest.WrapError(errors.Wrap(err, origName), origName, append(allStageNames, commonImageNames()...), true)
Expand Down
Loading

0 comments on commit 9e14155

Please sign in to comment.