Skip to content

Commit

Permalink
Merge pull request #3289 from jedevc/attestations-exporter-tar
Browse files Browse the repository at this point in the history 
Export attestations for tar exporter
  • Loading branch information
@jedevc
jedevc authored Nov 28, 2022
2 parents 7804e2c + aa76a1e commit 16d8570
Show file tree
Hide file tree
Showing 4 changed files with 264 additions and 208 deletions.
56 changes: 56 additions & 0 deletions client/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7277,6 +7277,62 @@ func testExportAttestations(t *testing.T, sb integration.Sandbox) {
require.Equal(t, subjects, attest2.Subject)
}
})

t.Run("tar", func(t *testing.T) {
dir := t.TempDir()
out := filepath.Join(dir, "out.tar")
outW, err := os.Create(out)
require.NoError(t, err)

_, err = c.Build(sb.Context(), SolveOpt{
Exports: []ExportEntry{
{
Type: ExporterTar,
Output: fixedWriteCloser(outW),
Attrs: map[string]string{
"attestation-prefix": "test.",
},
},
},
}, "", frontend, nil)
require.NoError(t, err)

dt, err := os.ReadFile(out)
require.NoError(t, err)

m, err := testutil.ReadTarToMap(dt, false)
require.NoError(t, err)

for _, p := range ps {
var attest intoto.Statement
dt := m[path.Join(strings.ReplaceAll(platforms.Format(p), "/", "_"), "test.attestation.json")].Data
require.NoError(t, json.Unmarshal(dt, &attest))

require.Equal(t, "https://in-toto.io/Statement/v0.1", attest.Type)
require.Equal(t, "https://example.com/attestations/v1.0", attest.PredicateType)
require.Equal(t, map[string]interface{}{"success": true}, attest.Predicate)

require.Equal(t, []intoto.Subject{{
Name: "greeting",
Digest: result.ToDigestMap(digest.Canonical.FromString("hello " + platforms.Format(p) + "!")),
}}, attest.Subject)

var attest2 intoto.Statement
dt = m[path.Join(strings.ReplaceAll(platforms.Format(p), "/", "_"), "test.attestation2.json")].Data
require.NoError(t, json.Unmarshal(dt, &attest2))

require.Equal(t, "https://in-toto.io/Statement/v0.1", attest2.Type)
require.Equal(t, "https://example.com/attestations2/v1.0", attest2.PredicateType)
require.Nil(t, attest2.Predicate)
subjects := []intoto.Subject{{
Name: "/attestation.json",
Digest: map[string]string{
"sha256": successDigest.Encoded(),
},
}}
require.Equal(t, subjects, attest2.Subject)
}
})
}

func testAttestationDefaultSubject(t *testing.T, sb integration.Sandbox) {
Expand Down
155 changes: 18 additions & 137 deletions exporter/local/export.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,28 +3,18 @@ package local
import (
"context"
"encoding/json"
"io"
"io/fs"
"os"
"path"
"strings"
"time"

"github.com/docker/docker/pkg/idtools"
intoto "github.com/in-toto/in-toto-golang/in_toto"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/exporter/attestation"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
"github.com/moby/buildkit/exporter/util/epoch"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/filesync"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/solver/result"
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/util/staticfs"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/tonistiigi/fsutil"
fstypes "github.com/tonistiigi/fsutil/types"
Expand Down Expand Up @@ -56,12 +46,17 @@ func (e *localExporter) Resolve(ctx context.Context, opt map[string]string) (exp
return nil, err
}

i := &localExporterInstance{localExporter: e, epoch: tm}
i := &localExporterInstance{
localExporter: e,
opts: CreateFSOpts{
Epoch: tm,
},
}

for k, v := range opt {
switch k {
case keyAttestationPrefix:
i.attestationPrefix = v
i.opts.AttestationPrefix = v
}
}

Expand All @@ -70,8 +65,7 @@ func (e *localExporter) Resolve(ctx context.Context, opt map[string]string) (exp

type localExporterInstance struct {
*localExporter
epoch *time.Time
attestationPrefix string
opts CreateFSOpts
}

func (e *localExporterInstance) Name() string {
Expand All @@ -86,11 +80,11 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source
timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()

if e.epoch == nil {
if e.opts.Epoch == nil {
if tm, ok, err := epoch.ParseSource(inp); err != nil {
return nil, err
} else if ok {
e.epoch = tm
e.opts.Epoch = tm
}
}

Expand All @@ -114,127 +108,14 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source

now := time.Now().Truncate(time.Second)

export := func(ctx context.Context, k string, p *ocispecs.Platform, ref cache.ImmutableRef, attestations []result.Attestation) func() error {
export := func(ctx context.Context, k string, ref cache.ImmutableRef, attestations []result.Attestation) func() error {
return func() error {
var src string
var err error
var idmap *idtools.IdentityMapping
if ref == nil {
src, err = os.MkdirTemp("", "buildkit")
if err != nil {
return err
}
defer os.RemoveAll(src)
} else {
mount, err := ref.Mount(ctx, true, session.NewGroup(sessionID))
if err != nil {
return err
}

lm := snapshot.LocalMounter(mount)

src, err = lm.Mount()
if err != nil {
return err
}

idmap = mount.IdentityMapping()

defer lm.Unmount()
}

walkOpt := &fsutil.WalkOpt{}
var idMapFunc func(p string, st *fstypes.Stat) fsutil.MapResult
if idmap != nil {
idMapFunc = func(p string, st *fstypes.Stat) fsutil.MapResult {
uid, gid, err := idmap.ToContainer(idtools.Identity{
UID: int(st.Uid),
GID: int(st.Gid),
})
if err != nil {
return fsutil.MapResultExclude
}
st.Uid = uint32(uid)
st.Gid = uint32(gid)
return fsutil.MapResultKeep
}
}
walkOpt.Map = func(p string, st *fstypes.Stat) fsutil.MapResult {
res := fsutil.MapResultKeep
if idMapFunc != nil {
res = idMapFunc(p, st)
}
if e.epoch != nil {
st.ModTime = e.epoch.UnixNano()
}
return res
}

outputFS := fsutil.NewFS(src, walkOpt)

attestations, err = attestation.Unbundle(ctx, session.NewGroup(sessionID), inp.Refs, attestations)
outputFS, cleanup, err := CreateFS(ctx, sessionID, k, ref, inp.Refs, attestations, now, e.opts)
if err != nil {
return err
}
if len(attestations) > 0 {
subjects := []intoto.Subject{}
err = outputFS.Walk(ctx, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
if !info.Mode().IsRegular() {
return nil
}
f, err := outputFS.Open(path)
if err != nil {
return err
}
defer f.Close()
d := digest.Canonical.Digester()
if _, err := io.Copy(d.Hash(), f); err != nil {
return err
}
subjects = append(subjects, intoto.Subject{
Name: path,
Digest: result.ToDigestMap(d.Digest()),
})
return nil
})
if err != nil {
return err
}

stmts, err := attestation.MakeInTotoStatements(ctx, session.NewGroup(sessionID), inp.Refs, attestations, subjects)
if err != nil {
return err
}
stmtFS := staticfs.NewFS()

names := map[string]struct{}{}
for i, stmt := range stmts {
dt, err := json.Marshal(stmt)
if err != nil {
return errors.Wrap(err, "failed to marshal attestation")
}

if attestations[i].Path == "" {
return errors.New("attestation does not have set path")
}
name := e.attestationPrefix + path.Base(attestations[i].Path)
if _, ok := names[name]; ok {
return errors.Errorf("duplicate attestation path name %s", name)
}
names[name] = struct{}{}

st := fstypes.Stat{
Mode: 0600,
Path: name,
ModTime: now.UnixNano(),
}
stmtFS.Add(name, st, dt)
}

outputFS = staticfs.NewMergeFS(outputFS, stmtFS)
if cleanup != nil {
defer cleanup()
}

lbl := "copying files"
Expand All @@ -244,8 +125,8 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source
Mode: uint32(os.ModeDir | 0755),
Path: strings.Replace(k, "/", "_", -1),
}
if e.epoch != nil {
st.ModTime = e.epoch.UnixNano()
if e.opts.Epoch != nil {
st.ModTime = e.opts.Epoch.UnixNano()
}

outputFS, err = fsutil.SubDirFS([]fsutil.Dir{{FS: outputFS, Stat: st}})
Expand All @@ -270,13 +151,13 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source
if !ok {
return nil, errors.Errorf("failed to find ref for ID %s", p.ID)
}
eg.Go(export(ctx, p.ID, &p.Platform, r, inp.Attestations[p.ID]))
eg.Go(export(ctx, p.ID, r, inp.Attestations[p.ID]))
if !isMap {
break
}
}
} else {
eg.Go(export(ctx, "", nil, inp.Ref, nil))
eg.Go(export(ctx, "", inp.Ref, nil))
}

if err := eg.Wait(); err != nil {
Expand Down
Loading

0 comments on commit 16d8570

Please sign in to comment.