From fe42fb4a2e9c585744a1d71bae42e12002d1e2e4 Mon Sep 17 00:00:00 2001 From: Yves Brissaud Date: Tue, 5 Nov 2019 16:06:35 +0100 Subject: [PATCH] save relocation map from cnab-to-oci When a relocation map is generated from `cnab-to-oci` save it within the same directory as the `bundle.json` file. More information about reloaction map can be found in corresponding `cnab-to-oci` issue: https://github.com/docker/cnab-to-oci/issues/58 The `bundle.Bundle` struct is now wrapped in a `relocated.Bundle` that will also contain the relocation map. Methods to fetch `bundle.json` and `relocation-map.json` as well as en entire bundle with the relocation map at once are moved to a `fetch` package to avoid dependency cycle. Signed-off-by: Yves Brissaud --- Gopkg.lock | 2 + e2e/build_test.go | 9 ++- e2e/relocation_test.go | 39 +++++++++++ internal/cnab/cnab.go | 32 ++++----- internal/commands/image/inspect.go | 2 +- internal/commands/image/list.go | 5 +- internal/commands/image/list_test.go | 34 ++++++---- internal/commands/image/render.go | 2 +- internal/commands/image/tag.go | 7 +- internal/commands/image/tag_test.go | 15 +++-- internal/commands/push.go | 25 +++---- internal/commands/run.go | 11 ++-- internal/commands/update.go | 2 +- internal/packager/bundle.go | 6 +- internal/relocated/bundle.go | 99 ++++++++++++++++++++++++++++ internal/store/bundle.go | 46 +++++-------- internal/store/bundle_test.go | 14 ++-- internal/store/digest.go | 7 +- internal/store/digest_test.go | 4 +- 19 files changed, 250 insertions(+), 111 deletions(-) create mode 100644 e2e/relocation_test.go create mode 100644 internal/relocated/bundle.go diff --git a/Gopkg.lock b/Gopkg.lock index 6f5ce6235..2386d22e1 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1427,6 +1427,7 @@ "github.com/docker/cli/cli/context/store", "github.com/docker/cli/cli/flags", "github.com/docker/cli/opts", + "github.com/docker/cnab-to-oci/relocation", "github.com/docker/cnab-to-oci/remotes", "github.com/docker/distribution/reference", "github.com/docker/docker/api/types", @@ -1440,6 +1441,7 @@ "github.com/docker/docker/pkg/term", "github.com/docker/docker/registry", "github.com/docker/go-units", + "github.com/docker/go/canonical/json", "github.com/imdario/mergo", "github.com/jackpal/gateway", "github.com/moby/buildkit/client", diff --git a/e2e/build_test.go b/e2e/build_test.go index 558113992..b4eec8f84 100644 --- a/e2e/build_test.go +++ b/e2e/build_test.go @@ -9,6 +9,8 @@ import ( "strings" "testing" + "github.com/docker/app/internal/relocated" + "gotest.tools/fs" "github.com/docker/app/internal/store" @@ -31,10 +33,7 @@ func TestBuild(t *testing.T) { cfg := getDockerConfigDir(t, cmd) f := path.Join(cfg, "app", "bundles", "docker.io", "library", "single", "_tags", "1.0.0", "bundle.json") - data, err := ioutil.ReadFile(f) - assert.NilError(t, err) - var bndl bundle.Bundle - err = json.Unmarshal(data, &bndl) + bndl, err := relocated.BundleFromFile(f) assert.NilError(t, err) built := []string{bndl.InvocationImages[0].Digest, bndl.Images["web"].Digest, bndl.Images["worker"].Digest} @@ -53,7 +52,7 @@ func TestBuild(t *testing.T) { bytes, err := ioutil.ReadFile(iidfile) assert.NilError(t, err) iid := string(bytes) - actualID, err := store.FromBundle(&bndl) + actualID, err := store.FromBundle(bndl) assert.NilError(t, err) assert.Equal(t, iid, fmt.Sprintf("sha256:%s", actualID.String())) }) diff --git a/e2e/relocation_test.go b/e2e/relocation_test.go new file mode 100644 index 000000000..4b30558fb --- /dev/null +++ b/e2e/relocation_test.go @@ -0,0 +1,39 @@ +package e2e + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "gotest.tools/assert" + "gotest.tools/icmd" +) + +func TestRelocationMapCreatedOnPull(t *testing.T) { + runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + cmd := info.configuredCmd + cfg := getDockerConfigDir(t, cmd) + + path := filepath.Join("testdata", "local") + ref := info.registryAddress + "/test/local:a-tag" + bundlePath := filepath.Join(cfg, "app", "bundles", strings.Replace(info.registryAddress, ":", "_", 1), "test", "local", "_tags", "a-tag") + + // Given a pushed application + build(t, cmd, dockerCli, ref, path) + cmd.Command = dockerCli.Command("app", "push", ref) + icmd.RunCmd(cmd).Assert(t, icmd.Success) + // And given application files are remove + assert.NilError(t, os.RemoveAll(bundlePath)) + _, err := os.Stat(filepath.Join(bundlePath, "bundle.json")) + assert.Assert(t, os.IsNotExist(err)) + + // When application is pulled + cmd.Command = dockerCli.Command("app", "pull", ref) + icmd.RunCmd(cmd).Assert(t, icmd.Success) + + // Then the relocation map should exist + _, err = os.Stat(filepath.Join(bundlePath, "relocation-map.json")) + assert.NilError(t, err) + }) +} diff --git a/internal/cnab/cnab.go b/internal/cnab/cnab.go index b07407e47..a9f71b89e 100644 --- a/internal/cnab/cnab.go +++ b/internal/cnab/cnab.go @@ -3,13 +3,12 @@ package cnab import ( "context" "fmt" - "io/ioutil" "os" - "github.com/deislabs/cnab-go/bundle" "github.com/docker/app/internal" "github.com/docker/app/internal/log" "github.com/docker/app/internal/packager" + "github.com/docker/app/internal/relocated" "github.com/docker/app/internal/store" appstore "github.com/docker/app/internal/store" "github.com/docker/cli/cli/command" @@ -48,30 +47,20 @@ func getAppNameKind(name string) (string, nameKind) { return name, nameKindReference } -func extractAndLoadAppBasedBundle(dockerCli command.Cli, name string) (*bundle.Bundle, string, error) { +func extractAndLoadAppBasedBundle(dockerCli command.Cli, name string) (*relocated.Bundle, string, error) { app, err := packager.Extract(name) if err != nil { return nil, "", err } defer app.Cleanup() bndl, err := packager.MakeBundleFromApp(dockerCli, app, nil) - return bndl, "", err -} - -// LoadBundleFromFile loads a bundle from a file -func LoadBundleFromFile(filename string) (*bundle.Bundle, error) { - b := &bundle.Bundle{} - data, err := ioutil.ReadFile(filename) - if err != nil { - return b, err - } - return bundle.Unmarshal(data) + return relocated.FromBundle(bndl), "", err } // ResolveBundle looks for a CNAB bundle which can be in a Docker App Package format or // a bundle stored locally or in the bundle store. It returns a built or found bundle, // a reference to the bundle if it is found in the bundlestore, and an error. -func ResolveBundle(dockerCli command.Cli, bundleStore appstore.BundleStore, name string) (*bundle.Bundle, string, error) { +func ResolveBundle(dockerCli command.Cli, bundleStore appstore.BundleStore, name string) (*relocated.Bundle, string, error) { // resolution logic: // - if there is a docker-app package in working directory or if a directory is given use packager.Extract // - pull the bundle from the registry and add it to the bundle store @@ -90,7 +79,7 @@ func ResolveBundle(dockerCli command.Cli, bundleStore appstore.BundleStore, name } // GetBundle searches for the bundle locally and tries to pull it if not found -func GetBundle(dockerCli command.Cli, bundleStore appstore.BundleStore, name string) (*bundle.Bundle, reference.Reference, error) { +func GetBundle(dockerCli command.Cli, bundleStore appstore.BundleStore, name string) (*relocated.Bundle, reference.Reference, error) { bndl, ref, err := getBundleFromStore(bundleStore, name) if err != nil { named, err := store.StringToNamedRef(name) @@ -108,7 +97,7 @@ func GetBundle(dockerCli command.Cli, bundleStore appstore.BundleStore, name str return bndl, ref, nil } -func getBundleFromStore(bundleStore appstore.BundleStore, name string) (*bundle.Bundle, reference.Reference, error) { +func getBundleFromStore(bundleStore appstore.BundleStore, name string) (*relocated.Bundle, reference.Reference, error) { ref, err := bundleStore.LookUp(name) if err != nil { logrus.Debugf("Unable to find reference %q in the bundle store", name) @@ -123,18 +112,19 @@ func getBundleFromStore(bundleStore appstore.BundleStore, name string) (*bundle. } // PullBundle pulls the bundle and stores it into the bundle store -func PullBundle(dockerCli command.Cli, bundleStore appstore.BundleStore, tagRef reference.Named) (*bundle.Bundle, error) { +func PullBundle(dockerCli command.Cli, bundleStore appstore.BundleStore, tagRef reference.Named) (*relocated.Bundle, error) { insecureRegistries, err := internal.InsecureRegistriesFromEngine(dockerCli) if err != nil { return nil, fmt.Errorf("could not retrieve insecure registries: %v", err) } - bndl, _, err := remotes.Pull(log.WithLogContext(context.Background()), reference.TagNameOnly(tagRef), remotes.CreateResolver(dockerCli.ConfigFile(), insecureRegistries...)) + bndl, relocationMap, err := remotes.Pull(log.WithLogContext(context.Background()), reference.TagNameOnly(tagRef), remotes.CreateResolver(dockerCli.ConfigFile(), insecureRegistries...)) if err != nil { return nil, err } - if _, err := bundleStore.Store(tagRef, bndl); err != nil { + relocatedBundle := &relocated.Bundle{Bundle: bndl, RelocationMap: relocationMap} + if _, err := bundleStore.Store(tagRef, relocatedBundle); err != nil { return nil, err } - return bndl, nil + return relocatedBundle, nil } diff --git a/internal/commands/image/inspect.go b/internal/commands/image/inspect.go index deb9358ca..fcc242134 100644 --- a/internal/commands/image/inspect.go +++ b/internal/commands/image/inspect.go @@ -70,7 +70,7 @@ func runInspect(dockerCli command.Cli, appname string, opts inspectOptions) erro if err != nil { return err } - installation.Bundle = bndl + installation.Bundle = bndl.Bundle driverImpl, errBuf, err := cnab.SetupDriver(installation, dockerCli, opts.InstallerContextOptions, os.Stdout) if err != nil { return err diff --git a/internal/commands/image/list.go b/internal/commands/image/list.go index c386cd71b..3eef5b390 100644 --- a/internal/commands/image/list.go +++ b/internal/commands/image/list.go @@ -7,7 +7,8 @@ import ( "strings" "text/tabwriter" - "github.com/deislabs/cnab-go/bundle" + "github.com/docker/app/internal/relocated" + "github.com/docker/app/internal/store" "github.com/docker/cli/cli/command" "github.com/docker/cli/cli/config" @@ -182,5 +183,5 @@ func getImageListColumns(options imageListOption) []imageListColumn { type pkg struct { ref reference.Reference - bundle *bundle.Bundle + bundle *relocated.Bundle } diff --git a/internal/commands/image/list_test.go b/internal/commands/image/list_test.go index 0bc57fa86..d34ee75b9 100644 --- a/internal/commands/image/list_test.go +++ b/internal/commands/image/list_test.go @@ -6,6 +6,8 @@ import ( "fmt" "testing" + "github.com/docker/app/internal/relocated" + "gotest.tools/assert" "github.com/deislabs/cnab-go/bundle" @@ -15,18 +17,18 @@ import ( ) type bundleStoreStubForListCmd struct { - refMap map[reference.Reference]*bundle.Bundle + refMap map[reference.Reference]*relocated.Bundle // in order to keep the reference in the same order between tests refList []reference.Reference } -func (b *bundleStoreStubForListCmd) Store(ref reference.Reference, bndle *bundle.Bundle) (reference.Digested, error) { - b.refMap[ref] = bndle +func (b *bundleStoreStubForListCmd) Store(ref reference.Reference, bndl *relocated.Bundle) (reference.Digested, error) { + b.refMap[ref] = bndl b.refList = append(b.refList, ref) - return store.FromBundle(bndle) + return store.FromBundle(bndl) } -func (b *bundleStoreStubForListCmd) Read(ref reference.Reference) (*bundle.Bundle, error) { +func (b *bundleStoreStubForListCmd) Read(ref reference.Reference) (*relocated.Bundle, error) { bndl, ok := b.refMap[ref] if ok { return bndl, nil @@ -54,17 +56,23 @@ func TestListCmd(t *testing.T) { parseReference(t, "foo/bar:1.0"), ref, } - bundles := []bundle.Bundle{ + bundles := []relocated.Bundle{ { - Name: "Digested App", + Bundle: &bundle.Bundle{ + Name: "Digested App", + }, }, { - Version: "1.0.0", - SchemaVersion: "1.0.0", - Name: "Foo App", + Bundle: &bundle.Bundle{ + Version: "1.0.0", + SchemaVersion: "1.0.0", + Name: "Foo App", + }, }, { - Name: "Quiet App", + Bundle: &bundle.Bundle{ + Name: "Quiet App", + }, }, } @@ -114,13 +122,13 @@ func parseReference(t *testing.T, s string) reference.Reference { return ref } -func testRunList(t *testing.T, refs []reference.Reference, bundles []bundle.Bundle, options imageListOption, expectedOutput string) { +func testRunList(t *testing.T, refs []reference.Reference, bundles []relocated.Bundle, options imageListOption, expectedOutput string) { var buf bytes.Buffer w := bufio.NewWriter(&buf) dockerCli, err := command.NewDockerCli(command.WithOutputStream(w)) assert.NilError(t, err) bundleStore := &bundleStoreStubForListCmd{ - refMap: make(map[reference.Reference]*bundle.Bundle), + refMap: make(map[reference.Reference]*relocated.Bundle), refList: []reference.Reference{}, } for i, ref := range refs { diff --git a/internal/commands/image/render.go b/internal/commands/image/render.go index ff107003c..4d3b70f5a 100644 --- a/internal/commands/image/render.go +++ b/internal/commands/image/render.go @@ -95,7 +95,7 @@ func prepareCustomAction(actionName string, dockerCli command.Cli, appname strin if err != nil { return nil, nil, nil, err } - installation.Bundle = bundle + installation.Bundle = bundle.Bundle if err := bdl.MergeBundleParameters(installation, bdl.WithFileParameters(opts.ParametersFiles), diff --git a/internal/commands/image/tag.go b/internal/commands/image/tag.go index 17b72f7d9..1cc97b79e 100644 --- a/internal/commands/image/tag.go +++ b/internal/commands/image/tag.go @@ -3,7 +3,8 @@ package image import ( "fmt" - "github.com/deislabs/cnab-go/bundle" + "github.com/docker/app/internal/relocated" + "github.com/docker/app/internal/store" "github.com/docker/cli/cli" "github.com/docker/cli/cli/config" @@ -46,7 +47,7 @@ func runTag(bundleStore store.BundleStore, srcAppImage, destAppImage string) err return storeBundle(srcRef, destAppImage, bundleStore) } -func readBundle(name string, bundleStore store.BundleStore) (*bundle.Bundle, error) { +func readBundle(name string, bundleStore store.BundleStore) (*relocated.Bundle, error) { cnabRef, err := bundleStore.LookUp(name) if err != nil { switch err.(type) { @@ -65,7 +66,7 @@ func readBundle(name string, bundleStore store.BundleStore) (*bundle.Bundle, err return bundle, nil } -func storeBundle(bundle *bundle.Bundle, name string, bundleStore store.BundleStore) error { +func storeBundle(bundle *relocated.Bundle, name string, bundleStore store.BundleStore) error { cnabRef, err := store.StringToNamedRef(name) if err != nil { return err diff --git a/internal/commands/image/tag_test.go b/internal/commands/image/tag_test.go index bf5daf1f1..230daf4b3 100644 --- a/internal/commands/image/tag_test.go +++ b/internal/commands/image/tag_test.go @@ -4,9 +4,10 @@ import ( "fmt" "testing" + "github.com/docker/app/internal/relocated" + "gotest.tools/assert" - "github.com/deislabs/cnab-go/bundle" "github.com/docker/distribution/reference" ) @@ -18,7 +19,7 @@ func parseRefOrDie(t *testing.T, ref string) reference.Named { } type bundleStoreStub struct { - ReadBundle *bundle.Bundle + ReadBundle *relocated.Bundle ReadError error StoredBundle string StoredError error @@ -27,7 +28,7 @@ type bundleStoreStub struct { LookUpError error } -func (b *bundleStoreStub) Store(ref reference.Reference, bndle *bundle.Bundle) (reference.Digested, error) { +func (b *bundleStoreStub) Store(ref reference.Reference, bndle *relocated.Bundle) (reference.Digested, error) { defer func() { b.StoredError = nil }() @@ -37,7 +38,7 @@ func (b *bundleStoreStub) Store(ref reference.Reference, bndle *bundle.Bundle) ( return b.StoredID, b.StoredError } -func (b *bundleStoreStub) Read(ref reference.Reference) (*bundle.Bundle, error) { +func (b *bundleStoreStub) Read(ref reference.Reference) (*relocated.Bundle, error) { defer func() { b.ReadBundle = nil b.ReadError = nil @@ -88,7 +89,7 @@ func TestUnexistingSource(t *testing.T) { func TestInvalidDestinationReference(t *testing.T) { // given a reference and a bundle is returned by bundleStore.LookUp and bundleStore.Read mockedBundleStore.LookUpRef = parseRefOrDie(t, "ref") - mockedBundleStore.ReadBundle = &bundle.Bundle{} + mockedBundleStore.ReadBundle = &relocated.Bundle{} // and given a bad destination reference const badRef = "b@d reference" @@ -100,7 +101,7 @@ func TestInvalidDestinationReference(t *testing.T) { func TestBundleNotStored(t *testing.T) { // given a reference and a bundle is returned by bundleStore.LookUp and bundleStore.Read mockedBundleStore.LookUpRef = parseRefOrDie(t, "src-app") - mockedBundleStore.ReadBundle = &bundle.Bundle{} + mockedBundleStore.ReadBundle = &relocated.Bundle{} // and given bundleStore.Store will return an error mockedBundleStore.StoredError = fmt.Errorf("error from bundleStore.Store") @@ -112,7 +113,7 @@ func TestBundleNotStored(t *testing.T) { func TestSuccessfulyTag(t *testing.T) { // given a reference and a bundle is returned by bundleStore.LookUp and bundleStore.Read mockedBundleStore.LookUpRef = parseRefOrDie(t, "src-app") - mockedBundleStore.ReadBundle = &bundle.Bundle{} + mockedBundleStore.ReadBundle = &relocated.Bundle{} // and given valid source and output references const ( srcRef = "src-app" diff --git a/internal/commands/push.go b/internal/commands/push.go index a5af7d4c2..79ad54d2b 100644 --- a/internal/commands/push.go +++ b/internal/commands/push.go @@ -8,8 +8,10 @@ import ( "os" "strings" + "github.com/docker/app/internal/relocated" + "github.com/docker/app/internal/store" + "github.com/containerd/containerd/platforms" - "github.com/deislabs/cnab-go/bundle" "github.com/docker/app/internal" "github.com/docker/app/internal/cnab" "github.com/docker/app/internal/log" @@ -51,9 +53,14 @@ func pushCmd(dockerCli command.Cli) *cobra.Command { } func runPush(dockerCli command.Cli, name string) error { + bundleStore, err := prepareBundleStore() + if err != nil { + return err + } + defer muteDockerCli(dockerCli)() // Get the bundle - bndl, ref, err := resolveReferenceAndBundle(dockerCli, name) + bndl, ref, err := resolveReferenceAndBundle(dockerCli, bundleStore, name) if err != nil { return err } @@ -68,12 +75,7 @@ func runPush(dockerCli command.Cli, name string) error { return pushBundle(dockerCli, bndl, cnabRef) } -func resolveReferenceAndBundle(dockerCli command.Cli, name string) (*bundle.Bundle, string, error) { - bundleStore, err := prepareBundleStore() - if err != nil { - return nil, "", err - } - +func resolveReferenceAndBundle(dockerCli command.Cli, bundleStore store.BundleStore, name string) (*relocated.Bundle, string, error) { bndl, ref, err := cnab.ResolveBundle(dockerCli, bundleStore, name) if err != nil { return nil, "", err @@ -84,7 +86,7 @@ func resolveReferenceAndBundle(dockerCli command.Cli, name string) (*bundle.Bund return bndl, ref, err } -func pushBundle(dockerCli command.Cli, bndl *bundle.Bundle, cnabRef reference.Named) error { +func pushBundle(dockerCli command.Cli, bndl *relocated.Bundle, cnabRef reference.Named) error { insecureRegistries, err := internal.InsecureRegistriesFromEngine(dockerCli) if err != nil { return errors.Wrap(err, "could not retrieve insecure registries") @@ -100,13 +102,14 @@ func pushBundle(dockerCli command.Cli, bndl *bundle.Bundle, cnabRef reference.Na remotes.WithPushImages(dockerCli.Client(), dockerCli.Out()), } // bundle fixup - relocationMap, err := remotes.FixupBundle(context.Background(), bndl, cnabRef, resolver, fixupOptions...) + relocationMap, err := remotes.FixupBundle(context.Background(), bndl.Bundle, cnabRef, resolver, fixupOptions...) if err != nil { return errors.Wrapf(err, "fixing up %q for push", cnabRef) } + bndl.RelocationMap = relocationMap // push bundle manifest logrus.Debugf("Pushing the bundle %q", cnabRef) - descriptor, err := remotes.Push(log.WithLogContext(context.Background()), bndl, relocationMap, cnabRef, resolver, true, withAppAnnotations) + descriptor, err := remotes.Push(log.WithLogContext(context.Background()), bndl.Bundle, bndl.RelocationMap, cnabRef, resolver, true, withAppAnnotations) if err != nil { return errors.Wrapf(err, "pushing to %q", cnabRef) } diff --git a/internal/commands/run.go b/internal/commands/run.go index 9a10ebaeb..c51f4f8cf 100644 --- a/internal/commands/run.go +++ b/internal/commands/run.go @@ -4,11 +4,12 @@ import ( "fmt" "os" + "github.com/docker/app/internal/relocated" + "github.com/deislabs/cnab-go/driver" "github.com/docker/app/internal/cliopts" "github.com/deislabs/cnab-go/action" - "github.com/deislabs/cnab-go/bundle" "github.com/deislabs/cnab-go/credentials" bdl "github.com/docker/app/internal/bundle" "github.com/docker/app/internal/cnab" @@ -75,7 +76,7 @@ func runCmd(dockerCli command.Cli) *cobra.Command { } func runCnab(dockerCli command.Cli, opts runOptions) error { - bndl, err := cnab.LoadBundleFromFile(opts.cnabBundle) + bndl, err := relocated.BundleFromFile(opts.cnabBundle) if err != nil { return errors.Wrapf(err, "failed to read bundle %q", opts.cnabBundle) } @@ -95,7 +96,7 @@ func runDockerApp(dockerCli command.Cli, appname string, opts runOptions) error return runBundle(dockerCli, bndl, opts, ref.String()) } -func runBundle(dockerCli command.Cli, bndl *bundle.Bundle, opts runOptions, ref string) (err error) { +func runBundle(dockerCli command.Cli, bndl *relocated.Bundle, opts runOptions, ref string) (err error) { _, installationStore, credentialStore, err := prepareStores(dockerCli.CurrentContext()) if err != nil { return err @@ -129,7 +130,7 @@ func runBundle(dockerCli command.Cli, bndl *bundle.Bundle, opts runOptions, ref if err != nil { return err } - installation.Bundle = bndl + installation.Bundle = bndl.Bundle if err := bdl.MergeBundleParameters(installation, bdl.WithFileParameters(opts.ParametersFiles), @@ -140,7 +141,7 @@ func runBundle(dockerCli command.Cli, bndl *bundle.Bundle, opts runOptions, ref ); err != nil { return err } - creds, err := prepareCredentialSet(bndl, opts.CredentialSetOpts(dockerCli, credentialStore)...) + creds, err := prepareCredentialSet(bndl.Bundle, opts.CredentialSetOpts(dockerCli, credentialStore)...) if err != nil { return err } diff --git a/internal/commands/update.go b/internal/commands/update.go index 12ba3f811..14f57bf7b 100644 --- a/internal/commands/update.go +++ b/internal/commands/update.go @@ -63,7 +63,7 @@ func runUpdate(dockerCli command.Cli, installationName string, opts updateOption if err != nil { return err } - installation.Bundle = b + installation.Bundle = b.Bundle } if err := bundle.MergeBundleParameters(installation, bundle.WithFileParameters(opts.ParametersFiles), diff --git a/internal/packager/bundle.go b/internal/packager/bundle.go index d482b8fdc..fd0158b77 100644 --- a/internal/packager/bundle.go +++ b/internal/packager/bundle.go @@ -6,6 +6,8 @@ import ( "fmt" "io/ioutil" + "github.com/docker/app/internal/relocated" + "github.com/deislabs/cnab-go/bundle" "github.com/docker/app/internal/store" "github.com/docker/app/types" @@ -71,7 +73,7 @@ func MakeCNABImageName(appName, appVersion, suffix string) (string, error) { } // PersistInBundleStore do store a bundle with optional reference and return it's ID -func PersistInBundleStore(ref reference.Reference, bndle *bundle.Bundle) (reference.Digested, error) { +func PersistInBundleStore(ref reference.Reference, bndl *bundle.Bundle) (reference.Digested, error) { appstore, err := store.NewApplicationStore(config.Dir()) if err != nil { return nil, err @@ -80,7 +82,7 @@ func PersistInBundleStore(ref reference.Reference, bndle *bundle.Bundle) (refere if err != nil { return nil, err } - return bundleStore.Store(ref, bndle) + return bundleStore.Store(ref, relocated.FromBundle(bndl)) } func GetNamedTagged(tag string) (reference.NamedTagged, error) { diff --git a/internal/relocated/bundle.go b/internal/relocated/bundle.go new file mode 100644 index 000000000..5418387db --- /dev/null +++ b/internal/relocated/bundle.go @@ -0,0 +1,99 @@ +package relocated + +import ( + "io/ioutil" + "os" + "path/filepath" + + "github.com/pkg/errors" + + "github.com/deislabs/cnab-go/bundle" + "github.com/docker/cnab-to-oci/relocation" + "github.com/docker/go/canonical/json" +) + +type Bundle struct { + *bundle.Bundle + RelocationMap relocation.ImageRelocationMap +} + +// FromBundle returns a RelocatedBundle with an empty relocation map. +func FromBundle(bndl *bundle.Bundle) *Bundle { + return &Bundle{ + Bundle: bndl, + RelocationMap: relocation.ImageRelocationMap{}, + } +} + +// BundleFromFile creates a relocated bundle based on the bundle file and relocation map. +func BundleFromFile(filename string) (*Bundle, error) { + bndl, err := bundleJSON(filename) + if err != nil { + return nil, errors.Wrapf(err, "failed to read bundle") + } + + relocationMapFileName := filepath.Join(filepath.Dir(filename), "relocation-map.json") + relocationMap, err := relocationMapJSON(relocationMapFileName) + if err != nil { + return nil, errors.Wrapf(err, "failed to read relocation map") + } + + return &Bundle{ + Bundle: bndl, + RelocationMap: relocationMap, + }, nil +} + +// writeRelocationMap serializes the relocation map and writes it to a file as JSON. +func (b *Bundle) writeRelocationMap(dest string, mode os.FileMode) error { + d, err := json.MarshalCanonical(b.RelocationMap) + if err != nil { + return err + } + return ioutil.WriteFile(dest, d, mode) +} + +// Store a bundle with the relocation map as json files. +func (b *Bundle) Store(dir string) error { + // store bundle.json + path := filepath.Join(dir, "bundle.json") + if err := b.WriteFile(path, 0644); err != nil { + return errors.Wrapf(err, "failed to store bundle") + } + + // store relocation map + relocationMapPath := filepath.Join(dir, "relocation-map.json") + if err := b.writeRelocationMap(relocationMapPath, 0644); err != nil { + return errors.Wrapf(err, "failed to store relocation map") + } + + return nil +} + +func bundleJSON(bundlePath string) (*bundle.Bundle, error) { + data, err := ioutil.ReadFile(bundlePath) + if err != nil { + return nil, errors.Wrapf(err, "failed to read file %s", bundlePath) + } + bndl, err := bundle.Unmarshal(data) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal file %s", bundlePath) + } + return bndl, nil +} + +func relocationMapJSON(relocationMapPath string) (relocation.ImageRelocationMap, error) { + relocationMap := relocation.ImageRelocationMap{} + _, err := os.Stat(relocationMapPath) + if os.IsNotExist(err) { + return relocationMap, nil + } + data, err := ioutil.ReadFile(relocationMapPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to read file %s", relocationMapPath) + } + if err := json.Unmarshal(data, &relocationMap); err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal file %s", relocationMapPath) + } + return relocationMap, nil +} diff --git a/internal/store/bundle.go b/internal/store/bundle.go index 419fd66c6..9b61f09ce 100644 --- a/internal/store/bundle.go +++ b/internal/store/bundle.go @@ -1,15 +1,14 @@ package store import ( - "encoding/json" "fmt" - "io/ioutil" "os" "path/filepath" "sort" "strings" - "github.com/deislabs/cnab-go/bundle" + "github.com/docker/app/internal/relocated" + "github.com/docker/distribution/reference" digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" @@ -18,8 +17,8 @@ import ( // type BundleStore interface { // Store do store the bundle with optional reference, and return it's unique ID - Store(ref reference.Reference, bndle *bundle.Bundle) (reference.Digested, error) - Read(ref reference.Reference) (*bundle.Bundle, error) + Store(ref reference.Reference, bndl *relocated.Bundle) (reference.Digested, error) + Read(ref reference.Reference) (*relocated.Bundle, error) List() ([]reference.Reference, error) Remove(ref reference.Reference) error LookUp(refOrID string) (reference.Reference, error) @@ -63,8 +62,8 @@ func NewBundleStore(path string) (BundleStore, error) { // \_ bundle.json // -func (b *bundleStore) Store(ref reference.Reference, bndle *bundle.Bundle) (reference.Digested, error) { - id, err := FromBundle(bndle) +func (b *bundleStore) Store(ref reference.Reference, bndl *relocated.Bundle) (reference.Digested, error) { + id, err := FromBundle(bndl) if err != nil { return nil, errors.Wrapf(err, "failed to store bundle %q", ref) } @@ -75,28 +74,25 @@ func (b *bundleStore) Store(ref reference.Reference, bndle *bundle.Bundle) (refe if err != nil { return id, errors.Wrapf(err, "failed to store bundle %q", ref) } - path := filepath.Join(dir, "bundle.json") if err := os.MkdirAll(dir, 0755); err != nil { return id, errors.Wrapf(err, "failed to store bundle %q", ref) } - if err = bndle.WriteFile(path, 0644); err != nil { - return id, errors.Wrapf(err, "failed to store bundle %q", ref) + + if err := bndl.Store(dir); err != nil { + return id, errors.Wrapf(err, "failed to store reloacted bundle %q", ref) } + b.refsMap.appendRef(id, ref) return id, nil } -func (b *bundleStore) Read(ref reference.Reference) (*bundle.Bundle, error) { +func (b *bundleStore) Read(ref reference.Reference) (*relocated.Bundle, error) { paths, err := b.storePaths(ref) if err != nil { return nil, errors.Wrapf(err, "failed to read bundle %q", ref) } - bndl, err := b.fetchBundleJSON(filepath.Join(paths[0], "bundle.json")) - if err != nil { - return nil, errors.Wrapf(err, "failed to read bundle %q", ref) - } - return bndl, nil + return relocated.BundleFromFile(filepath.Join(paths[0], "bundle.json")) } // Returns the list of all bundles present in the bundle store @@ -256,6 +252,10 @@ func (b *bundleStore) processBundleStoreFile(path string, info os.FileInfo, err return nil } + if info.Name() == "relocation-map.json" { + return nil + } + if !strings.HasSuffix(info.Name(), ".json") { return nil } @@ -272,7 +272,7 @@ func (b *bundleStore) processBundleStoreFile(path string, info os.FileInfo, err if err != nil { return err } - bndl, err := b.fetchBundleJSON(path) + bndl, err := relocated.BundleFromFile(path) if err != nil { return err } @@ -285,18 +285,6 @@ func (b *bundleStore) processBundleStoreFile(path string, info os.FileInfo, err return nil } -func (b *bundleStore) fetchBundleJSON(bundlePath string) (*bundle.Bundle, error) { - data, err := ioutil.ReadFile(bundlePath) - if err != nil { - return nil, errors.Wrapf(err, "failed to read file %s", bundlePath) - } - var bndl bundle.Bundle - if err := json.Unmarshal(data, &bndl); err != nil { - return nil, errors.Wrapf(err, "failed to read file %s", bundlePath) - } - return &bndl, nil -} - func (b *bundleStore) pathToReference(path string) (reference.Named, error) { // Clean the path and remove the local bundle store path cleanpath := filepath.ToSlash(path) diff --git a/internal/store/bundle_test.go b/internal/store/bundle_test.go index 9b560cc37..9ba5d45c2 100644 --- a/internal/store/bundle_test.go +++ b/internal/store/bundle_test.go @@ -8,6 +8,8 @@ import ( "path/filepath" "testing" + "github.com/docker/app/internal/relocated" + "github.com/deislabs/cnab-go/bundle" "github.com/docker/distribution/reference" "gotest.tools/assert" @@ -26,7 +28,7 @@ func TestStoreAndReadBundle(t *testing.T) { bundleStore, err := appstore.BundleStore() assert.NilError(t, err) - expectedBundle := &bundle.Bundle{Name: "bundle-name"} + expectedBundle := relocated.FromBundle(&bundle.Bundle{Name: "bundle-name"}) testcases := []struct { name string @@ -222,7 +224,7 @@ func TestList(t *testing.T) { assert.Equal(t, len(bundles), 0) }) - bndl := &bundle.Bundle{Name: "bundle-name"} + bndl := relocated.FromBundle(&bundle.Bundle{Name: "bundle-name"}) for _, ref := range refs { _, err = bundleStore.Store(ref, bndl) assert.NilError(t, err) @@ -260,7 +262,7 @@ func TestRemove(t *testing.T) { parseRefOrDie(t, "my-repo/b-bundle@sha256:"+testSha), } - bndl := &bundle.Bundle{Name: "bundle-name"} + bndl := relocated.FromBundle(&bundle.Bundle{Name: "bundle-name"}) for _, ref := range refs { _, err = bundleStore.Store(ref, bndl) assert.NilError(t, err) @@ -300,7 +302,7 @@ func TestLookUp(t *testing.T) { assert.NilError(t, err) bundleStore, err := appstore.BundleStore() assert.NilError(t, err) - bndl := &bundle.Bundle{Name: "bundle-name"} + bndl := relocated.FromBundle(&bundle.Bundle{Name: "bundle-name"}) // Adding the bundle referenced by id id, err := bundleStore.Store(nil, bndl) assert.NilError(t, err) @@ -384,7 +386,7 @@ func TestScanBundles(t *testing.T) { defer dockerConfigDir.Remove() // Adding a bundle which should be referenced by id only - bndl1 := &bundle.Bundle{Name: "bundle-1"} + bndl1 := relocated.FromBundle(&bundle.Bundle{Name: "bundle-1"}) id1, err := FromBundle(bndl1) assert.NilError(t, err) dir1 := dockerConfigDir.Join("app", "bundles", "_ids", id1.String()) @@ -392,7 +394,7 @@ func TestScanBundles(t *testing.T) { assert.NilError(t, ioutil.WriteFile(filepath.Join(dir1, "bundle.json"), []byte(`{"name": "bundle-1"}`), 0644)) // Adding a bundle which should be referenced by id and tag - bndl2 := &bundle.Bundle{Name: "bundle-2"} + bndl2 := relocated.FromBundle(&bundle.Bundle{Name: "bundle-2"}) id2, err := FromBundle(bndl2) assert.NilError(t, err) dir2 := dockerConfigDir.Join("app", "bundles", "_ids", id2.String()) diff --git a/internal/store/digest.go b/internal/store/digest.go index 71640b719..13f445542 100644 --- a/internal/store/digest.go +++ b/internal/store/digest.go @@ -6,7 +6,8 @@ import ( "io" "regexp" - "github.com/deislabs/cnab-go/bundle" + "github.com/docker/app/internal/relocated" + "github.com/docker/distribution/reference" "github.com/opencontainers/go-digest" "github.com/pkg/errors" @@ -49,8 +50,8 @@ func FromString(s string) (ID, error) { return ID{digest}, nil } -func FromBundle(bndle *bundle.Bundle) (ID, error) { - digest, err := ComputeDigest(bndle) +func FromBundle(bndl *relocated.Bundle) (ID, error) { + digest, err := ComputeDigest(bndl) return ID{digest}, err } diff --git a/internal/store/digest_test.go b/internal/store/digest_test.go index 7d15ba1cd..919435b9e 100644 --- a/internal/store/digest_test.go +++ b/internal/store/digest_test.go @@ -5,6 +5,8 @@ import ( "os" "testing" + "github.com/docker/app/internal/relocated" + "github.com/deislabs/cnab-go/bundle" "gotest.tools/assert" "gotest.tools/fs" @@ -18,7 +20,7 @@ func Test_storeByDigest(t *testing.T) { bundleStore, err := appstore.BundleStore() assert.NilError(t, err) - bndl := &bundle.Bundle{Name: "bundle-name"} + bndl := relocated.FromBundle(&bundle.Bundle{Name: "bundle-name"}) ref := parseRefOrDie(t, "test/simple:1.0") _, err = bundleStore.Store(ref, bndl) assert.NilError(t, err)