diff --git a/Gopkg.lock b/Gopkg.lock index 810f7e04e..7d2bd70b0 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1419,6 +1419,7 @@ "github.com/docker/cli/cli/context/store", "github.com/docker/cli/cli/flags", "github.com/docker/cli/opts", + "github.com/docker/cnab-to-oci/relocation", "github.com/docker/cnab-to-oci/remotes", "github.com/docker/distribution/reference", "github.com/docker/docker/api/types", @@ -1432,6 +1433,7 @@ "github.com/docker/docker/pkg/term", "github.com/docker/docker/registry", "github.com/docker/go-units", + "github.com/docker/go/canonical/json", "github.com/imdario/mergo", "github.com/moby/buildkit/client", "github.com/moby/buildkit/session", diff --git a/e2e/build_test.go b/e2e/build_test.go index 558113992..01dc92ffb 100644 --- a/e2e/build_test.go +++ b/e2e/build_test.go @@ -9,6 +9,8 @@ import ( "strings" "testing" + "github.com/docker/app/internal/fetch" + "gotest.tools/fs" "github.com/docker/app/internal/store" @@ -31,10 +33,7 @@ func TestBuild(t *testing.T) { cfg := getDockerConfigDir(t, cmd) f := path.Join(cfg, "app", "bundles", "docker.io", "library", "single", "_tags", "1.0.0", "bundle.json") - data, err := ioutil.ReadFile(f) - assert.NilError(t, err) - var bndl bundle.Bundle - err = json.Unmarshal(data, &bndl) + bndl, err := fetch.RelocatedBundleFromFile(f) assert.NilError(t, err) built := []string{bndl.InvocationImages[0].Digest, bndl.Images["web"].Digest, bndl.Images["worker"].Digest} @@ -53,7 +52,7 @@ func TestBuild(t *testing.T) { bytes, err := ioutil.ReadFile(iidfile) assert.NilError(t, err) iid := string(bytes) - actualID, err := store.FromBundle(&bndl) + actualID, err := store.FromBundle(bndl) assert.NilError(t, err) assert.Equal(t, iid, fmt.Sprintf("sha256:%s", actualID.String())) }) diff --git a/e2e/relocation_test.go b/e2e/relocation_test.go new file mode 100644 index 000000000..7646f6f7e --- /dev/null +++ b/e2e/relocation_test.go @@ -0,0 +1,61 @@ +package e2e + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "gotest.tools/assert" + "gotest.tools/icmd" +) + +func TestRelocationMapCreatedOnPush(t *testing.T) { + runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + cmd := info.configuredCmd + cfg := getDockerConfigDir(t, cmd) + + path := filepath.Join("testdata", "local") + ref := info.registryAddress + "/test/local:a-tag" + bundlePath := filepath.Join(cfg, "app", "bundles", strings.Replace(info.registryAddress, ":", "_", 1), "test", "local", "_tags", "a-tag") + + // Given a locally built application + build(t, cmd, dockerCli, ref, path) + + // When pushing to a registry + cmd.Command = dockerCli.Command("app", "push", ref) + icmd.RunCmd(cmd).Assert(t, icmd.Success) + + // Then the relocation map should exist + _, err := os.Stat(filepath.Join(bundlePath, "relocation-map.json")) + assert.NilError(t, err) + }) +} + +func TestRelocationMapCreatedOnPull(t *testing.T) { + runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + cmd := info.configuredCmd + cfg := getDockerConfigDir(t, cmd) + + path := filepath.Join("testdata", "local") + ref := info.registryAddress + "/test/local:a-tag" + bundlePath := filepath.Join(cfg, "app", "bundles", strings.Replace(info.registryAddress, ":", "_", 1), "test", "local", "_tags", "a-tag") + + // Given a pushed application + build(t, cmd, dockerCli, ref, path) + cmd.Command = dockerCli.Command("app", "push", ref) + icmd.RunCmd(cmd).Assert(t, icmd.Success) + // And given application files are remove + assert.NilError(t, os.RemoveAll(bundlePath)) + _, err := os.Stat(filepath.Join(bundlePath, "bundle.json")) + assert.Assert(t, os.IsNotExist(err)) + + // When application is pulled + cmd.Command = dockerCli.Command("app", "pull", ref) + icmd.RunCmd(cmd).Assert(t, icmd.Success) + + // Then the relocation map should exist + _, err = os.Stat(filepath.Join(bundlePath, "relocation-map.json")) + assert.NilError(t, err) + }) +} diff --git a/internal/cnab/cnab.go b/internal/cnab/cnab.go index b07407e47..a9f71b89e 100644 --- a/internal/cnab/cnab.go +++ b/internal/cnab/cnab.go @@ -3,13 +3,12 @@ package cnab import ( "context" "fmt" - "io/ioutil" "os" - "github.com/deislabs/cnab-go/bundle" "github.com/docker/app/internal" "github.com/docker/app/internal/log" "github.com/docker/app/internal/packager" + "github.com/docker/app/internal/relocated" "github.com/docker/app/internal/store" appstore "github.com/docker/app/internal/store" "github.com/docker/cli/cli/command" @@ -48,30 +47,20 @@ func getAppNameKind(name string) (string, nameKind) { return name, nameKindReference } -func extractAndLoadAppBasedBundle(dockerCli command.Cli, name string) (*bundle.Bundle, string, error) { +func extractAndLoadAppBasedBundle(dockerCli command.Cli, name string) (*relocated.Bundle, string, error) { app, err := packager.Extract(name) if err != nil { return nil, "", err } defer app.Cleanup() bndl, err := packager.MakeBundleFromApp(dockerCli, app, nil) - return bndl, "", err -} - -// LoadBundleFromFile loads a bundle from a file -func LoadBundleFromFile(filename string) (*bundle.Bundle, error) { - b := &bundle.Bundle{} - data, err := ioutil.ReadFile(filename) - if err != nil { - return b, err - } - return bundle.Unmarshal(data) + return relocated.FromBundle(bndl), "", err } // ResolveBundle looks for a CNAB bundle which can be in a Docker App Package format or // a bundle stored locally or in the bundle store. It returns a built or found bundle, // a reference to the bundle if it is found in the bundlestore, and an error. -func ResolveBundle(dockerCli command.Cli, bundleStore appstore.BundleStore, name string) (*bundle.Bundle, string, error) { +func ResolveBundle(dockerCli command.Cli, bundleStore appstore.BundleStore, name string) (*relocated.Bundle, string, error) { // resolution logic: // - if there is a docker-app package in working directory or if a directory is given use packager.Extract // - pull the bundle from the registry and add it to the bundle store @@ -90,7 +79,7 @@ func ResolveBundle(dockerCli command.Cli, bundleStore appstore.BundleStore, name } // GetBundle searches for the bundle locally and tries to pull it if not found -func GetBundle(dockerCli command.Cli, bundleStore appstore.BundleStore, name string) (*bundle.Bundle, reference.Reference, error) { +func GetBundle(dockerCli command.Cli, bundleStore appstore.BundleStore, name string) (*relocated.Bundle, reference.Reference, error) { bndl, ref, err := getBundleFromStore(bundleStore, name) if err != nil { named, err := store.StringToNamedRef(name) @@ -108,7 +97,7 @@ func GetBundle(dockerCli command.Cli, bundleStore appstore.BundleStore, name str return bndl, ref, nil } -func getBundleFromStore(bundleStore appstore.BundleStore, name string) (*bundle.Bundle, reference.Reference, error) { +func getBundleFromStore(bundleStore appstore.BundleStore, name string) (*relocated.Bundle, reference.Reference, error) { ref, err := bundleStore.LookUp(name) if err != nil { logrus.Debugf("Unable to find reference %q in the bundle store", name) @@ -123,18 +112,19 @@ func getBundleFromStore(bundleStore appstore.BundleStore, name string) (*bundle. } // PullBundle pulls the bundle and stores it into the bundle store -func PullBundle(dockerCli command.Cli, bundleStore appstore.BundleStore, tagRef reference.Named) (*bundle.Bundle, error) { +func PullBundle(dockerCli command.Cli, bundleStore appstore.BundleStore, tagRef reference.Named) (*relocated.Bundle, error) { insecureRegistries, err := internal.InsecureRegistriesFromEngine(dockerCli) if err != nil { return nil, fmt.Errorf("could not retrieve insecure registries: %v", err) } - bndl, _, err := remotes.Pull(log.WithLogContext(context.Background()), reference.TagNameOnly(tagRef), remotes.CreateResolver(dockerCli.ConfigFile(), insecureRegistries...)) + bndl, relocationMap, err := remotes.Pull(log.WithLogContext(context.Background()), reference.TagNameOnly(tagRef), remotes.CreateResolver(dockerCli.ConfigFile(), insecureRegistries...)) if err != nil { return nil, err } - if _, err := bundleStore.Store(tagRef, bndl); err != nil { + relocatedBundle := &relocated.Bundle{Bundle: bndl, RelocationMap: relocationMap} + if _, err := bundleStore.Store(tagRef, relocatedBundle); err != nil { return nil, err } - return bndl, nil + return relocatedBundle, nil } diff --git a/internal/commands/image/inspect.go b/internal/commands/image/inspect.go index 6a679383b..f7baebd3d 100644 --- a/internal/commands/image/inspect.go +++ b/internal/commands/image/inspect.go @@ -65,7 +65,7 @@ func runInspect(dockerCli command.Cli, appname string, opts inspectOptions) erro if err != nil { return err } - installation.Bundle = bndl + installation.Bundle = bndl.Bundle driverImpl, errBuf := cnab.PrepareDriver(dockerCli, cnab.BindMount{}, nil) a := &action.RunCustom{ diff --git a/internal/commands/image/list.go b/internal/commands/image/list.go index e4feb0ef8..a04944e17 100644 --- a/internal/commands/image/list.go +++ b/internal/commands/image/list.go @@ -3,11 +3,11 @@ package image import ( "bytes" "fmt" + "github.com/docker/app/internal/relocated" "io" "strings" "text/tabwriter" - "github.com/deislabs/cnab-go/bundle" "github.com/docker/app/internal/store" "github.com/docker/cli/cli/command" "github.com/docker/cli/cli/config" @@ -143,5 +143,5 @@ var ( type pkg struct { ref reference.Reference - bundle *bundle.Bundle + bundle *relocated.Bundle } diff --git a/internal/commands/image/list_test.go b/internal/commands/image/list_test.go index de4996820..43ce11912 100644 --- a/internal/commands/image/list_test.go +++ b/internal/commands/image/list_test.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "fmt" + "github.com/docker/app/internal/relocated" "testing" "gotest.tools/assert" @@ -15,18 +16,18 @@ import ( ) type bundleStoreStubForListCmd struct { - refMap map[reference.Reference]*bundle.Bundle + refMap map[reference.Reference]*relocated.Bundle // in order to keep the reference in the same order between tests refList []reference.Reference } -func (b *bundleStoreStubForListCmd) Store(ref reference.Reference, bndle *bundle.Bundle) (reference.Digested, error) { - b.refMap[ref] = bndle +func (b *bundleStoreStubForListCmd) Store(ref reference.Reference, bndl *relocated.Bundle) (reference.Digested, error) { + b.refMap[ref] = bndl b.refList = append(b.refList, ref) - return store.FromBundle(bndle) + return store.FromBundle(bndl) } -func (b *bundleStoreStubForListCmd) Read(ref reference.Reference) (*bundle.Bundle, error) { +func (b *bundleStoreStubForListCmd) Read(ref reference.Reference) (*relocated.Bundle, error) { bndl, ok := b.refMap[ref] if ok { return bndl, nil @@ -52,20 +53,20 @@ func TestListWithQuietFlag(t *testing.T) { dockerCli, err := command.NewDockerCli(command.WithOutputStream(w)) assert.NilError(t, err) bundleStore := &bundleStoreStubForListCmd{ - refMap: make(map[reference.Reference]*bundle.Bundle), + refMap: make(map[reference.Reference]*relocated.Bundle), refList: []reference.Reference{}, } ref1, err := store.FromString("a855ac937f2ed375ba4396bbc49c4093e124da933acd2713fb9bc17d7562a087") assert.NilError(t, err) ref2, err := reference.Parse("foo/bar:1.0") assert.NilError(t, err) - _, err = bundleStore.Store(ref1, &bundle.Bundle{}) + _, err = bundleStore.Store(ref1, relocated.FromBundle(&bundle.Bundle{})) assert.NilError(t, err) - _, err = bundleStore.Store(ref2, &bundle.Bundle{ + _, err = bundleStore.Store(ref2, relocated.FromBundle(&bundle.Bundle{ Version: "1.0.0", SchemaVersion: "1.0.0", Name: "Foo App", - }) + })) assert.NilError(t, err) err = runList(dockerCli, imageListOption{quiet: true}, bundleStore) assert.NilError(t, err) diff --git a/internal/commands/image/tag.go b/internal/commands/image/tag.go index 17b72f7d9..b922f2693 100644 --- a/internal/commands/image/tag.go +++ b/internal/commands/image/tag.go @@ -2,8 +2,8 @@ package image import ( "fmt" + "github.com/docker/app/internal/relocated" - "github.com/deislabs/cnab-go/bundle" "github.com/docker/app/internal/store" "github.com/docker/cli/cli" "github.com/docker/cli/cli/config" @@ -46,7 +46,7 @@ func runTag(bundleStore store.BundleStore, srcAppImage, destAppImage string) err return storeBundle(srcRef, destAppImage, bundleStore) } -func readBundle(name string, bundleStore store.BundleStore) (*bundle.Bundle, error) { +func readBundle(name string, bundleStore store.BundleStore) (*relocated.Bundle, error) { cnabRef, err := bundleStore.LookUp(name) if err != nil { switch err.(type) { @@ -65,7 +65,7 @@ func readBundle(name string, bundleStore store.BundleStore) (*bundle.Bundle, err return bundle, nil } -func storeBundle(bundle *bundle.Bundle, name string, bundleStore store.BundleStore) error { +func storeBundle(bundle *relocated.Bundle, name string, bundleStore store.BundleStore) error { cnabRef, err := store.StringToNamedRef(name) if err != nil { return err diff --git a/internal/commands/image/tag_test.go b/internal/commands/image/tag_test.go index bf5daf1f1..6a5b1220f 100644 --- a/internal/commands/image/tag_test.go +++ b/internal/commands/image/tag_test.go @@ -2,11 +2,11 @@ package image import ( "fmt" + "github.com/docker/app/internal/relocated" "testing" "gotest.tools/assert" - "github.com/deislabs/cnab-go/bundle" "github.com/docker/distribution/reference" ) @@ -18,7 +18,7 @@ func parseRefOrDie(t *testing.T, ref string) reference.Named { } type bundleStoreStub struct { - ReadBundle *bundle.Bundle + ReadBundle *relocated.Bundle ReadError error StoredBundle string StoredError error @@ -27,7 +27,7 @@ type bundleStoreStub struct { LookUpError error } -func (b *bundleStoreStub) Store(ref reference.Reference, bndle *bundle.Bundle) (reference.Digested, error) { +func (b *bundleStoreStub) Store(ref reference.Reference, bndle *relocated.Bundle) (reference.Digested, error) { defer func() { b.StoredError = nil }() @@ -37,7 +37,7 @@ func (b *bundleStoreStub) Store(ref reference.Reference, bndle *bundle.Bundle) ( return b.StoredID, b.StoredError } -func (b *bundleStoreStub) Read(ref reference.Reference) (*bundle.Bundle, error) { +func (b *bundleStoreStub) Read(ref reference.Reference) (*relocated.Bundle, error) { defer func() { b.ReadBundle = nil b.ReadError = nil @@ -88,7 +88,7 @@ func TestUnexistingSource(t *testing.T) { func TestInvalidDestinationReference(t *testing.T) { // given a reference and a bundle is returned by bundleStore.LookUp and bundleStore.Read mockedBundleStore.LookUpRef = parseRefOrDie(t, "ref") - mockedBundleStore.ReadBundle = &bundle.Bundle{} + mockedBundleStore.ReadBundle = &relocated.Bundle{} // and given a bad destination reference const badRef = "b@d reference" @@ -100,7 +100,7 @@ func TestInvalidDestinationReference(t *testing.T) { func TestBundleNotStored(t *testing.T) { // given a reference and a bundle is returned by bundleStore.LookUp and bundleStore.Read mockedBundleStore.LookUpRef = parseRefOrDie(t, "src-app") - mockedBundleStore.ReadBundle = &bundle.Bundle{} + mockedBundleStore.ReadBundle = &relocated.Bundle{} // and given bundleStore.Store will return an error mockedBundleStore.StoredError = fmt.Errorf("error from bundleStore.Store") @@ -112,7 +112,7 @@ func TestBundleNotStored(t *testing.T) { func TestSuccessfulyTag(t *testing.T) { // given a reference and a bundle is returned by bundleStore.LookUp and bundleStore.Read mockedBundleStore.LookUpRef = parseRefOrDie(t, "src-app") - mockedBundleStore.ReadBundle = &bundle.Bundle{} + mockedBundleStore.ReadBundle = &relocated.Bundle{} // and given valid source and output references const ( srcRef = "src-app" diff --git a/internal/commands/push.go b/internal/commands/push.go index e579e060c..2017b2d21 100644 --- a/internal/commands/push.go +++ b/internal/commands/push.go @@ -8,8 +8,10 @@ import ( "os" "strings" + "github.com/docker/app/internal/relocated" + "github.com/docker/app/internal/store" + "github.com/containerd/containerd/platforms" - "github.com/deislabs/cnab-go/bundle" "github.com/docker/app/internal" "github.com/docker/app/internal/cnab" "github.com/docker/app/internal/log" @@ -51,9 +53,14 @@ func pushCmd(dockerCli command.Cli) *cobra.Command { } func runPush(dockerCli command.Cli, name string) error { + bundleStore, err := prepareBundleStore() + if err != nil { + return err + } + defer muteDockerCli(dockerCli)() // Get the bundle - bndl, ref, err := resolveReferenceAndBundle(dockerCli, name) + bndl, ref, err := resolveReferenceAndBundle(dockerCli, bundleStore, name) if err != nil { return err } @@ -65,15 +72,16 @@ func runPush(dockerCli command.Cli, name string) error { cnabRef = reference.TagNameOnly(cnabRef) // Push the bundle - return pushBundle(dockerCli, bndl, cnabRef) -} - -func resolveReferenceAndBundle(dockerCli command.Cli, name string) (*bundle.Bundle, string, error) { - bundleStore, err := prepareBundleStore() - if err != nil { - return nil, "", err + if err := pushBundle(dockerCli, bndl, cnabRef); err != nil { + return err } + // Update the bundle to store the generated relocation map + _, err = bundleStore.Store(cnabRef, bndl) + return err +} + +func resolveReferenceAndBundle(dockerCli command.Cli, bundleStore store.BundleStore, name string) (*relocated.Bundle, string, error) { bndl, ref, err := cnab.ResolveBundle(dockerCli, bundleStore, name) if err != nil { return nil, "", err @@ -84,7 +92,7 @@ func resolveReferenceAndBundle(dockerCli command.Cli, name string) (*bundle.Bund return bndl, ref, err } -func pushBundle(dockerCli command.Cli, bndl *bundle.Bundle, cnabRef reference.Named) error { +func pushBundle(dockerCli command.Cli, bndl *relocated.Bundle, cnabRef reference.Named) error { insecureRegistries, err := internal.InsecureRegistriesFromEngine(dockerCli) if err != nil { return errors.Wrap(err, "could not retrieve insecure registries") @@ -100,13 +108,14 @@ func pushBundle(dockerCli command.Cli, bndl *bundle.Bundle, cnabRef reference.Na remotes.WithPushImages(dockerCli.Client(), dockerCli.Out()), } // bundle fixup - relocationMap, err := remotes.FixupBundle(context.Background(), bndl, cnabRef, resolver, fixupOptions...) + relocationMap, err := remotes.FixupBundle(context.Background(), bndl.Bundle, cnabRef, resolver, fixupOptions...) if err != nil { return errors.Wrapf(err, "fixing up %q for push", cnabRef) } + bndl.RelocationMap = relocationMap // push bundle manifest logrus.Debugf("Pushing the bundle %q", cnabRef) - descriptor, err := remotes.Push(log.WithLogContext(context.Background()), bndl, relocationMap, cnabRef, resolver, true, withAppAnnotations) + descriptor, err := remotes.Push(log.WithLogContext(context.Background()), bndl.Bundle, relocationMap, cnabRef, resolver, true, withAppAnnotations) if err != nil { return errors.Wrapf(err, "pushing to %q", cnabRef) } diff --git a/internal/commands/render.go b/internal/commands/render.go index 4138eb5d2..d547e2bed 100644 --- a/internal/commands/render.go +++ b/internal/commands/render.go @@ -84,7 +84,7 @@ func prepareCustomAction(actionName string, dockerCli command.Cli, appname strin if err != nil { return nil, nil, nil, err } - installation.Bundle = bundle + installation.Bundle = bundle.Bundle if err := mergeBundleParameters(installation, withFileParameters(paramsOpts.parametersFiles), diff --git a/internal/commands/run.go b/internal/commands/run.go index 5b7b734d9..25712e0f9 100644 --- a/internal/commands/run.go +++ b/internal/commands/run.go @@ -4,12 +4,14 @@ import ( "fmt" "os" + "github.com/docker/app/internal/fetch" + "github.com/docker/app/internal/relocated" + "github.com/deislabs/cnab-go/driver" "github.com/docker/cli/cli" "github.com/deislabs/cnab-go/action" - "github.com/deislabs/cnab-go/bundle" "github.com/deislabs/cnab-go/credentials" "github.com/docker/app/internal/cnab" "github.com/docker/app/internal/store" @@ -72,7 +74,7 @@ func runCmd(dockerCli command.Cli) *cobra.Command { } func runCnab(dockerCli command.Cli, opts runOptions) error { - bndl, err := cnab.LoadBundleFromFile(opts.cnabBundle) + bndl, err := fetch.RelocatedBundleFromFile(opts.cnabBundle) if err != nil { return errors.Wrapf(err, "failed to read bundle %q", opts.cnabBundle) } @@ -92,7 +94,7 @@ func runDockerApp(dockerCli command.Cli, appname string, opts runOptions) error return runBundle(dockerCli, bndl, opts, ref.String()) } -func runBundle(dockerCli command.Cli, bndl *bundle.Bundle, opts runOptions, ref string) error { +func runBundle(dockerCli command.Cli, bndl *relocated.Bundle, opts runOptions, ref string) error { opts.SetDefaultTargetContext(dockerCli) bind, err := cnab.RequiredBindMount(opts.targetContext, opts.orchestrator, dockerCli.ContextStore()) @@ -129,7 +131,7 @@ func runBundle(dockerCli command.Cli, bndl *bundle.Bundle, opts runOptions, ref } driverImpl, errBuf := cnab.PrepareDriver(dockerCli, bind, nil) - installation.Bundle = bndl + installation.Bundle = bndl.Bundle if err := mergeBundleParameters(installation, withFileParameters(opts.parametersFiles), @@ -140,7 +142,7 @@ func runBundle(dockerCli command.Cli, bndl *bundle.Bundle, opts runOptions, ref ); err != nil { return err } - creds, err := prepareCredentialSet(bndl, opts.CredentialSetOpts(dockerCli, credentialStore)...) + creds, err := prepareCredentialSet(bndl.Bundle, opts.CredentialSetOpts(dockerCli, credentialStore)...) if err != nil { return err } diff --git a/internal/commands/update.go b/internal/commands/update.go index 1dc26583b..d6ef3b7e3 100644 --- a/internal/commands/update.go +++ b/internal/commands/update.go @@ -60,7 +60,7 @@ func runUpdate(dockerCli command.Cli, installationName string, opts updateOption if err != nil { return err } - installation.Bundle = b + installation.Bundle = b.Bundle } if err := mergeBundleParameters(installation, withFileParameters(opts.parametersFiles), diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go new file mode 100644 index 000000000..1fe53d1c4 --- /dev/null +++ b/internal/fetch/fetch.go @@ -0,0 +1,60 @@ +package fetch + +import ( + "io/ioutil" + "os" + "path/filepath" + + "github.com/deislabs/cnab-go/bundle" + "github.com/docker/app/internal/relocated" + "github.com/docker/cnab-to-oci/relocation" + "github.com/docker/go/canonical/json" + "github.com/pkg/errors" +) + +func BundleJSON(bundlePath string) (*bundle.Bundle, error) { + data, err := ioutil.ReadFile(bundlePath) + if err != nil { + return nil, errors.Wrapf(err, "failed to read file %s", bundlePath) + } + bndl, err := bundle.Unmarshal(data) + if err != nil { + return nil, errors.Wrapf(err, "failed to read file %s", bundlePath) + } + return bndl, nil +} + +func RelocationMapJSON(relocationMapPath string) (relocation.ImageRelocationMap, error) { + relocationMap := relocation.ImageRelocationMap{} + _, err := os.Stat(relocationMapPath) + if os.IsNotExist(err) { + return relocationMap, nil + } + data, err := ioutil.ReadFile(relocationMapPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to read file %s", relocationMapPath) + } + if err := json.Unmarshal(data, &relocationMap); err != nil { + return nil, errors.Wrapf(err, "failed to read file %s", relocationMapPath) + } + return relocationMap, nil +} + +// FetchRelocatedBundleFromFile creates a relocated bundle based on the bundle file and relocation map. +func RelocatedBundleFromFile(filename string) (*relocated.Bundle, error) { + bndl, err := BundleJSON(filename) + if err != nil { + return nil, errors.Wrapf(err, "failed to read bundle") + } + + relocationMapFileName := filepath.Join(filepath.Dir(filename), "relocation-map.json") + relocationMap, err := RelocationMapJSON(relocationMapFileName) + if err != nil { + return nil, errors.Wrapf(err, "failed to read relocation map") + } + + return &relocated.Bundle{ + Bundle: bndl, + RelocationMap: relocationMap, + }, nil +} diff --git a/internal/packager/bundle.go b/internal/packager/bundle.go index d482b8fdc..95a410ac6 100644 --- a/internal/packager/bundle.go +++ b/internal/packager/bundle.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "github.com/docker/app/internal/relocated" "io/ioutil" "github.com/deislabs/cnab-go/bundle" @@ -71,7 +72,7 @@ func MakeCNABImageName(appName, appVersion, suffix string) (string, error) { } // PersistInBundleStore do store a bundle with optional reference and return it's ID -func PersistInBundleStore(ref reference.Reference, bndle *bundle.Bundle) (reference.Digested, error) { +func PersistInBundleStore(ref reference.Reference, bndl *bundle.Bundle) (reference.Digested, error) { appstore, err := store.NewApplicationStore(config.Dir()) if err != nil { return nil, err @@ -80,7 +81,7 @@ func PersistInBundleStore(ref reference.Reference, bndle *bundle.Bundle) (refere if err != nil { return nil, err } - return bundleStore.Store(ref, bndle) + return bundleStore.Store(ref, relocated.FromBundle(bndl)) } func GetNamedTagged(tag string) (reference.NamedTagged, error) { diff --git a/internal/relocated/relocated_bundle.go b/internal/relocated/relocated_bundle.go new file mode 100644 index 000000000..368cfae34 --- /dev/null +++ b/internal/relocated/relocated_bundle.go @@ -0,0 +1,32 @@ +package relocated + +import ( + "io/ioutil" + "os" + + "github.com/deislabs/cnab-go/bundle" + "github.com/docker/cnab-to-oci/relocation" + "github.com/docker/go/canonical/json" +) + +type Bundle struct { + *bundle.Bundle + RelocationMap relocation.ImageRelocationMap +} + +// FromBundle returns a RelocatedBundle with an empty relocation map. +func FromBundle(bndl *bundle.Bundle) *Bundle { + return &Bundle{ + Bundle: bndl, + RelocationMap: relocation.ImageRelocationMap{}, + } +} + +// WriteRelocationMap serializes the relocation map and writes it to a file as JSON. +func (b *Bundle) WriteRelocationMap(dest string, mode os.FileMode) error { + d, err := json.MarshalCanonical(b.RelocationMap) + if err != nil { + return err + } + return ioutil.WriteFile(dest, d, mode) +} diff --git a/internal/store/bundle.go b/internal/store/bundle.go index 419fd66c6..695781dde 100644 --- a/internal/store/bundle.go +++ b/internal/store/bundle.go @@ -1,15 +1,14 @@ package store import ( - "encoding/json" "fmt" - "io/ioutil" + "github.com/docker/app/internal/fetch" + "github.com/docker/app/internal/relocated" "os" "path/filepath" "sort" "strings" - "github.com/deislabs/cnab-go/bundle" "github.com/docker/distribution/reference" digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" @@ -18,8 +17,8 @@ import ( // type BundleStore interface { // Store do store the bundle with optional reference, and return it's unique ID - Store(ref reference.Reference, bndle *bundle.Bundle) (reference.Digested, error) - Read(ref reference.Reference) (*bundle.Bundle, error) + Store(ref reference.Reference, bndl *relocated.Bundle) (reference.Digested, error) + Read(ref reference.Reference) (*relocated.Bundle, error) List() ([]reference.Reference, error) Remove(ref reference.Reference) error LookUp(refOrID string) (reference.Reference, error) @@ -63,8 +62,8 @@ func NewBundleStore(path string) (BundleStore, error) { // \_ bundle.json // -func (b *bundleStore) Store(ref reference.Reference, bndle *bundle.Bundle) (reference.Digested, error) { - id, err := FromBundle(bndle) +func (b *bundleStore) Store(ref reference.Reference, bndl *relocated.Bundle) (reference.Digested, error) { + id, err := FromBundle(bndl) if err != nil { return nil, errors.Wrapf(err, "failed to store bundle %q", ref) } @@ -75,28 +74,33 @@ func (b *bundleStore) Store(ref reference.Reference, bndle *bundle.Bundle) (refe if err != nil { return id, errors.Wrapf(err, "failed to store bundle %q", ref) } - path := filepath.Join(dir, "bundle.json") if err := os.MkdirAll(dir, 0755); err != nil { return id, errors.Wrapf(err, "failed to store bundle %q", ref) } - if err = bndle.WriteFile(path, 0644); err != nil { + + // store bundle.json + path := filepath.Join(dir, "bundle.json") + if err = bndl.WriteFile(path, 0644); err != nil { return id, errors.Wrapf(err, "failed to store bundle %q", ref) } + + // store relocation map + relocationMapPath := filepath.Join(dir, "relocation-map.json") + if err = bndl.WriteRelocationMap(relocationMapPath, 0644); err != nil { + return id, errors.Wrapf(err, "failed to store relocation map for bundle %q", ref) + } + b.refsMap.appendRef(id, ref) return id, nil } -func (b *bundleStore) Read(ref reference.Reference) (*bundle.Bundle, error) { +func (b *bundleStore) Read(ref reference.Reference) (*relocated.Bundle, error) { paths, err := b.storePaths(ref) if err != nil { return nil, errors.Wrapf(err, "failed to read bundle %q", ref) } - bndl, err := b.fetchBundleJSON(filepath.Join(paths[0], "bundle.json")) - if err != nil { - return nil, errors.Wrapf(err, "failed to read bundle %q", ref) - } - return bndl, nil + return fetch.RelocatedBundleFromFile(filepath.Join(paths[0], "bundle.json")) } // Returns the list of all bundles present in the bundle store @@ -256,6 +260,10 @@ func (b *bundleStore) processBundleStoreFile(path string, info os.FileInfo, err return nil } + if info.Name() == "relocation-map.json" { + return nil + } + if !strings.HasSuffix(info.Name(), ".json") { return nil } @@ -272,11 +280,12 @@ func (b *bundleStore) processBundleStoreFile(path string, info os.FileInfo, err if err != nil { return err } - bndl, err := b.fetchBundleJSON(path) + bndl, err := fetch.BundleJSON(path) if err != nil { return err } - id, err := FromBundle(bndl) + // we don't care about relocation map here, we just care about finding bundles + id, err := FromBundle(relocated.FromBundle(bndl)) if err != nil { return err } @@ -285,18 +294,6 @@ func (b *bundleStore) processBundleStoreFile(path string, info os.FileInfo, err return nil } -func (b *bundleStore) fetchBundleJSON(bundlePath string) (*bundle.Bundle, error) { - data, err := ioutil.ReadFile(bundlePath) - if err != nil { - return nil, errors.Wrapf(err, "failed to read file %s", bundlePath) - } - var bndl bundle.Bundle - if err := json.Unmarshal(data, &bndl); err != nil { - return nil, errors.Wrapf(err, "failed to read file %s", bundlePath) - } - return &bndl, nil -} - func (b *bundleStore) pathToReference(path string) (reference.Named, error) { // Clean the path and remove the local bundle store path cleanpath := filepath.ToSlash(path) diff --git a/internal/store/bundle_test.go b/internal/store/bundle_test.go index 9b560cc37..80bc11faa 100644 --- a/internal/store/bundle_test.go +++ b/internal/store/bundle_test.go @@ -2,6 +2,7 @@ package store import ( "fmt" + "github.com/docker/app/internal/relocated" "io/ioutil" "os" "path" @@ -26,7 +27,7 @@ func TestStoreAndReadBundle(t *testing.T) { bundleStore, err := appstore.BundleStore() assert.NilError(t, err) - expectedBundle := &bundle.Bundle{Name: "bundle-name"} + expectedBundle := relocated.FromBundle(&bundle.Bundle{Name: "bundle-name"}) testcases := []struct { name string @@ -222,7 +223,7 @@ func TestList(t *testing.T) { assert.Equal(t, len(bundles), 0) }) - bndl := &bundle.Bundle{Name: "bundle-name"} + bndl := relocated.FromBundle(&bundle.Bundle{Name: "bundle-name"}) for _, ref := range refs { _, err = bundleStore.Store(ref, bndl) assert.NilError(t, err) @@ -260,7 +261,7 @@ func TestRemove(t *testing.T) { parseRefOrDie(t, "my-repo/b-bundle@sha256:"+testSha), } - bndl := &bundle.Bundle{Name: "bundle-name"} + bndl := relocated.FromBundle(&bundle.Bundle{Name: "bundle-name"}) for _, ref := range refs { _, err = bundleStore.Store(ref, bndl) assert.NilError(t, err) @@ -300,7 +301,7 @@ func TestLookUp(t *testing.T) { assert.NilError(t, err) bundleStore, err := appstore.BundleStore() assert.NilError(t, err) - bndl := &bundle.Bundle{Name: "bundle-name"} + bndl := relocated.FromBundle(&bundle.Bundle{Name: "bundle-name"}) // Adding the bundle referenced by id id, err := bundleStore.Store(nil, bndl) assert.NilError(t, err) @@ -384,7 +385,7 @@ func TestScanBundles(t *testing.T) { defer dockerConfigDir.Remove() // Adding a bundle which should be referenced by id only - bndl1 := &bundle.Bundle{Name: "bundle-1"} + bndl1 := relocated.FromBundle(&bundle.Bundle{Name: "bundle-1"}) id1, err := FromBundle(bndl1) assert.NilError(t, err) dir1 := dockerConfigDir.Join("app", "bundles", "_ids", id1.String()) @@ -392,7 +393,7 @@ func TestScanBundles(t *testing.T) { assert.NilError(t, ioutil.WriteFile(filepath.Join(dir1, "bundle.json"), []byte(`{"name": "bundle-1"}`), 0644)) // Adding a bundle which should be referenced by id and tag - bndl2 := &bundle.Bundle{Name: "bundle-2"} + bndl2 := relocated.FromBundle(&bundle.Bundle{Name: "bundle-2"}) id2, err := FromBundle(bndl2) assert.NilError(t, err) dir2 := dockerConfigDir.Join("app", "bundles", "_ids", id2.String()) diff --git a/internal/store/digest.go b/internal/store/digest.go index 71640b719..0c6b6f83c 100644 --- a/internal/store/digest.go +++ b/internal/store/digest.go @@ -3,10 +3,10 @@ package store import ( "bytes" "fmt" + "github.com/docker/app/internal/relocated" "io" "regexp" - "github.com/deislabs/cnab-go/bundle" "github.com/docker/distribution/reference" "github.com/opencontainers/go-digest" "github.com/pkg/errors" @@ -49,8 +49,8 @@ func FromString(s string) (ID, error) { return ID{digest}, nil } -func FromBundle(bndle *bundle.Bundle) (ID, error) { - digest, err := ComputeDigest(bndle) +func FromBundle(bndl *relocated.Bundle) (ID, error) { + digest, err := ComputeDigest(bndl) return ID{digest}, err } diff --git a/internal/store/digest_test.go b/internal/store/digest_test.go index 7d15ba1cd..da6223910 100644 --- a/internal/store/digest_test.go +++ b/internal/store/digest_test.go @@ -1,6 +1,7 @@ package store import ( + "github.com/docker/app/internal/relocated" "io/ioutil" "os" "testing" @@ -18,7 +19,7 @@ func Test_storeByDigest(t *testing.T) { bundleStore, err := appstore.BundleStore() assert.NilError(t, err) - bndl := &bundle.Bundle{Name: "bundle-name"} + bndl := relocated.FromBundle(&bundle.Bundle{Name: "bundle-name"}) ref := parseRefOrDie(t, "test/simple:1.0") _, err = bundleStore.Store(ref, bndl) assert.NilError(t, err)