From ea06786a63dfe903e679a837a3eb3fa7c63a2013 Mon Sep 17 00:00:00 2001 From: Silvin Lubecki Date: Mon, 12 Aug 2019 18:15:04 +0200 Subject: [PATCH] Introduce relocation map generation - Fixup does not mutate the bundle anymore, unless we want it too using auto-update-bundle, then it will update the digest, size and media type sections - Fixup now generates a relocation map, with original service/invocation images as keys and the digested references of the pushed images as values - Pull now generates the relocation map too. The result of pulling a bundle is a bundle.json and a relocation map. - Bundle is now stored as is in the registry, it is no more re-generated from the OCI index. Bundle is stored using the canonical form. - Split --output flag for fixup and pull commands into 2 flags: --bundle and --relocation-map - Harden e2e test, checking fixed bundle and generated relocation map Signed-off-by: Radu M Signed-off-by: Silvin Lubecki --- Gopkg.lock | 1 + cmd/cnab-to-oci/fixup.go | 27 ++- cmd/cnab-to-oci/pull.go | 24 +- cmd/cnab-to-oci/push.go | 15 +- converter/convert.go | 215 ++++++++-------- converter/convert_test.go | 104 +++----- converter/types.go | 33 +-- converter/types_test.go | 5 +- e2e/e2e_test.go | 101 ++++++-- e2e/testdata/bundle.json.golden.template | 1 + e2e/testdata/hello-world/bundle.json.template | 54 +---- examples/helloworld-cnab/bundle.json | 3 +- relocation/types.go | 4 + remotes/fixup.go | 84 ++++--- remotes/fixup_test.go | 229 +++++++++++++++++- remotes/fixuphelpers.go | 6 +- remotes/fixupoptions.go | 9 + remotes/pull.go | 47 ++-- remotes/pull_test.go | 69 ++---- remotes/push.go | 53 ++-- remotes/push_test.go | 86 ++----- tests/helpers.go | 23 +- 22 files changed, 680 insertions(+), 513 deletions(-) create mode 100644 e2e/testdata/bundle.json.golden.template create mode 100644 relocation/types.go diff --git a/Gopkg.lock b/Gopkg.lock index f66acb1..9c55c3b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -470,6 +470,7 @@ "github.com/docker/distribution/reference", "github.com/docker/distribution/registry/client/auth", "github.com/docker/docker/registry", + "github.com/docker/go/canonical/json", "github.com/opencontainers/go-digest", "github.com/opencontainers/image-spec/specs-go", "github.com/opencontainers/image-spec/specs-go/v1", diff --git a/cmd/cnab-to-oci/fixup.go b/cmd/cnab-to-oci/fixup.go index 111cd93..3f042d2 100644 --- a/cmd/cnab-to-oci/fixup.go +++ b/cmd/cnab-to-oci/fixup.go @@ -2,7 +2,6 @@ package main import ( "context" - "encoding/json" "fmt" "io/ioutil" "os" @@ -12,14 +11,17 @@ import ( "github.com/docker/cli/cli/config" "github.com/docker/cnab-to-oci/remotes" "github.com/docker/distribution/reference" + "github.com/docker/go/canonical/json" "github.com/spf13/cobra" ) type fixupOptions struct { input string - output string + bundle string + relocationMap string targetRef string insecureRegistries []string + autoUpdateBundle bool } func fixupCmd() *cobra.Command { @@ -34,9 +36,11 @@ func fixupCmd() *cobra.Command { return runFixup(opts) }, } - cmd.Flags().StringVarP(&opts.output, "output", "o", "fixed-bundle.json", "specify the output file") + cmd.Flags().StringVar(&opts.bundle, "bundle", "fixed-bundle.json", "fixed bundle output file (- to print on standard output)") + cmd.Flags().StringVar(&opts.relocationMap, "relocation-map", "relocation-map.json", "relocation map output file (- to print on standard output)") cmd.Flags().StringVarP(&opts.targetRef, "target", "t", "", "reference where the bundle will be pushed") cmd.Flags().StringSliceVar(&opts.insecureRegistries, "insecure-registries", nil, "Use plain HTTP for those registries") + cmd.Flags().BoolVar(&opts.autoUpdateBundle, "auto-update-bundle", false, "Updates the bundle image properties with the one resolved on the registry") return cmd } @@ -53,18 +57,21 @@ func runFixup(opts fixupOptions) error { if err != nil { return err } - if err := remotes.FixupBundle(context.Background(), &b, ref, createResolver(opts.insecureRegistries), remotes.WithEventCallback(displayEvent)); err != nil { - return err + + fixupOptions := []remotes.FixupOption{ + remotes.WithEventCallback(displayEvent), } - bundleJSON, err = json.MarshalIndent(b, "", "\t") + if opts.autoUpdateBundle { + fixupOptions = append(fixupOptions, remotes.WithAutoBundleUpdate()) + } + relocationMap, err := remotes.FixupBundle(context.Background(), &b, ref, createResolver(opts.insecureRegistries), fixupOptions...) if err != nil { return err } - if opts.output == "-" { - fmt.Fprintln(os.Stdout, string(bundleJSON)) - return nil + if err := writeOutput(opts.bundle, b); err != nil { + return err } - return ioutil.WriteFile(opts.output, bundleJSON, 0644) + return writeOutput(opts.relocationMap, relocationMap) } func displayEvent(ev remotes.FixupEvent) { diff --git a/cmd/cnab-to-oci/pull.go b/cmd/cnab-to-oci/pull.go index ad107a2..a641c39 100644 --- a/cmd/cnab-to-oci/pull.go +++ b/cmd/cnab-to-oci/pull.go @@ -2,18 +2,19 @@ package main import ( "context" - "encoding/json" "fmt" "io/ioutil" "os" "github.com/docker/cnab-to-oci/remotes" "github.com/docker/distribution/reference" + "github.com/docker/go/canonical/json" "github.com/spf13/cobra" ) type pullOptions struct { - output string + bundle string + relocationMap string targetRef string insecureRegistries []string } @@ -30,7 +31,8 @@ func pullCmd() *cobra.Command { }, } - cmd.Flags().StringVarP(&opts.output, "output", "o", "pulled.json", "output file") + cmd.Flags().StringVar(&opts.bundle, "bundle", "pulled.json", "bundle output file (- to print on standard output)") + cmd.Flags().StringVar(&opts.relocationMap, "relocation-map", "relocation-map.json", "relocation map output file (- to print on standard output)") cmd.Flags().StringSliceVar(&opts.insecureRegistries, "insecure-registries", nil, "Use plain HTTP for those registries") return cmd } @@ -40,17 +42,25 @@ func runPull(opts pullOptions) error { if err != nil { return err } - b, err := remotes.Pull(context.Background(), ref, createResolver(opts.insecureRegistries)) + + b, relocationMap, err := remotes.Pull(context.Background(), ref, createResolver(opts.insecureRegistries)) if err != nil { return err } - bytes, err := json.MarshalIndent(b, "", "\t") + if err := writeOutput(opts.bundle, b); err != nil { + return err + } + return writeOutput(opts.relocationMap, relocationMap) +} + +func writeOutput(file string, data interface{}) error { + bytes, err := json.MarshalCanonical(data) if err != nil { return err } - if opts.output == "-" { + if file == "-" { fmt.Fprintln(os.Stdout, string(bytes)) return nil } - return ioutil.WriteFile(opts.output, bytes, 0644) + return ioutil.WriteFile(file, bytes, 0644) } diff --git a/cmd/cnab-to-oci/push.go b/cmd/cnab-to-oci/push.go index 9784cab..ee72507 100644 --- a/cmd/cnab-to-oci/push.go +++ b/cmd/cnab-to-oci/push.go @@ -20,6 +20,7 @@ type pushOptions struct { allowFallbacks bool invocationPlatforms []string componentPlatforms []string + autoUpdateBundle bool } func pushCmd() *cobra.Command { @@ -42,6 +43,8 @@ func pushCmd() *cobra.Command { cmd.Flags().BoolVar(&opts.allowFallbacks, "allow-fallbacks", true, "Enable automatic compatibility fallbacks for registries without support for custom media type, or OCI manifests") cmd.Flags().StringSliceVar(&opts.invocationPlatforms, "invocation-platforms", nil, "Platforms to push (for multi-arch invocation images)") cmd.Flags().StringSliceVar(&opts.componentPlatforms, "component-platforms", nil, "Platforms to push (for multi-arch component images)") + cmd.Flags().BoolVar(&opts.autoUpdateBundle, "auto-update-bundle", false, "Updates the bundle image properties with the one resolved on the registry") + return cmd } @@ -60,13 +63,19 @@ func runPush(opts pushOptions) error { return err } - err = remotes.FixupBundle(context.Background(), &b, ref, resolver, remotes.WithEventCallback(displayEvent), + fixupOptions := []remotes.FixupOption{ + remotes.WithEventCallback(displayEvent), remotes.WithInvocationImagePlatforms(opts.invocationPlatforms), - remotes.WithComponentImagePlatforms(opts.componentPlatforms)) + remotes.WithComponentImagePlatforms(opts.componentPlatforms), + } + if opts.autoUpdateBundle { + fixupOptions = append(fixupOptions, remotes.WithAutoBundleUpdate()) + } + relocationMap, err := remotes.FixupBundle(context.Background(), &b, ref, resolver, fixupOptions...) if err != nil { return err } - d, err := remotes.Push(context.Background(), &b, ref, resolver, opts.allowFallbacks) + d, err := remotes.Push(context.Background(), &b, relocationMap, ref, resolver, opts.allowFallbacks) if err != nil { return err } diff --git a/converter/convert.go b/converter/convert.go index 59dc017..35a5f83 100644 --- a/converter/convert.go +++ b/converter/convert.go @@ -9,6 +9,7 @@ import ( "github.com/containerd/containerd/images" "github.com/deislabs/cnab-go/bundle" + "github.com/docker/cnab-to-oci/relocation" "github.com/docker/distribution/reference" ocischema "github.com/opencontainers/image-spec/specs-go" ocischemav1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -60,12 +61,13 @@ func GetBundleConfigManifestDescriptor(ix *ocischemav1.Index) (ocischemav1.Descr } // ConvertBundleToOCIIndex converts a CNAB bundle into an OCI Index representation -func ConvertBundleToOCIIndex(b *bundle.Bundle, targetRef reference.Named, bundleConfigManifestRef ocischemav1.Descriptor) (*ocischemav1.Index, error) { +func ConvertBundleToOCIIndex(b *bundle.Bundle, targetRef reference.Named, + bundleConfigManifestRef ocischemav1.Descriptor, relocationMap relocation.ImageRelocationMap) (*ocischemav1.Index, error) { annotations, err := makeAnnotations(b) if err != nil { return nil, err } - manifests, err := makeManifests(b, targetRef, bundleConfigManifestRef) + manifests, err := makeManifests(b, targetRef, bundleConfigManifestRef, relocationMap) if err != nil { return nil, err } @@ -79,24 +81,59 @@ func ConvertBundleToOCIIndex(b *bundle.Bundle, targetRef reference.Named, bundle return &result, nil } -// ConvertOCIIndexToBundle converts an OCI index to a CNAB bundle representation -func ConvertOCIIndexToBundle(ix *ocischemav1.Index, config *BundleConfig, originRepo reference.Named) (*bundle.Bundle, error) { - b := &bundle.Bundle{ - SchemaVersion: CNABVersion, - Actions: config.Actions, - Credentials: config.Credentials, - Definitions: config.Definitions, - Parameters: config.Parameters, - Outputs: config.Outputs, - Custom: config.Custom, - } - if err := parseTopLevelAnnotations(ix.Annotations, b); err != nil { - return nil, err - } - if err := parseManifests(ix.Manifests, b, originRepo); err != nil { - return nil, err +// GenerateRelocationMap generates the bundle relocation map +func GenerateRelocationMap(ix *ocischemav1.Index, b *bundle.Bundle, originRepo reference.Named) (relocation.ImageRelocationMap, error) { + relocationMap := relocation.ImageRelocationMap{} + + for _, d := range ix.Manifests { + switch d.MediaType { + case ocischemav1.MediaTypeImageManifest, ocischemav1.MediaTypeImageIndex: + case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList: + default: + return nil, fmt.Errorf("unsupported manifest descriptor %q with mediatype %q", d.Digest, d.MediaType) + } + descriptorType, ok := d.Annotations[CNABDescriptorTypeAnnotation] + if !ok { + return nil, fmt.Errorf("manifest descriptor %q has no CNAB descriptor type annotation %q", d.Digest, CNABDescriptorTypeAnnotation) + } + if descriptorType == CNABDescriptorTypeConfig { + continue + } + // strip tag/digest from originRepo + originRepo, err := reference.ParseNormalizedNamed(originRepo.Name()) + if err != nil { + return nil, fmt.Errorf("failed to create a digested reference for manifest descriptor %q: %s", d.Digest, err) + } + ref, err := reference.WithDigest(originRepo, d.Digest) + if err != nil { + return nil, fmt.Errorf("failed to create a digested reference for manifest descriptor %q: %s", d.Digest, err) + } + refFamiliar := reference.FamiliarString(ref) + switch descriptorType { + // The current descriptor is an invocation image + case CNABDescriptorTypeInvocation: + if len(b.InvocationImages) == 0 { + return nil, fmt.Errorf("unknown invocation image: %q", d.Digest) + } + relocationMap[b.InvocationImages[0].Image] = refFamiliar + + // The current descriptor is a component image + case CNABDescriptorTypeComponent: + componentName, ok := d.Annotations[CNABDescriptorComponentNameAnnotation] + if !ok { + return nil, fmt.Errorf("component name missing in descriptor %q", d.Digest) + } + c, ok := b.Images[componentName] + if !ok { + return nil, fmt.Errorf("component %q not found in bundle", componentName) + } + relocationMap[c.Image] = refFamiliar + default: + return nil, fmt.Errorf("invalid CNAB descriptor type %q in descriptor %q", descriptorType, d.Digest) + } } - return b, nil + + return relocationMap, nil } func makeAnnotations(b *bundle.Bundle) (map[string]string, error) { @@ -124,29 +161,8 @@ func makeAnnotations(b *bundle.Bundle) (map[string]string, error) { return result, nil } -func parseTopLevelAnnotations(annotations map[string]string, into *bundle.Bundle) error { - var ok bool - if into.Name, ok = annotations[ocischemav1.AnnotationTitle]; !ok { - return errors.New("manifest is missing title annotation " + ocischemav1.AnnotationTitle) - } - if into.Version, ok = annotations[ocischemav1.AnnotationVersion]; !ok { - return errors.New("manifest is missing version annotation " + ocischemav1.AnnotationVersion) - } - into.Description = annotations[ocischemav1.AnnotationDescription] - if maintainersJSON, ok := annotations[ocischemav1.AnnotationAuthors]; ok { - if err := json.Unmarshal([]byte(maintainersJSON), &into.Maintainers); err != nil { - return fmt.Errorf("unable to parse maintainers: %s", err) - } - } - if keywordsJSON, ok := annotations[CNABKeywordsAnnotation]; ok { - if err := json.Unmarshal([]byte(keywordsJSON), &into.Keywords); err != nil { - return fmt.Errorf("unable to parse keywords: %s", err) - } - } - return nil -} - -func makeManifests(b *bundle.Bundle, targetReference reference.Named, bundleConfigManifestReference ocischemav1.Descriptor) ([]ocischemav1.Descriptor, error) { +func makeManifests(b *bundle.Bundle, targetReference reference.Named, + bundleConfigManifestReference ocischemav1.Descriptor, relocationMap relocation.ImageRelocationMap) ([]ocischemav1.Descriptor, error) { if len(b.InvocationImages) != 1 { return nil, errors.New("only one invocation image supported") } @@ -155,7 +171,7 @@ func makeManifests(b *bundle.Bundle, targetReference reference.Named, bundleConf } bundleConfigManifestReference.Annotations[CNABDescriptorTypeAnnotation] = CNABDescriptorTypeConfig manifests := []ocischemav1.Descriptor{bundleConfigManifestReference} - invocationImage, err := makeDescriptor(b.InvocationImages[0].BaseImage, targetReference) + invocationImage, err := makeDescriptor(b.InvocationImages[0].BaseImage, targetReference, relocationMap) if err != nil { return nil, fmt.Errorf("invalid invocation image: %s", err) } @@ -166,7 +182,7 @@ func makeManifests(b *bundle.Bundle, targetReference reference.Named, bundleConf images := makeSortedImages(b.Images) for _, name := range images { img := b.Images[name] - image, err := makeDescriptor(img.BaseImage, targetReference) + image, err := makeDescriptor(img.BaseImage, targetReference, relocationMap) if err != nil { return nil, fmt.Errorf("invalid image: %s", err) } @@ -188,92 +204,57 @@ func makeSortedImages(images map[string]bundle.Image) []string { return result } -func parseManifests(descriptors []ocischemav1.Descriptor, into *bundle.Bundle, originRepo reference.Named) error { - for _, d := range descriptors { - var imageType string - switch d.MediaType { - case ocischemav1.MediaTypeImageManifest, ocischemav1.MediaTypeImageIndex: - imageType = "oci" - case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList: - imageType = "docker" - default: - return fmt.Errorf("unsupported manifest descriptor %q with mediatype %q", d.Digest, d.MediaType) - } - descriptorType, ok := d.Annotations[CNABDescriptorTypeAnnotation] - if !ok { - return fmt.Errorf("manifest descriptor %q has no CNAB descriptor type annotation %q", d.Digest, CNABDescriptorTypeAnnotation) - } - if descriptorType == CNABDescriptorTypeConfig { - continue - } - // strip tag/digest from originRepo - originRepo, err := reference.ParseNormalizedNamed(originRepo.Name()) - if err != nil { - return fmt.Errorf("failed to create a digested reference for manifest descriptor %q: %s", d.Digest, err) - } - ref, err := reference.WithDigest(originRepo, d.Digest) - if err != nil { - return fmt.Errorf("failed to create a digested reference for manifest descriptor %q: %s", d.Digest, err) - } - refFamiliar := reference.FamiliarString(ref) - switch descriptorType { - // The current descriptor is an invocation image - case CNABDescriptorTypeInvocation: - into.InvocationImages = append(into.InvocationImages, bundle.InvocationImage{ - BaseImage: bundle.BaseImage{ - Image: refFamiliar, - ImageType: imageType, - MediaType: d.MediaType, - Size: uint64(d.Size), - }, - }) - // The current descriptor is a component image - case CNABDescriptorTypeComponent: - componentName, ok := d.Annotations[CNABDescriptorComponentNameAnnotation] - if !ok { - return fmt.Errorf("component name missing in descriptor %q", d.Digest) - } - if into.Images == nil { - into.Images = make(map[string]bundle.Image) - } - into.Images[componentName] = bundle.Image{ - BaseImage: bundle.BaseImage{ - Image: refFamiliar, - ImageType: imageType, - MediaType: d.MediaType, - Size: uint64(d.Size), - }, - } - default: - return fmt.Errorf("invalid CNAB descriptor type %q in descriptor %q", descriptorType, d.Digest) - } +func makeDescriptor(baseImage bundle.BaseImage, targetReference reference.Named, relocationMap relocation.ImageRelocationMap) (ocischemav1.Descriptor, error) { + relocatedImage, ok := relocationMap[baseImage.Image] + if !ok { + return ocischemav1.Descriptor{}, fmt.Errorf("image %q not present in the relocation map", baseImage.Image) } - return nil -} -func makeDescriptor(baseImage bundle.BaseImage, targetReference reference.Named) (ocischemav1.Descriptor, error) { - named, err := reference.ParseNormalizedNamed(baseImage.Image) + named, err := reference.ParseNormalizedNamed(relocatedImage) if err != nil { - return ocischemav1.Descriptor{}, fmt.Errorf("image %q is not a valid image reference: %s", baseImage.Image, err) + return ocischemav1.Descriptor{}, fmt.Errorf("image %q is not a valid image reference: %s", relocatedImage, err) } if named.Name() != targetReference.Name() { - return ocischemav1.Descriptor{}, fmt.Errorf("image %q is not in the same repository as %q", baseImage.Image, targetReference.String()) + return ocischemav1.Descriptor{}, fmt.Errorf("image %q is not in the same repository as %q", relocatedImage, targetReference.String()) } digested, ok := named.(reference.Digested) if !ok { - return ocischemav1.Descriptor{}, fmt.Errorf("image %q is not a digested reference", baseImage.Image) + return ocischemav1.Descriptor{}, fmt.Errorf("image %q is not a digested reference", relocatedImage) + } + mediaType, err := getMediaType(baseImage, relocatedImage) + if err != nil { + return ocischemav1.Descriptor{}, err + } + if baseImage.Size == 0 { + return ocischemav1.Descriptor{}, fmt.Errorf("image %q size is not set", relocatedImage) + } + + return ocischemav1.Descriptor{ + Digest: digested.Digest(), + MediaType: mediaType, + Size: int64(baseImage.Size), + }, nil +} + +func getMediaType(baseImage bundle.BaseImage, relocatedImage string) (string, error) { + mediaType := baseImage.MediaType + if mediaType == "" { + switch baseImage.ImageType { + case "docker": + mediaType = images.MediaTypeDockerSchema2Manifest + case "oci": + mediaType = ocischemav1.MediaTypeImageManifest + default: + return "", fmt.Errorf("unsupported image type %q for image %q", baseImage.ImageType, relocatedImage) + } } - switch baseImage.MediaType { + switch mediaType { case ocischemav1.MediaTypeImageManifest: case images.MediaTypeDockerSchema2Manifest: case ocischemav1.MediaTypeImageIndex: case images.MediaTypeDockerSchema2ManifestList: default: - return ocischemav1.Descriptor{}, fmt.Errorf("unsupported media type %q for image %q", baseImage.MediaType, baseImage.Image) + return "", fmt.Errorf("unsupported media type %q for image %q", baseImage.MediaType, relocatedImage) } - return ocischemav1.Descriptor{ - Digest: digested.Digest(), - MediaType: baseImage.MediaType, - Size: int64(baseImage.Size), - }, nil + return mediaType, nil } diff --git a/converter/convert_test.go b/converter/convert_test.go index 41cad32..c781887 100644 --- a/converter/convert_test.go +++ b/converter/convert_test.go @@ -10,10 +10,6 @@ import ( "gotest.tools/assert" ) -func makeTestBundleConfig() *BundleConfig { - return CreateBundleConfig(tests.MakeTestBundle()) -} - func TestConvertFromFixedUpBundleToOCI(t *testing.T) { bundleConfigDescriptor := ocischemav1.Descriptor{ Digest: "sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341", @@ -23,19 +19,21 @@ func TestConvertFromFixedUpBundleToOCI(t *testing.T) { targetRef := "my.registry/namespace/my-app:0.1.0" src := tests.MakeTestBundle() + relocationMap := tests.MakeRelocationMap() + expected := tests.MakeTestOCIIndex() // Convert from bundle to OCI index named, err := reference.ParseNormalizedNamed(targetRef) assert.NilError(t, err) - actual, err := ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor) + actual, err := ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor, relocationMap) assert.NilError(t, err) assert.DeepEqual(t, expected, actual) // Nil maintainers does not add annotation src = tests.MakeTestBundle() src.Maintainers = nil - actual, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor) + actual, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor, relocationMap) assert.NilError(t, err) _, hasMaintainers := actual.Annotations[ocischemav1.AnnotationAuthors] assert.Assert(t, !hasMaintainers) @@ -43,7 +41,7 @@ func TestConvertFromFixedUpBundleToOCI(t *testing.T) { // Nil keywords does not add annotation src = tests.MakeTestBundle() src.Keywords = nil - actual, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor) + actual, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor, relocationMap) assert.NilError(t, err) _, hasKeywords := actual.Annotations[CNABKeywordsAnnotation] assert.Assert(t, !hasKeywords) @@ -51,46 +49,55 @@ func TestConvertFromFixedUpBundleToOCI(t *testing.T) { // Multiple invocation images is not supported src = tests.MakeTestBundle() src.InvocationImages = append(src.InvocationImages, src.InvocationImages[0]) - _, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor) + _, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor, relocationMap) assert.ErrorContains(t, err, "only one invocation image supported") // Invalid media type src = tests.MakeTestBundle() src.InvocationImages[0].MediaType = "some-invalid-mediatype" - _, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor) - assert.ErrorContains(t, err, `unsupported media type "some-invalid-mediatype" for image "my.registry/namespace/my-app@sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341"`) + _, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor, relocationMap) + assert.ErrorContains(t, err, `unsupported media type "some-invalid-mediatype" for image "my.registry/namespace/my-app@sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343"`) // All images must be in the same repository src = tests.MakeTestBundle() - src.InvocationImages[0].BaseImage.Image = "my.registry/namespace/other-repo@sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341" - _, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor) + badRelocationMap := tests.MakeRelocationMap() + badRelocationMap["my.registry/namespace/my-app-invoc"] = "my.registry/namespace/other-repo@sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343" + _, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor, badRelocationMap) assert.ErrorContains(t, err, `invalid invocation image: image `+ - `"my.registry/namespace/other-repo@sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341" is not in the same repository as "my.registry/namespace/my-app:0.1.0"`) + `"my.registry/namespace/other-repo@sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343" is not in the same repository as "my.registry/namespace/my-app:0.1.0"`) // Image reference must be digested src = tests.MakeTestBundle() - src.InvocationImages[0].BaseImage.Image = "my.registry/namespace/my-app:not-digested" - _, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor) + badRelocationMap = tests.MakeRelocationMap() + badRelocationMap["my.registry/namespace/my-app-invoc"] = "my.registry/namespace/my-app:not-digested" + _, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor, badRelocationMap) assert.ErrorContains(t, err, "invalid invocation image: image \"my.registry/namespace/"+ "my-app:not-digested\" is not a digested reference") // Invalid reference src = tests.MakeTestBundle() - src.InvocationImages[0].BaseImage.Image = "Some/iNvalid/Ref" - _, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor) + badRelocationMap = tests.MakeRelocationMap() + badRelocationMap["my.registry/namespace/my-app-invoc"] = "Some/iNvalid/Ref" + _, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor, badRelocationMap) assert.ErrorContains(t, err, "invalid invocation image: "+ "image \"Some/iNvalid/Ref\" is not a valid image reference: invalid reference format: repository name must be lowercase") + // Invalid size + src = tests.MakeTestBundle() + src.InvocationImages[0].Size = 0 + _, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor, relocationMap) + assert.ErrorContains(t, err, "size is not set") + // mediatype ociindex src = tests.MakeTestBundle() src.InvocationImages[0].MediaType = ocischemav1.MediaTypeImageIndex - _, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor) + _, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor, relocationMap) assert.NilError(t, err) // mediatype docker manifestlist src = tests.MakeTestBundle() src.InvocationImages[0].MediaType = "application/vnd.docker.distribution.manifest.list.v2+json" - _, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor) + _, err = ConvertBundleToOCIIndex(src, named, bundleConfigDescriptor, relocationMap) assert.NilError(t, err) } @@ -141,62 +148,17 @@ func TestGetConfigDescriptor(t *testing.T) { assert.ErrorContains(t, err, "bundle config not found") } -func TestConvertFromOCIToBundle(t *testing.T) { +func TestGenerateRelocationMap(t *testing.T) { targetRef := "my.registry/namespace/my-app:0.1.0" named, err := reference.ParseNormalizedNamed(targetRef) assert.NilError(t, err) + ix := tests.MakeTestOCIIndex() - config := makeTestBundleConfig() - expected := tests.MakeTestBundle() + b := tests.MakeTestBundle() + + expected := tests.MakeRelocationMap() - result, err := ConvertOCIIndexToBundle(ix, config, named) + relocationMap, err := GenerateRelocationMap(ix, b, named) assert.NilError(t, err) - assert.DeepEqual(t, expected, result) - - // Without title annotation - delete(ix.Annotations, ocischemav1.AnnotationTitle) - _, err = ConvertOCIIndexToBundle(ix, config, named) - assert.ErrorContains(t, err, "manifest is missing title annotation") - - // Without version annotation - ix = tests.MakeTestOCIIndex() - delete(ix.Annotations, ocischemav1.AnnotationVersion) - _, err = ConvertOCIIndexToBundle(ix, config, named) - assert.ErrorContains(t, err, "manifest is missing version annotation") - - // Invalid authors annotation - ix = tests.MakeTestOCIIndex() - ix.Annotations[ocischemav1.AnnotationAuthors] = "Some garbage" - _, err = ConvertOCIIndexToBundle(ix, config, named) - assert.ErrorContains(t, err, "unable to parse maintainers") - - // Invalid keywords annotation - ix = tests.MakeTestOCIIndex() - ix.Annotations[CNABKeywordsAnnotation] = "Some garbage" - _, err = ConvertOCIIndexToBundle(ix, config, named) - assert.ErrorContains(t, err, "unable to parse keywords") - - // bad media type - ix = tests.MakeTestOCIIndex() - ix.Manifests[1].MediaType = "Some garbage" - _, err = ConvertOCIIndexToBundle(ix, config, named) - assert.ErrorContains(t, err, "unsupported manifest descriptor") - - // no cnab type (invocation/component) - ix = tests.MakeTestOCIIndex() - delete(ix.Manifests[1].Annotations, CNABDescriptorTypeAnnotation) - _, err = ConvertOCIIndexToBundle(ix, config, named) - assert.ErrorContains(t, err, "has no CNAB descriptor type annotation \"io.cnab.manifest.type\"") - - // bad cnab type - ix = tests.MakeTestOCIIndex() - ix.Manifests[1].Annotations[CNABDescriptorTypeAnnotation] = "Some garbage" - _, err = ConvertOCIIndexToBundle(ix, config, named) - assert.ErrorContains(t, err, "invalid CNAB descriptor type \"Some garbage\" in descriptor") - - // component name missing - ix = tests.MakeTestOCIIndex() - delete(ix.Manifests[2].Annotations, CNABDescriptorComponentNameAnnotation) - _, err = ConvertOCIIndexToBundle(ix, config, named) - assert.ErrorContains(t, err, "component name missing in descriptor") + assert.DeepEqual(t, relocationMap, expected) } diff --git a/converter/types.go b/converter/types.go index 8732dc9..1770221 100644 --- a/converter/types.go +++ b/converter/types.go @@ -1,13 +1,10 @@ package converter import ( - "encoding/json" - - "github.com/deislabs/cnab-go/bundle/definition" - "github.com/deislabs/cnab-go/bundle" "github.com/docker/distribution" "github.com/docker/distribution/manifest/schema2" + "github.com/docker/go/canonical/json" digest "github.com/opencontainers/go-digest" ocischema "github.com/opencontainers/image-spec/specs-go" ocischemav1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -18,17 +15,6 @@ const ( CNABConfigMediaType = "application/vnd.cnab.config.v1+json" ) -// BundleConfig describes a cnab bundle runtime config -type BundleConfig struct { - SchemaVersion string `json:"schemaVersion" mapstructure:"schemaVersion"` - Actions map[string]bundle.Action `json:"actions,omitempty" mapstructure:"actions,omitempty"` - Definitions definition.Definitions `json:"definitions" mapstructure:"definitions"` - Parameters map[string]bundle.Parameter `json:"parameters" mapstructure:"parameters"` - Outputs map[string]bundle.Output `json:"outputs" mapstructure:"outputs"` - Credentials map[string]bundle.Credential `json:"credentials" mapstructure:"credentials"` - Custom map[string]interface{} `json:"custom,omitempty" mapstructure:"custom"` -} - // PreparedBundleConfig contains the config blob, image manifest (and fallback), and descriptors for a CNAB config type PreparedBundleConfig struct { ConfigBlob []byte @@ -38,22 +24,9 @@ type PreparedBundleConfig struct { Fallback *PreparedBundleConfig } -// CreateBundleConfig creates a bundle config from a CNAB -func CreateBundleConfig(b *bundle.Bundle) *BundleConfig { - return &BundleConfig{ - SchemaVersion: CNABVersion, - Actions: b.Actions, - Definitions: b.Definitions, - Parameters: b.Parameters, - Outputs: b.Outputs, - Credentials: b.Credentials, - Custom: b.Custom, - } -} - // PrepareForPush serializes a bundle config, generates its image manifest, and its manifest descriptor -func (c *BundleConfig) PrepareForPush() (*PreparedBundleConfig, error) { - blob, err := json.Marshal(c) +func PrepareForPush(b *bundle.Bundle) (*PreparedBundleConfig, error) { + blob, err := json.MarshalCanonical(b) if err != nil { return nil, err } diff --git a/converter/types_test.go b/converter/types_test.go index 3a44974..474c710 100644 --- a/converter/types_test.go +++ b/converter/types_test.go @@ -3,12 +3,13 @@ package converter import ( "testing" + "github.com/deislabs/cnab-go/bundle" "gotest.tools/assert" ) func TestPrepareForPush(t *testing.T) { - b := &BundleConfig{} - prepared, err := b.PrepareForPush() + b := &bundle.Bundle{} + prepared, err := PrepareForPush(b) assert.NilError(t, err) // First try with OCI format and specific CNAB media type. Fallback should be set. diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index d48766b..5158d52 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -1,6 +1,7 @@ package e2e import ( + "encoding/json" "fmt" "io/ioutil" "os" @@ -25,9 +26,13 @@ func TestPushAndPullCNAB(t *testing.T) { invocationImageName := registry + "/e2e/hello-world:0.1.0-invoc" serviceImageName := registry + "/e2e/http-echo" + appImageName := registry + "/myuser" + // Build invocation image - runCmd(t, icmd.Command("docker", "build", "-f", filepath.Join("testdata", "hello-world", "invocation-image", "Dockerfile"), - "-t", invocationImageName, filepath.Join("testdata", "hello-world", "invocation-image"))) + cmd := icmd.Command("docker", "build", "-f", filepath.Join("testdata", "hello-world", "invocation-image", "Dockerfile"), + "-t", invocationImageName, filepath.Join("testdata", "hello-world", "invocation-image")) + cmd.Env = append(os.Environ(), "DOCKER_BUILDKIT=1") + runCmd(t, cmd) // Fetch service image runCmd(t, icmd.Command("docker", "pull", "hashicorp/http-echo")) @@ -39,53 +44,59 @@ func TestPushAndPullCNAB(t *testing.T) { }() // Push the images to the registry - runCmd(t, icmd.Command("docker", "push", invocationImageName)) + output := runCmd(t, icmd.Command("docker", "push", invocationImageName)) + invocDigest := getDigest(t, output) + runCmd(t, icmd.Command("docker", "push", serviceImageName)) // Templatize the bundle - tmpl, err := template.ParseFiles(filepath.Join("testdata", "hello-world", "bundle.json.template")) - assert.NilError(t, err) - data := struct { - InvocationImage string - ServiceImage string - }{ - invocationImageName, - serviceImageName, - } - f, err := os.Create(dir.Join("bundle.json")) - assert.NilError(t, err) - defer f.Close() - err = tmpl.Execute(f, data) - assert.NilError(t, err) + applyTemplate(t, serviceImageName, invocationImageName, invocDigest, filepath.Join("testdata", "hello-world", "bundle.json.template"), dir.Join("bundle.json")) // Save the fixed bundle runCmd(t, icmd.Command("cnab-to-oci", "fixup", dir.Join("bundle.json"), - "--target", registry+"/myuser", + "--target", appImageName, "--insecure-registries", registry, - "--output", dir.Join("fixed-bundle.json"))) + "--bundle", dir.Join("fixed-bundle.json"), + "--relocation-map", dir.Join("relocation.json"), + "--auto-update-bundle")) + + // Check the fixed bundle + applyTemplate(t, serviceImageName, invocationImageName, invocDigest, filepath.Join("testdata", "bundle.json.golden.template"), filepath.Join("testdata", "bundle.json.golden")) + buf, err := ioutil.ReadFile(dir.Join("fixed-bundle.json")) + assert.NilError(t, err) + golden.Assert(t, string(buf), "bundle.json.golden") + + // Check the relocation map + checkRelocationMap(t, serviceImageName, invocationImageName, appImageName, dir.Join("relocation.json")) // Re fix-up, checking it works twice runCmd(t, icmd.Command("cnab-to-oci", "fixup", dir.Join("bundle.json"), - "--target", registry+"/myuser", + "--target", appImageName, "--insecure-registries", registry, - "--output", dir.Join("fixed-bundle.json"))) + "--bundle", dir.Join("fixed-bundle.json"), + "--auto-update-bundle")) // Push the CNAB to the registry and get the digest out := runCmd(t, icmd.Command("cnab-to-oci", "push", dir.Join("bundle.json"), - "--target", registry+"/myuser", - "--insecure-registries", registry)) + "--target", appImageName, + "--insecure-registries", registry, + "--auto-update-bundle")) re := regexp.MustCompile(`"(.*)"`) digest := re.FindAllStringSubmatch(out, -1)[0][1] // Pull the CNAB from the registry - runCmd(t, icmd.Command("cnab-to-oci", "pull", registry+"/myuser@"+digest, - "--output", dir.Join("pulled-bundle.json"), + runCmd(t, icmd.Command("cnab-to-oci", "pull", fmt.Sprintf("%s@%s", appImageName, digest), + "--bundle", dir.Join("pulled-bundle.json"), + "--relocation-map", dir.Join("pulled-relocation.json"), "--insecure-registries", registry)) pulledBundle, err := ioutil.ReadFile(dir.Join("pulled-bundle.json")) assert.NilError(t, err) + pulledRelocation, err := ioutil.ReadFile(dir.Join("pulled-relocation.json")) + assert.NilError(t, err) // Check the fixed bundle.json is equal to the pulled bundle.json golden.Assert(t, string(pulledBundle), dir.Join("fixed-bundle.json")) + golden.Assert(t, string(pulledRelocation), dir.Join("relocation.json")) } func runCmd(t *testing.T, cmd icmd.Cmd) string { @@ -95,3 +106,43 @@ func runCmd(t *testing.T, cmd icmd.Cmd) string { result.Assert(t, icmd.Success) return result.Stdout() } + +func applyTemplate(t *testing.T, serviceImageName, invocationImageName, invocationDigest, templateFile, resultFile string) { + tmpl, err := template.ParseFiles(templateFile) + assert.NilError(t, err) + data := struct { + InvocationImage string + InvocationDigest string + ServiceImage string + }{ + invocationImageName, + invocationDigest, + serviceImageName, + } + f, err := os.Create(resultFile) + assert.NilError(t, err) + defer f.Close() + err = tmpl.Execute(f, data) + assert.NilError(t, err) +} + +func checkRelocationMap(t *testing.T, serviceImageName, invocationImageName, appImageName, relocationMapFile string) { + data, err := ioutil.ReadFile(relocationMapFile) + assert.NilError(t, err) + relocationMap := map[string]string{} + err = json.Unmarshal(data, &relocationMap) + assert.NilError(t, err) + + // Check the relocated images are in the app repository + assert.Assert(t, strings.HasPrefix(relocationMap[serviceImageName], appImageName)) + assert.Assert(t, strings.HasPrefix(relocationMap[invocationImageName], appImageName)) +} + +func getDigest(t *testing.T, output string) string { + re := regexp.MustCompile(`digest: (.+) size:`) + result := re.FindStringSubmatch(output) + assert.Equal(t, len(result), 2) + digest := result[1] + assert.Assert(t, digest != "") + return digest +} diff --git a/e2e/testdata/bundle.json.golden.template b/e2e/testdata/bundle.json.golden.template new file mode 100644 index 0000000..f80be0d --- /dev/null +++ b/e2e/testdata/bundle.json.golden.template @@ -0,0 +1 @@ +{"actions":{"io.cnab.status":{}},"definitions":{"port":{"default":"8080","type":"string"},"text":{"default":"Hello, World!","type":"string"}},"description":"Hello, World!","images":{"hello":{"contentDigest":"sha256:61d5cb94d7e546518a7bbd5bee06bfad0ecea8f56a75b084522a43dccbbcd845","description":"hello","image":"{{ .ServiceImage }}","imageType":"docker","mediaType":"application/vnd.docker.distribution.manifest.v2+json","size":528}},"invocationImages":[{"contentDigest":"{{ .InvocationDigest }}","image":"{{ .InvocationImage }}","imageType":"docker","mediaType":"application/vnd.docker.distribution.manifest.v2+json","size":941}],"maintainers":[{"email":"user@email.com","name":"user"}],"name":"hello-world","parameters":{"fields":{"definition":"","destination":null}},"schemaVersion":"v1.0.0-WD","version":"0.1.0"} \ No newline at end of file diff --git a/e2e/testdata/hello-world/bundle.json.template b/e2e/testdata/hello-world/bundle.json.template index 2c26261..70d3995 100644 --- a/e2e/testdata/hello-world/bundle.json.template +++ b/e2e/testdata/hello-world/bundle.json.template @@ -1,53 +1 @@ -{ - "schemaVersion": "v1.0.0-WD", - "name": "hello-world", - "version": "0.1.0", - "description": "Hello, World!", - "maintainers": [ - { - "name": "user", - "email": "user@email.com" - } - ], - "invocationImages": [ - { - "imageType": "docker", - "image": "{{ .InvocationImage }}" - } - ], - "images": { - "hello": { - "imageType": "docker", - "image": "{{ .ServiceImage }}" - } - }, - "actions": { - "io.cnab.status": {} - }, - "parameters": { - "fields": { - "port": { - "definition": "port", - "destination": { - "env": "PORT" - } - }, - "text": { - "definition": "text", - "destination": { - "env": "HELLO_TEXT" - } - } - } - }, - "definitions": { - "port": { - "default": "8080", - "type": "string" - }, - "text": { - "default": "Hello, World!", - "type": "string" - } - } -} \ No newline at end of file +{"actions":{"io.cnab.status":{}},"definitions":{"port":{"default":"8080","type":"string"},"text":{"default":"Hello, World!","type":"string"}},"description":"Hello, World!","images":{"hello":{"description":"hello","image":"{{ .ServiceImage }}","imageType":"docker"}},"invocationImages":[{"image":"{{ .InvocationImage }}","imageType":"docker"}],"maintainers":[{"email":"user@email.com","name":"user"}],"name":"hello-world","parameters":{"fields":{"port":{"definition":"port","destination":{"env":"PORT"}},"text":{"definition":"text","destination":{"env":"HELLO_TEXT"}}}},"schemaVersion":"v1.0.0-WD","version":"0.1.0"} \ No newline at end of file diff --git a/examples/helloworld-cnab/bundle.json b/examples/helloworld-cnab/bundle.json index 3c41638..7fd9431 100644 --- a/examples/helloworld-cnab/bundle.json +++ b/examples/helloworld-cnab/bundle.json @@ -18,7 +18,8 @@ "invocationImages": [ { "imageType": "docker", - "image": "cnab/helloworld:0.1.1" + "image": "cnab/helloworld:0.1.1", + "size": 42 } ], "images": null, diff --git a/relocation/types.go b/relocation/types.go new file mode 100644 index 0000000..ffe4ea7 --- /dev/null +++ b/relocation/types.go @@ -0,0 +1,4 @@ +package relocation + +// ImageRelocationMap stores the mapping between the original image reference as key, and the relocated reference as a value. +type ImageRelocationMap map[string]string diff --git a/remotes/fixup.go b/remotes/fixup.go index cf811ca..4403940 100644 --- a/remotes/fixup.go +++ b/remotes/fixup.go @@ -11,20 +11,21 @@ import ( "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/remotes" "github.com/deislabs/cnab-go/bundle" + "github.com/docker/cnab-to-oci/relocation" "github.com/docker/distribution/reference" ocischemav1 "github.com/opencontainers/image-spec/specs-go/v1" ) // FixupBundle checks that all the references are present in the referenced repository, otherwise it will mount all // the manifests to that repository. The bundle is then patched with the new digested references. -func FixupBundle(ctx context.Context, b *bundle.Bundle, ref reference.Named, resolver remotes.Resolver, opts ...FixupOption) error { +func FixupBundle(ctx context.Context, b *bundle.Bundle, ref reference.Named, resolver remotes.Resolver, opts ...FixupOption) (relocation.ImageRelocationMap, error) { logger := log.G(ctx) logger.Debugf("Fixing up bundle %s", ref) - // Configure the fixup and the even loop + // Configure the fixup and the event loop cfg, err := newFixupConfig(b, ref, resolver, opts...) if err != nil { - return err + return nil, err } events := make(chan FixupEvent) @@ -43,37 +44,64 @@ func FixupBundle(ctx context.Context, b *bundle.Bundle, ref reference.Named, res // Fixup invocation images if len(b.InvocationImages) != 1 { - return fmt.Errorf("only one invocation image supported for bundle %q", ref) + return nil, fmt.Errorf("only one invocation image supported for bundle %q", ref) } - if b.InvocationImages[0].BaseImage, err = fixupImage(ctx, b.InvocationImages[0].BaseImage, cfg, events, cfg.invocationImagePlatformFilter); err != nil { - return err + + relocationMap := relocation.ImageRelocationMap{} + if err := fixupImage(ctx, &b.InvocationImages[0].BaseImage, relocationMap, cfg, events, cfg.invocationImagePlatformFilter); err != nil { + return nil, err } // Fixup images for name, original := range b.Images { - if original.BaseImage, err = fixupImage(ctx, original.BaseImage, cfg, events, cfg.componentImagePlatformFilter); err != nil { - return err + if err := fixupImage(ctx, &original.BaseImage, relocationMap, cfg, events, cfg.componentImagePlatformFilter); err != nil { + return nil, err } b.Images[name] = original } logger.Debug("Bundle fixed") - return nil + return relocationMap, nil } -func fixupImage(ctx context.Context, baseImage bundle.BaseImage, cfg fixupConfig, events chan<- FixupEvent, platformFilter platforms.Matcher) (bundle.BaseImage, error) { - log.G(ctx).Debugf("Fixing image %s", baseImage.Image) +func fixupImage(ctx context.Context, baseImage *bundle.BaseImage, relocationMap relocation.ImageRelocationMap, cfg fixupConfig, events chan<- FixupEvent, platformFilter platforms.Matcher) error { + log.G(ctx).Debugf("Updating entry in relocation map for %q", baseImage.Image) ctx = withMutedContext(ctx) notifyEvent, progress := makeEventNotifier(events, baseImage.Image, cfg.targetRef) notifyEvent(FixupEventTypeCopyImageStart, "", nil) // Fixup Base image - fixupInfo, err := fixupBaseImage(ctx, &baseImage, cfg.targetRef, cfg.resolver) + fixupInfo, err := fixupBaseImage(ctx, baseImage, cfg.targetRef, cfg.resolver) if err != nil { return notifyError(notifyEvent, err) } + // Update the relocation map with the original image name and the digested reference of the image pushed inside the bundle repository + newRef, err := reference.WithDigest(fixupInfo.targetRepo, fixupInfo.resolvedDescriptor.Digest) + if err != nil { + return err + } + + relocationMap[baseImage.Image] = newRef.String() + + // if the autoUpdateBundle flag is passed, mutate the bundle with the resolved digest, mediaType, and size + if cfg.autoBundleUpdate { + baseImage.Digest = fixupInfo.resolvedDescriptor.Digest.String() + baseImage.Size = uint64(fixupInfo.resolvedDescriptor.Size) + baseImage.MediaType = fixupInfo.resolvedDescriptor.MediaType + } else { + if baseImage.Digest != fixupInfo.resolvedDescriptor.Digest.String() { + return fmt.Errorf("image %q digest differs %q after fixup: %q", baseImage.Image, baseImage.Digest, fixupInfo.resolvedDescriptor.Digest.String()) + } + if baseImage.Size != uint64(fixupInfo.resolvedDescriptor.Size) { + return fmt.Errorf("image %q size differs %d after fixup: %d", baseImage.Image, baseImage.Size, fixupInfo.resolvedDescriptor.Size) + } + if baseImage.MediaType != fixupInfo.resolvedDescriptor.MediaType { + return fmt.Errorf("image %q media type differs %q after fixup: %q", baseImage.Image, baseImage.MediaType, fixupInfo.resolvedDescriptor.MediaType) + } + } + if fixupInfo.sourceRef.Name() == fixupInfo.targetRepo.Name() { notifyEvent(FixupEventTypeCopyImageEnd, "Nothing to do: image reference is already present in repository"+fixupInfo.targetRepo.String(), nil) - return baseImage, nil + return nil } sourceFetcher, err := makeSourceFetcher(ctx, cfg.resolver, fixupInfo.sourceRef.Name()) @@ -82,7 +110,7 @@ func fixupImage(ctx context.Context, baseImage bundle.BaseImage, cfg fixupConfig } // Fixup platforms - if err := fixupPlatforms(ctx, &baseImage, &fixupInfo, sourceFetcher, platformFilter); err != nil { + if err := fixupPlatforms(ctx, baseImage, relocationMap, &fixupInfo, sourceFetcher, platformFilter); err != nil { return notifyError(notifyEvent, err) } @@ -97,12 +125,21 @@ func fixupImage(ctx context.Context, baseImage bundle.BaseImage, cfg fixupConfig } notifyEvent(FixupEventTypeCopyImageEnd, "", nil) - return baseImage, nil + return nil } -func fixupPlatforms(ctx context.Context, baseImage *bundle.BaseImage, fixupInfo *imageFixupInfo, sourceFetcher sourceFetcherAdder, filter platforms.Matcher) error { +func fixupPlatforms(ctx context.Context, + baseImage *bundle.BaseImage, + relocationMap relocation.ImageRelocationMap, + fixupInfo *imageFixupInfo, + sourceFetcher sourceFetcherAdder, + filter platforms.Matcher) error { + + logger := log.G(ctx) + logger.Debugf("Fixup platforms for image %v, with relocation map %v", baseImage, relocationMap) if filter == nil || - (fixupInfo.resolvedDescriptor.MediaType != ocischemav1.MediaTypeImageIndex && fixupInfo.resolvedDescriptor.MediaType != images.MediaTypeDockerSchema2ManifestList) { + (fixupInfo.resolvedDescriptor.MediaType != ocischemav1.MediaTypeImageIndex && + fixupInfo.resolvedDescriptor.MediaType != images.MediaTypeDockerSchema2ManifestList) { // no platform filter if platform is empty, or if the descriptor is not an OCI Index / Docker Manifest list return nil } @@ -140,11 +177,7 @@ func fixupPlatforms(ctx context.Context, baseImage *bundle.BaseImage, fixupInfo descriptor.Digest = d descriptor.Size = int64(len(manifestBytes)) fixupInfo.resolvedDescriptor = descriptor - newRef, err := reference.WithDigest(fixupInfo.targetRepo, d) - if err != nil { - return err - } - baseImage.Image = newRef.String() + return nil } @@ -172,13 +205,6 @@ func fixupBaseImage(ctx context.Context, if err != nil { return imageFixupInfo{}, fmt.Errorf("failed to resolve %q, push the image to the registry before pushing the bundle: %s", sourceImageRef, err) } - digested, err := reference.WithDigest(targetRepoOnly, descriptor.Digest) - if err != nil { - return imageFixupInfo{}, err - } - baseImage.Image = reference.FamiliarString(digested) - baseImage.MediaType = descriptor.MediaType - baseImage.Size = uint64(descriptor.Size) return imageFixupInfo{ resolvedDescriptor: descriptor, sourceRef: sourceImageRef, diff --git a/remotes/fixup_test.go b/remotes/fixup_test.go index 2b15b1d..f59f0b9 100644 --- a/remotes/fixup_test.go +++ b/remotes/fixup_test.go @@ -12,12 +12,233 @@ import ( "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/remotes" "github.com/deislabs/cnab-go/bundle" + "github.com/docker/cnab-to-oci/relocation" "github.com/docker/distribution/reference" "github.com/opencontainers/go-digest" ocischemav1 "github.com/opencontainers/image-spec/specs-go/v1" "gotest.tools/assert" ) +func TestFixupBundleWithAutoUpdate(t *testing.T) { + index := ocischemav1.Manifest{} + bufManifest, err := json.Marshal(index) + assert.NilError(t, err) + fetcher := &mockFetcher{indexBuffers: []*bytes.Buffer{ + // Manifest index + bytes.NewBuffer(bufManifest), + }} + pusher := &mockPusher{} + resolver := &mockResolver{ + pusher: pusher, + fetcher: fetcher, + resolvedDescriptors: []ocischemav1.Descriptor{ + // Resolving source Invocation image manifest descriptor my.registry/namespace/my-app-invoc + { + MediaType: ocischemav1.MediaTypeImageManifest, + Size: 42, + Digest: "sha256:beef1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343", + }, + // Target Invocation image manifest descriptor my.registry/namespace/my-app@sha256:beef1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343 for mounting + {}, + // Resolving source service image manifest descriptor my.registry/namespace/my-service + { + MediaType: ocischemav1.MediaTypeImageManifest, + Size: 43, + Digest: "sha256:beef1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0344", + }, + // Target service image manifest descriptor my.registry/namespace/my-app@sha256:beef1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0344 for mounting + {}, + }, + } + b := &bundle.Bundle{ + SchemaVersion: "v1.0.0-WD", + InvocationImages: []bundle.InvocationImage{ + { + BaseImage: bundle.BaseImage{ + Image: "my.registry/namespace/my-app-invoc", + ImageType: "docker", + }, + }, + }, + Images: map[string]bundle.Image{ + "my-service": { + BaseImage: bundle.BaseImage{ + Image: "my.registry/namespace/my-service", + ImageType: "docker", + }, + }, + }, + Name: "my-app", + Version: "0.1.0", + } + ref, err := reference.ParseNamed("my.registry/namespace/my-app") + assert.NilError(t, err) + _, err = FixupBundle(context.TODO(), b, ref, resolver, WithAutoBundleUpdate()) + assert.NilError(t, err) + expectedBundle := &bundle.Bundle{ + SchemaVersion: "v1.0.0-WD", + InvocationImages: []bundle.InvocationImage{ + { + BaseImage: bundle.BaseImage{ + Image: "my.registry/namespace/my-app-invoc", + ImageType: "docker", + MediaType: ocischemav1.MediaTypeImageManifest, + Size: 42, + Digest: "sha256:beef1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343", + }, + }, + }, + Images: map[string]bundle.Image{ + "my-service": { + BaseImage: bundle.BaseImage{ + Image: "my.registry/namespace/my-service", + ImageType: "docker", + MediaType: ocischemav1.MediaTypeImageManifest, + Size: 43, + Digest: "sha256:beef1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0344", + }, + }, + }, + Name: "my-app", + Version: "0.1.0", + } + assert.DeepEqual(t, b, expectedBundle) +} + +func TestFixupBundleFailsWithDifferentDigests(t *testing.T) { + index := ocischemav1.Manifest{} + bufManifest, err := json.Marshal(index) + assert.NilError(t, err) + fetcher := &mockFetcher{indexBuffers: []*bytes.Buffer{ + // Manifest index + bytes.NewBuffer(bufManifest), + }} + pusher := &mockPusher{} + resolver := &mockResolver{ + pusher: pusher, + fetcher: fetcher, + resolvedDescriptors: []ocischemav1.Descriptor{ + // Invocation image manifest descriptor + { + MediaType: ocischemav1.MediaTypeImageManifest, + Size: 42, + Digest: "sha256:c0ffeea7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343", + }, + {}, + }, + } + b := &bundle.Bundle{ + SchemaVersion: "v1.0.0-WD", + InvocationImages: []bundle.InvocationImage{ + { + BaseImage: bundle.BaseImage{ + Image: "my.registry/namespace/my-app-invoc", + ImageType: "docker", + Digest: "beef00a7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343", + Size: 42, + MediaType: ocischemav1.MediaTypeImageManifest, + }, + }, + }, + Name: "my-app", + Version: "0.1.0", + } + ref, err := reference.ParseNamed("my.registry/namespace/my-app") + assert.NilError(t, err) + _, err = FixupBundle(context.TODO(), b, ref, resolver) + assert.ErrorContains(t, err, "digest differs") +} + +func TestFixupBundleFailsWithDifferentSizes(t *testing.T) { + index := ocischemav1.Manifest{} + bufManifest, err := json.Marshal(index) + assert.NilError(t, err) + fetcher := &mockFetcher{indexBuffers: []*bytes.Buffer{ + // Manifest index + bytes.NewBuffer(bufManifest), + }} + pusher := &mockPusher{} + resolver := &mockResolver{ + pusher: pusher, + fetcher: fetcher, + resolvedDescriptors: []ocischemav1.Descriptor{ + // Invocation image manifest descriptor + { + MediaType: ocischemav1.MediaTypeImageManifest, + Size: 43, + Digest: "sha256:c0ffeea7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343", + }, + {}, + }, + } + + b := &bundle.Bundle{ + SchemaVersion: "v1.0.0-WD", + InvocationImages: []bundle.InvocationImage{ + { + BaseImage: bundle.BaseImage{ + Image: "my.registry/namespace/my-app-invoc", + ImageType: "docker", + Digest: "sha256:c0ffeea7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343", + Size: 42, + MediaType: ocischemav1.MediaTypeImageManifest, + }, + }, + }, + Name: "my-app", + Version: "0.1.0", + } + ref, err := reference.ParseNamed("my.registry/namespace/my-app") + assert.NilError(t, err) + _, err = FixupBundle(context.TODO(), b, ref, resolver) + assert.ErrorContains(t, err, "size differs") +} + +func TestFixupBundleFailsWithDifferentMediaTypes(t *testing.T) { + index := ocischemav1.Manifest{} + bufManifest, err := json.Marshal(index) + assert.NilError(t, err) + fetcher := &mockFetcher{indexBuffers: []*bytes.Buffer{ + // Manifest index + bytes.NewBuffer(bufManifest), + }} + pusher := &mockPusher{} + resolver := &mockResolver{ + pusher: pusher, + fetcher: fetcher, + resolvedDescriptors: []ocischemav1.Descriptor{ + // Invocation image manifest descriptor + { + MediaType: ocischemav1.MediaTypeImageIndex, + Size: 42, + Digest: "sha256:c0ffeea7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343", + }, + {}, + }, + } + + b := &bundle.Bundle{ + SchemaVersion: "v1.0.0-WD", + InvocationImages: []bundle.InvocationImage{ + { + BaseImage: bundle.BaseImage{ + Image: "my.registry/namespace/my-app-invoc", + ImageType: "docker", + Digest: "sha256:c0ffeea7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343", + Size: 42, + MediaType: ocischemav1.MediaTypeImageManifest, + }, + }, + }, + Name: "my-app", + Version: "0.1.0", + } + ref, err := reference.ParseNamed("my.registry/namespace/my-app") + assert.NilError(t, err) + _, err = FixupBundle(context.TODO(), b, ref, resolver) + assert.ErrorContains(t, err, "media type differs") +} + func TestFixupPlatformShortPaths(t *testing.T) { // those cases should not need to fetch any data cases := []struct { @@ -52,7 +273,7 @@ func TestFixupPlatformShortPaths(t *testing.T) { if c.platform != "" { filter = platforms.NewMatcher(platforms.MustParse(c.platform)) } - assert.NilError(t, fixupPlatforms(context.Background(), &bundle.BaseImage{}, &imageFixupInfo{ + assert.NilError(t, fixupPlatforms(context.Background(), &bundle.BaseImage{}, relocation.ImageRelocationMap{}, &imageFixupInfo{ resolvedDescriptor: ocischemav1.Descriptor{ MediaType: c.mediaType, }, @@ -106,7 +327,7 @@ func TestFixupPlatforms(t *testing.T) { assert.NilError(t, err) sourceRef, err := reference.WithDigest(sourceRepo, sourceDigest) assert.NilError(t, err) - bi := &bundle.BaseImage{ + bi := bundle.BaseImage{ Image: sourceRef.String(), } fixupInfo := &imageFixupInfo{ @@ -123,7 +344,7 @@ func TestFixupPlatforms(t *testing.T) { sourceFetcher := newSourceFetcherWithLocalData(bytesFetcher(sourceBytes)) // fixup - err = fixupPlatforms(context.Background(), bi, fixupInfo, sourceFetcher, filter) + err = fixupPlatforms(context.Background(), &bi, relocation.ImageRelocationMap{}, fixupInfo, sourceFetcher, filter) if c.expectedError != "" { assert.ErrorContains(t, err, c.expectedError) return @@ -131,7 +352,7 @@ func TestFixupPlatforms(t *testing.T) { assert.NilError(t, err) // baseImage.Image should have changed - assert.Check(t, bi.Image != sourceRef.String()) + // assert.Check(t, bi.Image != sourceRef.String()) // resolved digest should have changed assert.Check(t, fixupInfo.resolvedDescriptor.Digest != sourceDigest) diff --git a/remotes/fixuphelpers.go b/remotes/fixuphelpers.go index 980a522..0255b31 100644 --- a/remotes/fixuphelpers.go +++ b/remotes/fixuphelpers.go @@ -97,9 +97,9 @@ func makeManifestWalker(ctx context.Context, sourceFetcher remotes.Fetcher, return walker.walk(scheduler.ctx(), fixupInfo.resolvedDescriptor, nil), cleaner, nil } -func notifyError(notifyEvent eventNotifier, err error) (bundle.BaseImage, error) { +func notifyError(notifyEvent eventNotifier, err error) error { notifyEvent(FixupEventTypeCopyImageEnd, "", err) - return bundle.BaseImage{}, err + return err } func checkBaseImage(baseImage *bundle.BaseImage) error { @@ -119,7 +119,7 @@ func checkBaseImage(baseImage *bundle.BaseImage) error { case images.MediaTypeDockerSchema2ManifestList: case "": default: - return fmt.Errorf("image media type %q is not supported", baseImage.ImageType) + return fmt.Errorf("image media type %q is not supported", baseImage.MediaType) } return nil diff --git a/remotes/fixupoptions.go b/remotes/fixupoptions.go index 698fd3d..118bbb3 100644 --- a/remotes/fixupoptions.go +++ b/remotes/fixupoptions.go @@ -25,6 +25,7 @@ type fixupConfig struct { resolver remotes.Resolver invocationImagePlatformFilter platforms.Matcher componentImagePlatformFilter platforms.Matcher + autoBundleUpdate bool } // FixupOption is a helper for configuring a FixupBundle @@ -105,3 +106,11 @@ func WithParallelism(maxConcurrentJobs int, jobsBufferLength int) FixupOption { return nil } } + +// WithAutoBundleUpdate updates the bundle with content digests and size provided by the registry +func WithAutoBundleUpdate() FixupOption { + return func(cfg *fixupConfig) error { + cfg.autoBundleUpdate = true + return nil + } +} diff --git a/remotes/pull.go b/remotes/pull.go index 143b4cc..a212b84 100644 --- a/remotes/pull.go +++ b/remotes/pull.go @@ -12,23 +12,28 @@ import ( "github.com/deislabs/cnab-go/bundle" "github.com/docker/cli/opts" "github.com/docker/cnab-to-oci/converter" + "github.com/docker/cnab-to-oci/relocation" "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client/auth" ocischemav1 "github.com/opencontainers/image-spec/specs-go/v1" ) // Pull pulls a bundle from an OCI Image Index manifest -func Pull(ctx context.Context, ref reference.Named, resolver remotes.Resolver) (*bundle.Bundle, error) { +func Pull(ctx context.Context, ref reference.Named, resolver remotes.Resolver) (*bundle.Bundle, relocation.ImageRelocationMap, error) { log.G(ctx).Debugf("Pulling CNAB Bundle %s", ref) index, err := getIndex(ctx, ref, resolver) if err != nil { - return nil, err + return nil, nil, err } - config, err := getConfig(ctx, ref, resolver, index) + b, err := getBundle(ctx, ref, resolver, index) if err != nil { - return nil, err + return nil, nil, err + } + relocationMap, err := converter.GenerateRelocationMap(&index, b, ref) + if err != nil { + return nil, nil, err } - return converter.ConvertOCIIndexToBundle(&index, &config, ref) + return b, relocationMap, nil } func getIndex(ctx context.Context, ref auth.Scope, resolver remotes.Resolver) (ocischemav1.Index, error) { @@ -58,25 +63,25 @@ func getIndex(ctx context.Context, ref auth.Scope, resolver remotes.Resolver) (o return index, nil } -func getConfig(ctx context.Context, ref opts.NamedOption, resolver remotes.Resolver, index ocischemav1.Index) (converter.BundleConfig, error) { +func getBundle(ctx context.Context, ref opts.NamedOption, resolver remotes.Resolver, index ocischemav1.Index) (*bundle.Bundle, error) { repoOnly, err := reference.ParseNormalizedNamed(ref.Name()) if err != nil { - return converter.BundleConfig{}, fmt.Errorf("invalid bundle config manifest reference name %q: %s", ref, err) + return nil, fmt.Errorf("invalid bundle manifest reference name %q: %s", ref, err) } // config is wrapped in an image manifest. So we first pull the manifest // and then the config blob within it configManifestDescriptor, err := getConfigManifestDescriptor(ctx, ref, index) if err != nil { - return converter.BundleConfig{}, err + return nil, err } manifest, err := getConfigManifest(ctx, ref, repoOnly, resolver, configManifestDescriptor) if err != nil { - return converter.BundleConfig{}, err + return nil, err } - // Pull now the config itself + // Pull now the bundle itself return getBundleConfig(ctx, ref, repoOnly, resolver, manifest) } @@ -114,13 +119,13 @@ func getConfigManifest(ctx context.Context, ref opts.NamedOption, repoOnly refer return manifest, err } -func getBundleConfig(ctx context.Context, ref opts.NamedOption, repoOnly reference.Named, resolver remotes.Resolver, manifest ocischemav1.Manifest) (converter.BundleConfig, error) { +func getBundleConfig(ctx context.Context, ref opts.NamedOption, repoOnly reference.Named, resolver remotes.Resolver, manifest ocischemav1.Manifest) (*bundle.Bundle, error) { logger := log.G(ctx) - logger.Debugf("Fetching Bundle Config %s", manifest.Config.Digest) + logger.Debugf("Fetching Bundle %s", manifest.Config.Digest) configRef, err := reference.WithDigest(repoOnly, manifest.Config.Digest) if err != nil { - return converter.BundleConfig{}, fmt.Errorf("invalid bundle config reference name %q: %s", ref, err) + return nil, fmt.Errorf("invalid bundle reference name %q: %s", ref, err) } configPayload, err := pullPayload(ctx, resolver, configRef.String(), ocischemav1.Descriptor{ Digest: manifest.Config.Digest, @@ -128,15 +133,15 @@ func getBundleConfig(ctx context.Context, ref opts.NamedOption, repoOnly referen Size: manifest.Config.Size, }) if err != nil { - return converter.BundleConfig{}, fmt.Errorf("failed to pull bundle config %q: %s", ref, err) + return nil, fmt.Errorf("failed to pull bundle %q: %s", ref, err) } - var config converter.BundleConfig - if err := json.Unmarshal(configPayload, &config); err != nil { - return converter.BundleConfig{}, fmt.Errorf("failed to pull bundle config %q: %s", ref, err) + var b bundle.Bundle + if err := json.Unmarshal(configPayload, &b); err != nil { + return nil, fmt.Errorf("failed to pull bundle %q: %s", ref, err) } - logPayload(logger, config) + logPayload(logger, b) - return config, nil + return &b, nil } func pullPayload(ctx context.Context, resolver remotes.Resolver, reference string, descriptor ocischemav1.Descriptor) ([]byte, error) { @@ -150,5 +155,7 @@ func pullPayload(ctx context.Context, resolver remotes.Resolver, reference strin return nil, err } defer reader.Close() - return ioutil.ReadAll(reader) + + result, err := ioutil.ReadAll(reader) + return result, err } diff --git a/remotes/pull_test.go b/remotes/pull_test.go index 434796e..e00117c 100644 --- a/remotes/pull_test.go +++ b/remotes/pull_test.go @@ -4,10 +4,10 @@ import ( "bytes" "context" "encoding/json" + "fmt" "os" "testing" - "github.com/docker/cnab-to-oci/converter" "github.com/docker/cnab-to-oci/tests" "github.com/docker/distribution/reference" ocischemav1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -30,8 +30,8 @@ func TestPull(t *testing.T) { "layers": null }`) - config := converter.CreateBundleConfig(tests.MakeTestBundle()) - bufBundleConfig, err := json.Marshal(config) + b := tests.MakeTestBundle() + bufBundle, err := json.Marshal(b) assert.NilError(t, err) fetcher := &mockFetcher{indexBuffers: []*bytes.Buffer{ @@ -40,7 +40,7 @@ func TestPull(t *testing.T) { // Bundle config manifest bytes.NewBuffer(bundleConfigManifestDescriptor), // Bundle config - bytes.NewBuffer(bufBundleConfig), + bytes.NewBuffer(bufBundle), }} resolver := &mockResolver{ fetcher: fetcher, @@ -60,7 +60,7 @@ func TestPull(t *testing.T) { assert.NilError(t, err) // Pull the CNAB and get the bundle - b, err := Pull(context.Background(), ref, resolver) + b, _, err = Pull(context.Background(), ref, resolver) assert.NilError(t, err) expectedBundle := tests.MakeTestBundle() assert.DeepEqual(t, expectedBundle, b) @@ -75,15 +75,22 @@ func ExamplePull() { panic(err) } - // Pull the CNAB and get the bundle - resultBundle, err := Pull(context.Background(), ref, resolver) + // Pull the CNAB, get the bundle and the associated relocation map + resultBundle, relocationMap, err := Pull(context.Background(), ref, resolver) if err != nil { panic(err) } resultBundle.WriteTo(os.Stdout) + buf, err := json.Marshal(relocationMap) + if err != nil { + panic(err) + } + fmt.Printf("\n") + fmt.Println(string(buf)) // Output: - //{"actions":{"action-1":{"modifies":true}},"credentials":{"cred-1":{"env":"env-var","path":"/some/path"}},"definitions":{"param1Type":{"default":"hello","enum":["value1",true,1],"type":["string","boolean","number"]}},"description":"description","images":{"image-1":{"description":"","image":"my.registry/namespace/my-app@sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341","imageType":"oci","mediaType":"application/vnd.oci.image.manifest.v1+json","size":507}},"invocationImages":[{"image":"my.registry/namespace/my-app@sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341","imageType":"docker","mediaType":"application/vnd.docker.distribution.manifest.v2+json","size":506}],"keywords":["keyword1","keyword2"],"maintainers":[{"email":"docker@docker.com","name":"docker","url":"docker.com"}],"name":"my-app","parameters":{"param1":{"definition":"param1Type","destination":{"env":"env_var","path":"/some/path"}}},"schemaVersion":"v1.0.0-WD","version":"0.1.0"} + //{"actions":{"action-1":{"modifies":true}},"credentials":{"cred-1":{"env":"env-var","path":"/some/path"}},"custom":{"my-key":"my-value"},"definitions":{"output1Type":{"type":"string"},"param1Type":{"default":"hello","enum":["value1",true,1],"type":["string","boolean","number"]}},"description":"description","images":{"another-image":{"contentDigest":"sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0342","description":"","image":"my.registry/namespace/another-image","imageType":"oci","mediaType":"application/vnd.oci.image.manifest.v1+json","size":507},"image-1":{"contentDigest":"sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341","description":"","image":"my.registry/namespace/image-1","imageType":"oci","mediaType":"application/vnd.oci.image.manifest.v1+json","size":507}},"invocationImages":[{"contentDigest":"sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343","image":"my.registry/namespace/my-app-invoc","imageType":"docker","mediaType":"application/vnd.docker.distribution.manifest.v2+json","size":506}],"keywords":["keyword1","keyword2"],"maintainers":[{"email":"docker@docker.com","name":"docker","url":"docker.com"}],"name":"my-app","outputs":{"output1":{"applyTo":["install"],"definition":"output1Type","description":"magic","path":"/cnab/app/outputs/magic"}},"parameters":{"param1":{"definition":"param1Type","destination":{"env":"env_var","path":"/some/path"}}},"schemaVersion":"v1.0.0-WD","version":"0.1.0"} + //{"my.registry/namespace/image-1":"my.registry/namespace/my-app@sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341","my.registry/namespace/my-app-invoc":"my.registry/namespace/my-app@sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341"} } const ( @@ -137,55 +144,21 @@ const ( }, "layers": null }` - - bufBundleConfig = `{ - "schema_version": "v1.0.0-WD", - "actions": { - "action-1": { - "modifies": true - } - }, - "definitions": { - "param1Type": { - "default": "hello", - "enum": [ - "value1", - true, - 1 - ], - "type": [ - "string", - "boolean", - "number" - ] - } - }, - "parameters": { - "param1": { - "definition": "param1Type", - "destination": { - "path": "/some/path", - "env": "env_var" - } - } - }, - "credentials": { - "cred-1": { - "path": "/some/path", - "env": "env-var" - } - } -}` ) func createExampleResolver() *mockResolver { + b := tests.MakeTestBundle() + bufBundleConfig, err := json.Marshal(b) + if err != nil { + panic(err) + } buf := []*bytes.Buffer{ // Bundle index bytes.NewBuffer([]byte(bufBundleManifest)), // Bundle config manifest bytes.NewBuffer([]byte(bundleConfigManifestDescriptor)), // Bundle config - bytes.NewBuffer([]byte(bufBundleConfig)), + bytes.NewBuffer(bufBundleConfig), } fetcher := &mockFetcher{indexBuffers: buf} pusher := &mockPusher{} diff --git a/remotes/push.go b/remotes/push.go index da6cce6..d72c434 100644 --- a/remotes/push.go +++ b/remotes/push.go @@ -11,6 +11,7 @@ import ( "github.com/containerd/containerd/remotes" "github.com/deislabs/cnab-go/bundle" "github.com/docker/cnab-to-oci/converter" + "github.com/docker/cnab-to-oci/relocation" "github.com/docker/distribution/reference" "github.com/opencontainers/go-digest" ocischemav1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -21,7 +22,13 @@ import ( type ManifestOption func(*ocischemav1.Index) error // Push pushes a bundle as an OCI Image Index manifest -func Push(ctx context.Context, b *bundle.Bundle, ref reference.Named, resolver remotes.Resolver, allowFallbacks bool, options ...ManifestOption) (ocischemav1.Descriptor, error) { +func Push(ctx context.Context, + b *bundle.Bundle, + relocationMap relocation.ImageRelocationMap, + ref reference.Named, + resolver remotes.Resolver, + allowFallbacks bool, + options ...ManifestOption) (ocischemav1.Descriptor, error) { log.G(ctx).Debugf("Pushing CNAB Bundle %s", ref) confManifestDescriptor, err := pushConfig(ctx, b, ref, resolver, allowFallbacks) @@ -29,7 +36,7 @@ func Push(ctx context.Context, b *bundle.Bundle, ref reference.Named, resolver r return ocischemav1.Descriptor{}, err } - indexDescriptor, err := pushIndex(ctx, b, ref, resolver, allowFallbacks, confManifestDescriptor, options...) + indexDescriptor, err := pushIndex(ctx, b, relocationMap, ref, resolver, allowFallbacks, confManifestDescriptor, options...) if err != nil { return ocischemav1.Descriptor{}, err } @@ -46,7 +53,7 @@ func pushConfig(ctx context.Context, logger := log.G(ctx) logger.Debugf("Pushing CNAB Bundle Config") - bundleConfig, err := converter.CreateBundleConfig(b).PrepareForPush() + bundleConfig, err := converter.PrepareForPush(b) if err != nil { return ocischemav1.Descriptor{}, err } @@ -59,12 +66,12 @@ func pushConfig(ctx context.Context, return confManifestDescriptor, nil } -func pushIndex(ctx context.Context, b *bundle.Bundle, ref reference.Named, resolver remotes.Resolver, allowFallbacks bool, +func pushIndex(ctx context.Context, b *bundle.Bundle, relocationMap relocation.ImageRelocationMap, ref reference.Named, resolver remotes.Resolver, allowFallbacks bool, confManifestDescriptor ocischemav1.Descriptor, options ...ManifestOption) (ocischemav1.Descriptor, error) { logger := log.G(ctx) logger.Debug("Pushing CNAB Index") - indexDescriptor, indexPayload, err := prepareIndex(b, ref, confManifestDescriptor, options...) + indexDescriptor, indexPayload, err := prepareIndex(b, relocationMap, ref, confManifestDescriptor, options...) if err != nil { return ocischemav1.Descriptor{}, err } @@ -81,18 +88,18 @@ func pushIndex(ctx context.Context, b *bundle.Bundle, ref reference.Named, resol } logger.Debugf("Unable to push OCI Index: %v", err) // retry with a docker manifestlist - return pushDockerManifestList(ctx, b, ref, resolver, confManifestDescriptor, options...) + return pushDockerManifestList(ctx, b, relocationMap, ref, resolver, confManifestDescriptor, options...) } logger.Debugf("CNAB Index pushed") return indexDescriptor, nil } -func pushDockerManifestList(ctx context.Context, b *bundle.Bundle, ref reference.Named, resolver remotes.Resolver, +func pushDockerManifestList(ctx context.Context, b *bundle.Bundle, relocationMap relocation.ImageRelocationMap, ref reference.Named, resolver remotes.Resolver, confManifestDescriptor ocischemav1.Descriptor, options ...ManifestOption) (ocischemav1.Descriptor, error) { logger := log.G(ctx) - indexDescriptor, indexPayload, err := prepareIndexNonOCI(b, ref, confManifestDescriptor, options...) + indexDescriptor, indexPayload, err := prepareIndexNonOCI(b, relocationMap, ref, confManifestDescriptor, options...) if err != nil { return ocischemav1.Descriptor{}, err } @@ -101,15 +108,21 @@ func pushDockerManifestList(ctx context.Context, b *bundle.Bundle, ref reference logger.Debug("Manifest list Descriptor") logPayload(logger, indexDescriptor) - if err := pushPayload(ctx, resolver, ref.String(), indexDescriptor, indexPayload); err != nil { - logger.Debugf("Unable to push Index with Manifest list: %v", err) + if err := pushPayload(ctx, + resolver, ref.String(), + indexDescriptor, + indexPayload); err != nil { return ocischemav1.Descriptor{}, err } return indexDescriptor, nil } -func prepareIndex(b *bundle.Bundle, ref reference.Named, confDescriptor ocischemav1.Descriptor, options ...ManifestOption) (ocischemav1.Descriptor, []byte, error) { - ix, err := convertIndexAndApplyOptions(b, ref, confDescriptor, options...) +func prepareIndex(b *bundle.Bundle, + relocationMap relocation.ImageRelocationMap, + ref reference.Named, + confDescriptor ocischemav1.Descriptor, + options ...ManifestOption) (ocischemav1.Descriptor, []byte, error) { + ix, err := convertIndexAndApplyOptions(b, relocationMap, ref, confDescriptor, options...) if err != nil { return ocischemav1.Descriptor{}, nil, err } @@ -130,8 +143,12 @@ type ociIndexWrapper struct { MediaType string `json:"mediaType,omitempty"` } -func convertIndexAndApplyOptions(b *bundle.Bundle, ref reference.Named, confDescriptor ocischemav1.Descriptor, options ...ManifestOption) (*ocischemav1.Index, error) { - ix, err := converter.ConvertBundleToOCIIndex(b, ref, confDescriptor) +func convertIndexAndApplyOptions(b *bundle.Bundle, + relocationMap relocation.ImageRelocationMap, + ref reference.Named, + confDescriptor ocischemav1.Descriptor, + options ...ManifestOption) (*ocischemav1.Index, error) { + ix, err := converter.ConvertBundleToOCIIndex(b, ref, confDescriptor, relocationMap) if err != nil { return nil, err } @@ -143,8 +160,12 @@ func convertIndexAndApplyOptions(b *bundle.Bundle, ref reference.Named, confDesc return ix, nil } -func prepareIndexNonOCI(b *bundle.Bundle, ref reference.Named, confDescriptor ocischemav1.Descriptor, options ...ManifestOption) (ocischemav1.Descriptor, []byte, error) { - ix, err := convertIndexAndApplyOptions(b, ref, confDescriptor, options...) +func prepareIndexNonOCI(b *bundle.Bundle, + relocationMap relocation.ImageRelocationMap, + ref reference.Named, + confDescriptor ocischemav1.Descriptor, + options ...ManifestOption) (ocischemav1.Descriptor, []byte, error) { + ix, err := convertIndexAndApplyOptions(b, relocationMap, ref, confDescriptor, options...) if err != nil { return ocischemav1.Descriptor{}, nil, err } diff --git a/remotes/push_test.go b/remotes/push_test.go index 4c8343c..39cd051 100644 --- a/remotes/push_test.go +++ b/remotes/push_test.go @@ -2,7 +2,6 @@ package remotes import ( "context" - "encoding/json" "errors" "fmt" "strings" @@ -12,80 +11,26 @@ import ( "github.com/docker/cnab-to-oci/converter" "github.com/docker/cnab-to-oci/tests" "github.com/docker/distribution/reference" + "github.com/docker/go/canonical/json" ocischemav1 "github.com/opencontainers/image-spec/specs-go/v1" "gotest.tools/assert" ) const ( - expectedBundleConfig = `{ - "schemaVersion": "v1.0.0-WD", - "actions": { - "action-1": { - "modifies": true - } - }, - "definitions": { - "output1Type": { - "type": "string" - }, - "param1Type": { - "default": "hello", - "enum": [ - "value1", - true, - 1 - ], - "type": [ - "string", - "boolean", - "number" - ] - } - }, - "parameters": { - "param1": { - "definition": "param1Type", - "destination": { - "path": "/some/path", - "env": "env_var" - } - } - }, - "outputs": { - "output1": { - "definition": "output1Type", - "applyTo": [ - "install" - ], - "description": "magic", - "path": "/cnab/app/outputs/magic" - } - }, - "credentials": { - "cred-1": { - "path": "/some/path", - "env": "env-var" - } - }, - "custom": { - "my-key": "my-value" - } -}` - expectedBundleManifest = `{ "schemaVersion": 2, "manifests": [ { "mediaType":"application/vnd.oci.image.manifest.v1+json", - "digest":"sha256:75b3dd7d430a5c5f20908dcb63099adedd555850735dbae833ab3312c6e42208", - "size":188, + "digest":"sha256:519a4617071a6c4dc66df2bbd988a34226113933de963b5b75704d6f63b2970a", + "size":189, "annotations":{ "io.cnab.manifest.type":"config" } }, { "mediaType": "application/vnd.docker.distribution.manifest.v2+json", - "digest": "sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341", + "digest": "sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343", "size": 506, "annotations": { "io.cnab.manifest.type": "invocation" @@ -93,7 +38,7 @@ const ( }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", - "digest": "sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341", + "digest": "sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0342", "size": 507, "annotations": { "io.cnab.component.name": "another-image", @@ -125,14 +70,14 @@ const ( "mediaType": "application/vnd.docker.distribution.manifest.v2+json", "config": { "mediaType": "application/vnd.docker.container.image.v1+json", - "size": 539, - "digest": "sha256:583d58ecba680e28cd4f55fa673d377915259dfb5a5a09b79f4196e53517495f" + "size": 1487, + "digest": "sha256:0391c78ee4a5dfaacc7426078ff37fa53d9f64370c76e1716d94ddcf893a106d" }, "layers": [ { "mediaType": "application/vnd.docker.container.image.v1+json", - "size": 539, - "digest": "sha256:583d58ecba680e28cd4f55fa673d377915259dfb5a5a09b79f4196e53517495f" + "size": 1487, + "digest": "sha256:0391c78ee4a5dfaacc7426078ff37fa53d9f64370c76e1716d94ddcf893a106d" } ] }` @@ -142,11 +87,13 @@ func TestPush(t *testing.T) { pusher := &mockPusher{} resolver := &mockResolver{pusher: pusher} b := tests.MakeTestBundle() + expectedBundleConfig, err := json.MarshalCanonical(b) + assert.NilError(t, err) ref, err := reference.ParseNamed("my.registry/namespace/my-app:my-tag") assert.NilError(t, err) // push the bundle - _, err = Push(context.Background(), b, ref, resolver, true) + _, err = Push(context.Background(), b, tests.MakeRelocationMap(), ref, resolver, true) assert.NilError(t, err) assert.Equal(t, len(resolver.pushedReferences), 3) assert.Equal(t, len(pusher.pushedDescriptors), 3) @@ -155,7 +102,7 @@ func TestPush(t *testing.T) { // check pushed config assert.Equal(t, "my.registry/namespace/my-app", resolver.pushedReferences[0]) assert.Equal(t, converter.CNABConfigMediaType, pusher.pushedDescriptors[0].MediaType) - assert.Equal(t, oneLiner(expectedBundleConfig), pusher.buffers[0].String()) + assert.Equal(t, oneLiner(string(expectedBundleConfig)), pusher.buffers[0].String()) // check pushed config manifest assert.Equal(t, "my.registry/namespace/my-app", resolver.pushedReferences[1]) @@ -185,7 +132,8 @@ func TestFallbackConfigManifest(t *testing.T) { assert.NilError(t, err) // push the bundle - _, err = Push(context.Background(), b, ref, resolver, true) + relocationMap := tests.MakeRelocationMap() + _, err = Push(context.Background(), b, relocationMap, ref, resolver, true) assert.NilError(t, err) assert.Equal(t, expectedConfigManifest, pusher.buffers[3].String()) } @@ -203,7 +151,7 @@ func ExamplePush() { } // Push the bundle here - descriptor, err := Push(context.Background(), b, ref, resolver, true) + descriptor, err := Push(context.Background(), b, tests.MakeRelocationMap(), ref, resolver, true) if err != nil { panic(err) } @@ -218,7 +166,7 @@ func ExamplePush() { // Output: // { // "mediaType": "application/vnd.oci.image.index.v1+json", - // "digest": "sha256:ad9bf48bfc84342aae1017a486722b7b22c82a5f31bb2c4f6da81255e5aa09b5", + // "digest": "sha256:df2c5a3ef8b04f87439f22fc5179326a8e2e84ca94e3e9eac630952ef711b6ae", // "size": 1363 // } } diff --git a/tests/helpers.go b/tests/helpers.go index d669a99..7daae39 100644 --- a/tests/helpers.go +++ b/tests/helpers.go @@ -3,6 +3,7 @@ package tests import ( "github.com/deislabs/cnab-go/bundle" "github.com/deislabs/cnab-go/bundle/definition" + "github.com/docker/cnab-to-oci/relocation" "github.com/docker/distribution/manifest/schema2" ocischema "github.com/opencontainers/image-spec/specs-go" ocischemav1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -29,28 +30,31 @@ func MakeTestBundle() *bundle.Bundle { Images: map[string]bundle.Image{ "image-1": { BaseImage: bundle.BaseImage{ - Image: "my.registry/namespace/my-app@sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341", + Image: "my.registry/namespace/image-1", ImageType: "oci", MediaType: "application/vnd.oci.image.manifest.v1+json", Size: 507, + Digest: "sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341", }, }, "another-image": { BaseImage: bundle.BaseImage{ - Image: "my.registry/namespace/my-app@sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341", + Image: "my.registry/namespace/another-image", ImageType: "oci", MediaType: "application/vnd.oci.image.manifest.v1+json", Size: 507, + Digest: "sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0342", }, }, }, InvocationImages: []bundle.InvocationImage{ { BaseImage: bundle.BaseImage{ - Image: "my.registry/namespace/my-app@sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341", + Image: "my.registry/namespace/my-app-invoc", ImageType: "docker", MediaType: "application/vnd.docker.distribution.manifest.v2+json", Size: 506, + Digest: "sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343", }, }, }, @@ -122,7 +126,7 @@ func MakeTestOCIIndex() *ocischemav1.Index { }, }, { - Digest: "sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341", + Digest: "sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343", MediaType: "application/vnd.docker.distribution.manifest.v2+json", Size: 506, Annotations: map[string]string{ @@ -130,7 +134,7 @@ func MakeTestOCIIndex() *ocischemav1.Index { }, }, { - Digest: "sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341", + Digest: "sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0342", MediaType: "application/vnd.oci.image.manifest.v1+json", Size: 507, Annotations: map[string]string{ @@ -150,3 +154,12 @@ func MakeTestOCIIndex() *ocischemav1.Index { }, } } + +// MakeRelocationMap generates a fake relocation map +func MakeRelocationMap() relocation.ImageRelocationMap { + return relocation.ImageRelocationMap{ + "my.registry/namespace/image-1": "my.registry/namespace/my-app@sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341", + "my.registry/namespace/another-image": "my.registry/namespace/my-app@sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0342", + "my.registry/namespace/my-app-invoc": "my.registry/namespace/my-app@sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343", + } +}