From 69b639b4d4282b78fb2a987fba7c14993027aa50 Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Tue, 3 Dec 2019 15:58:01 -0500 Subject: [PATCH] Fix bug in entrypoint lookup for steps by digest Before this change, any resolved digests would be simply appended to the input image name, which would result in invalid duplicate digests when the input was specified by digest. This change uses ref.Context() instead, which omits the digest (or tag), and appends the resolved digest to the end of that. This also slightly refactors entrypoint_lookup_impl.go to make it slightly easier to test, and adds a simple unit test for this specific error condition. More tests will be included in future changes. --- Gopkg.lock | 7 +- examples/taskruns/step-by-digest.yaml | 11 + pkg/pod/entrypoint_lookup_impl.go | 13 +- pkg/pod/entrypoint_lookup_impl_test.go | 62 ++ .../go-containerregistry/pkg/v1/mutate/doc.go | 16 + .../pkg/v1/mutate/mutate.go | 559 ++++++++++++++++++ .../pkg/v1/mutate/rebase.go | 97 +++ .../pkg/v1/tarball/doc.go | 17 + .../pkg/v1/tarball/image.go | 340 +++++++++++ .../pkg/v1/tarball/layer.go | 157 +++++ .../pkg/v1/tarball/write.go | 194 ++++++ 11 files changed, 1468 insertions(+), 5 deletions(-) create mode 100644 examples/taskruns/step-by-digest.yaml create mode 100644 pkg/pod/entrypoint_lookup_impl_test.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/mutate/doc.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/mutate/rebase.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/tarball/doc.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go diff --git a/Gopkg.lock b/Gopkg.lock index a8e7111b83f..785611146f4 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -267,7 +267,7 @@ version = "v0.3.0" [[projects]] - digest = "1:729f263f2e42c3dddf2de8f62b7d1236560346d89b546f6ac87fdf1a92c9547c" + digest = "1:2e9b6076a7020566594ca46f36481acc240a5652ebf1a69d0d1665180647bd7f" name = "github.com/google/go-containerregistry" packages = [ "pkg/authn", @@ -276,11 +276,13 @@ "pkg/v1", "pkg/v1/empty", "pkg/v1/layout", + "pkg/v1/mutate", "pkg/v1/partial", "pkg/v1/random", "pkg/v1/remote", "pkg/v1/remote/transport", "pkg/v1/stream", + "pkg/v1/tarball", "pkg/v1/types", "pkg/v1/v1util", ] @@ -1356,10 +1358,9 @@ "github.com/google/go-containerregistry/pkg/v1", "github.com/google/go-containerregistry/pkg/v1/empty", "github.com/google/go-containerregistry/pkg/v1/layout", - "github.com/google/go-containerregistry/pkg/v1/partial", + "github.com/google/go-containerregistry/pkg/v1/mutate", "github.com/google/go-containerregistry/pkg/v1/random", "github.com/google/go-containerregistry/pkg/v1/remote", - "github.com/google/go-containerregistry/pkg/v1/types", "github.com/hashicorp/go-multierror", "github.com/hashicorp/golang-lru", "github.com/jenkins-x/go-scm/scm", diff --git a/examples/taskruns/step-by-digest.yaml b/examples/taskruns/step-by-digest.yaml new file mode 100644 index 00000000000..c17da5ebbe7 --- /dev/null +++ b/examples/taskruns/step-by-digest.yaml @@ -0,0 +1,11 @@ +apiVersion: tekton.dev/v1alpha1 +kind: TaskRun +metadata: + generateName: step-by-digest- +spec: + taskSpec: + steps: + # Step images can be specified by digest. + - image: busybox@sha256:1303dbf110c57f3edf68d9f5a16c082ec06c4cf7604831669faf2c712260b5a0 + # NB: command is not set, so it must be looked up from the registry. + args: ['-c', 'echo hello'] diff --git a/pkg/pod/entrypoint_lookup_impl.go b/pkg/pod/entrypoint_lookup_impl.go index 2bf87d1c009..1b73aea0555 100644 --- a/pkg/pod/entrypoint_lookup_impl.go +++ b/pkg/pod/entrypoint_lookup_impl.go @@ -22,6 +22,7 @@ import ( "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/authn/k8schain" "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" lru "github.com/hashicorp/golang-lru" "k8s.io/client-go/kubernetes" @@ -75,6 +76,15 @@ func (e *entrypointCache) Get(imageName, namespace, serviceAccountName string) ( if err != nil { return nil, name.Digest{}, fmt.Errorf("Error getting image manifest: %v", err) } + ep, digest, err := imageData(ref, img) + if err != nil { + return nil, name.Digest{}, err + } + e.lru.Add(digest.String(), ep) // Populate the cache. + return ep, digest, err +} + +func imageData(ref name.Reference, img v1.Image) ([]string, name.Digest, error) { digest, err := img.Digest() if err != nil { return nil, name.Digest{}, fmt.Errorf("Error getting image digest: %v", err) @@ -87,9 +97,8 @@ func (e *entrypointCache) Get(imageName, namespace, serviceAccountName string) ( if len(ep) == 0 { ep = cfg.Config.Cmd } - e.lru.Add(digest.String(), ep) // Populate the cache. - d, err = name.NewDigest(imageName+"@"+digest.String(), name.WeakValidation) + d, err := name.NewDigest(ref.Context().String()+"@"+digest.String(), name.WeakValidation) if err != nil { return nil, name.Digest{}, fmt.Errorf("Error constructing resulting digest: %v", err) } diff --git a/pkg/pod/entrypoint_lookup_impl_test.go b/pkg/pod/entrypoint_lookup_impl_test.go new file mode 100644 index 00000000000..ee66da0a9ba --- /dev/null +++ b/pkg/pod/entrypoint_lookup_impl_test.go @@ -0,0 +1,62 @@ +/* +Copyright 2019 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pod + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" +) + +func TestImageData(t *testing.T) { + // Generate an Image with an Entrypoint configured. + img, err := random.Image(1, 1) + if err != nil { + t.Fatalf("random.Image: %v", err) + } + img, err = mutate.Config(img, v1.Config{ + Entrypoint: []string{"my", "entrypoint"}, + }) + if err != nil { + t.Fatalf("mutate.Config: %v", err) + } + // Get the generated image's digest. + dig, err := img.Digest() + if err != nil { + t.Fatalf("Digest: %v", err) + } + ref, err := name.ParseReference("ubuntu@"+dig.String(), name.WeakValidation) + if err != nil { + t.Fatalf("ParseReference(%q): %v", dig, err) + } + + // Get the image data (entrypoint and digest) for the image. + gotEP, gotDig, err := imageData(ref, img) + if err != nil { + t.Fatalf("imageData: %v", err) + } + if d := cmp.Diff([]string{"my", "entrypoint"}, gotEP); d != "" { + t.Errorf("Diff(-want, +got): %s", d) + } + if d := cmp.Diff("index.docker.io/library/ubuntu@"+dig.String(), gotDig.String()); d != "" { + t.Errorf("Diff(-want, +got): %s", d) + } +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/doc.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/doc.go new file mode 100644 index 00000000000..dfbd9951e07 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/doc.go @@ -0,0 +1,16 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package mutate provides facilities for mutating v1.Images of any kind. +package mutate diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go new file mode 100644 index 00000000000..a327e7594fd --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go @@ -0,0 +1,559 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mutate + +import ( + "archive/tar" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "path/filepath" + "strings" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/stream" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/v1util" +) + +const whiteoutPrefix = ".wh." + +// Addendum contains layers and history to be appended +// to a base image +type Addendum struct { + Layer v1.Layer + History v1.History +} + +// AppendLayers applies layers to a base image +func AppendLayers(base v1.Image, layers ...v1.Layer) (v1.Image, error) { + additions := make([]Addendum, 0, len(layers)) + for _, layer := range layers { + additions = append(additions, Addendum{Layer: layer}) + } + + return Append(base, additions...) +} + +// Append will apply the list of addendums to the base image +func Append(base v1.Image, adds ...Addendum) (v1.Image, error) { + if len(adds) == 0 { + return base, nil + } + if err := validate(adds); err != nil { + return nil, err + } + + return &image{ + base: base, + adds: adds, + }, nil +} + +// Config mutates the provided v1.Image to have the provided v1.Config +func Config(base v1.Image, cfg v1.Config) (v1.Image, error) { + cf, err := base.ConfigFile() + if err != nil { + return nil, err + } + + cf.Config = cfg + + return configFile(base, cf) +} + +func configFile(base v1.Image, cfg *v1.ConfigFile) (v1.Image, error) { + m, err := base.Manifest() + if err != nil { + return nil, err + } + + image := &image{ + base: base, + manifest: m.DeepCopy(), + configFile: cfg, + } + + return image, nil +} + +// CreatedAt mutates the provided v1.Image to have the provided v1.Time +func CreatedAt(base v1.Image, created v1.Time) (v1.Image, error) { + cf, err := base.ConfigFile() + if err != nil { + return nil, err + } + + cfg := cf.DeepCopy() + cfg.Created = created + + return configFile(base, cfg) +} + +type image struct { + base v1.Image + adds []Addendum + + computed bool + configFile *v1.ConfigFile + manifest *v1.Manifest + diffIDMap map[v1.Hash]v1.Layer + digestMap map[v1.Hash]v1.Layer +} + +var _ v1.Image = (*image)(nil) + +func (i *image) MediaType() (types.MediaType, error) { return i.base.MediaType() } + +func (i *image) compute() error { + // Don't re-compute if already computed. + if i.computed { + return nil + } + var configFile *v1.ConfigFile + if i.configFile != nil { + configFile = i.configFile + } else { + cf, err := i.base.ConfigFile() + if err != nil { + return err + } + configFile = cf.DeepCopy() + } + diffIDs := configFile.RootFS.DiffIDs + history := configFile.History + + diffIDMap := make(map[v1.Hash]v1.Layer) + digestMap := make(map[v1.Hash]v1.Layer) + + for _, add := range i.adds { + diffID, err := add.Layer.DiffID() + if err != nil { + return err + } + diffIDs = append(diffIDs, diffID) + history = append(history, add.History) + diffIDMap[diffID] = add.Layer + } + + m, err := i.base.Manifest() + if err != nil { + return err + } + manifest := m.DeepCopy() + manifestLayers := manifest.Layers + for _, add := range i.adds { + d := v1.Descriptor{ + MediaType: types.DockerLayer, + } + + var err error + if d.Size, err = add.Layer.Size(); err != nil { + return err + } + + if d.Digest, err = add.Layer.Digest(); err != nil { + return err + } + + manifestLayers = append(manifestLayers, d) + digestMap[d.Digest] = add.Layer + } + + configFile.RootFS.DiffIDs = diffIDs + configFile.History = history + + manifest.Layers = manifestLayers + + rcfg, err := json.Marshal(configFile) + if err != nil { + return err + } + d, sz, err := v1.SHA256(bytes.NewBuffer(rcfg)) + if err != nil { + return err + } + manifest.Config.Digest = d + manifest.Config.Size = sz + + i.configFile = configFile + i.manifest = manifest + i.diffIDMap = diffIDMap + i.digestMap = digestMap + i.computed = true + return nil +} + +// Layers returns the ordered collection of filesystem layers that comprise this image. +// The order of the list is oldest/base layer first, and most-recent/top layer last. +func (i *image) Layers() ([]v1.Layer, error) { + if err := i.compute(); err == stream.ErrNotComputed { + // Image contains a streamable layer which has not yet been + // consumed. Just return the layers we have in case the caller + // is going to consume the layers. + layers, err := i.base.Layers() + if err != nil { + return nil, err + } + for _, add := range i.adds { + layers = append(layers, add.Layer) + } + return layers, nil + } else if err != nil { + return nil, err + } + + diffIDs, err := partial.DiffIDs(i) + if err != nil { + return nil, err + } + ls := make([]v1.Layer, 0, len(diffIDs)) + for _, h := range diffIDs { + l, err := i.LayerByDiffID(h) + if err != nil { + return nil, err + } + ls = append(ls, l) + } + return ls, nil +} + +// ConfigName returns the hash of the image's config file. +func (i *image) ConfigName() (v1.Hash, error) { + if err := i.compute(); err != nil { + return v1.Hash{}, err + } + return partial.ConfigName(i) +} + +// ConfigFile returns this image's config file. +func (i *image) ConfigFile() (*v1.ConfigFile, error) { + if err := i.compute(); err != nil { + return nil, err + } + return i.configFile, nil +} + +// RawConfigFile returns the serialized bytes of ConfigFile() +func (i *image) RawConfigFile() ([]byte, error) { + if err := i.compute(); err != nil { + return nil, err + } + return json.Marshal(i.configFile) +} + +// Digest returns the sha256 of this image's manifest. +func (i *image) Digest() (v1.Hash, error) { + if err := i.compute(); err != nil { + return v1.Hash{}, err + } + return partial.Digest(i) +} + +// Manifest returns this image's Manifest object. +func (i *image) Manifest() (*v1.Manifest, error) { + if err := i.compute(); err != nil { + return nil, err + } + return i.manifest, nil +} + +// RawManifest returns the serialized bytes of Manifest() +func (i *image) RawManifest() ([]byte, error) { + if err := i.compute(); err != nil { + return nil, err + } + return json.Marshal(i.manifest) +} + +// LayerByDigest returns a Layer for interacting with a particular layer of +// the image, looking it up by "digest" (the compressed hash). +func (i *image) LayerByDigest(h v1.Hash) (v1.Layer, error) { + if cn, err := i.ConfigName(); err != nil { + return nil, err + } else if h == cn { + return partial.ConfigLayer(i) + } + if layer, ok := i.digestMap[h]; ok { + return layer, nil + } + return i.base.LayerByDigest(h) +} + +// LayerByDiffID is an analog to LayerByDigest, looking up by "diff id" +// (the uncompressed hash). +func (i *image) LayerByDiffID(h v1.Hash) (v1.Layer, error) { + if layer, ok := i.diffIDMap[h]; ok { + return layer, nil + } + return i.base.LayerByDiffID(h) +} + +func validate(adds []Addendum) error { + for _, add := range adds { + if add.Layer == nil { + return errors.New("Unable to add a nil layer to the image") + } + } + return nil +} + +// Extract takes an image and returns an io.ReadCloser containing the image's +// flattened filesystem. +// +// Callers can read the filesystem contents by passing the reader to +// tar.NewReader, or io.Copy it directly to some output. +// +// If a caller doesn't read the full contents, they should Close it to free up +// resources used during extraction. +// +// Adapted from https://github.com/google/containerregistry/blob/master/client/v2_2/docker_image_.py#L731 +func Extract(img v1.Image) io.ReadCloser { + pr, pw := io.Pipe() + + go func() { + // Close the writer with any errors encountered during + // extraction. These errors will be returned by the reader end + // on subsequent reads. If err == nil, the reader will return + // EOF. + pw.CloseWithError(extract(img, pw)) + }() + + return pr +} + +func extract(img v1.Image, w io.Writer) error { + tarWriter := tar.NewWriter(w) + defer tarWriter.Close() + + fileMap := map[string]bool{} + + layers, err := img.Layers() + if err != nil { + return fmt.Errorf("retrieving image layers: %v", err) + } + // we iterate through the layers in reverse order because it makes handling + // whiteout layers more efficient, since we can just keep track of the removed + // files as we see .wh. layers and ignore those in previous layers. + for i := len(layers) - 1; i >= 0; i-- { + layer := layers[i] + layerReader, err := layer.Uncompressed() + if err != nil { + return fmt.Errorf("reading layer contents: %v", err) + } + tarReader := tar.NewReader(layerReader) + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("reading tar: %v", err) + } + + basename := filepath.Base(header.Name) + dirname := filepath.Dir(header.Name) + tombstone := strings.HasPrefix(basename, whiteoutPrefix) + if tombstone { + basename = basename[len(whiteoutPrefix):] + } + + // check if we have seen value before + // if we're checking a directory, don't filepath.Join names + var name string + if header.Typeflag == tar.TypeDir { + name = header.Name + } else { + name = filepath.Join(dirname, basename) + } + + if _, ok := fileMap[name]; ok { + continue + } + + // check for a whited out parent directory + if inWhiteoutDir(fileMap, name) { + continue + } + + // mark file as handled. non-directory implicitly tombstones + // any entries with a matching (or child) name + fileMap[name] = tombstone || !(header.Typeflag == tar.TypeDir) + if !tombstone { + tarWriter.WriteHeader(header) + if header.Size > 0 { + if _, err := io.Copy(tarWriter, tarReader); err != nil { + return err + } + } + } + } + } + return nil +} + +func inWhiteoutDir(fileMap map[string]bool, file string) bool { + for { + if file == "" { + break + } + dirname := filepath.Dir(file) + if file == dirname { + break + } + if val, ok := fileMap[dirname]; ok && val { + return true + } + file = dirname + } + return false +} + +// Time sets all timestamps in an image to the given timestamp. +func Time(img v1.Image, t time.Time) (v1.Image, error) { + newImage := empty.Image + + layers, err := img.Layers() + if err != nil { + + return nil, fmt.Errorf("Error getting image layers: %v", err) + } + + // Strip away all timestamps from layers + var newLayers []v1.Layer + for _, layer := range layers { + newLayer, err := layerTime(layer, t) + if err != nil { + return nil, fmt.Errorf("Error setting layer times: %v", err) + } + newLayers = append(newLayers, newLayer) + } + + newImage, err = AppendLayers(newImage, newLayers...) + if err != nil { + return nil, fmt.Errorf("Error appending layers: %v", err) + } + + ocf, err := img.ConfigFile() + if err != nil { + return nil, fmt.Errorf("Error getting original config file: %v", err) + } + + cf, err := newImage.ConfigFile() + if err != nil { + return nil, fmt.Errorf("Error setting config file: %v", err) + } + + cfg := cf.DeepCopy() + + // Copy basic config over + cfg.Config = ocf.Config + cfg.ContainerConfig = ocf.ContainerConfig + + // Strip away timestamps from the config file + cfg.Created = v1.Time{Time: t} + + for _, h := range cfg.History { + h.Created = v1.Time{Time: t} + } + + return configFile(newImage, cfg) +} + +func layerTime(layer v1.Layer, t time.Time) (v1.Layer, error) { + layerReader, err := layer.Uncompressed() + if err != nil { + return nil, fmt.Errorf("Error getting layer: %v", err) + } + w := new(bytes.Buffer) + tarWriter := tar.NewWriter(w) + defer tarWriter.Close() + + tarReader := tar.NewReader(layerReader) + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("Error reading layer: %v", err) + } + + header.ModTime = t + if err := tarWriter.WriteHeader(header); err != nil { + return nil, fmt.Errorf("Error writing tar header: %v", err) + } + + if header.Typeflag == tar.TypeReg { + if _, err = io.Copy(tarWriter, tarReader); err != nil { + return nil, fmt.Errorf("Error writing layer file: %v", err) + } + } + } + + if err := tarWriter.Close(); err != nil { + return nil, err + } + + b := w.Bytes() + // gzip the contents, then create the layer + opener := func() (io.ReadCloser, error) { + g, err := v1util.GzipReadCloser(ioutil.NopCloser(bytes.NewReader(b))) + if err != nil { + return nil, fmt.Errorf("Error compressing layer: %v", err) + } + + return g, nil + } + layer, err = tarball.LayerFromOpener(opener) + if err != nil { + return nil, fmt.Errorf("Error creating layer: %v", err) + } + + return layer, nil +} + +// Canonical is a helper function to combine Time and configFile +// to remove any randomness during a docker build. +func Canonical(img v1.Image) (v1.Image, error) { + // Set all timestamps to 0 + created := time.Time{} + img, err := Time(img, created) + if err != nil { + return nil, err + } + + cf, err := img.ConfigFile() + if err != nil { + return nil, err + } + + // Get rid of host-dependent random config + cfg := cf.DeepCopy() + + cfg.Container = "" + cfg.Config.Hostname = "" + cfg.ContainerConfig.Hostname = "" + cfg.DockerVersion = "" + + return configFile(img, cfg) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/rebase.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/rebase.go new file mode 100644 index 00000000000..6d1fe8d51e9 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/rebase.go @@ -0,0 +1,97 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mutate + +import ( + "fmt" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" +) + +// Rebase returns a new v1.Image where the oldBase in orig is replaced by newBase. +func Rebase(orig, oldBase, newBase v1.Image) (v1.Image, error) { + // Verify that oldBase's layers are present in orig, otherwise orig is + // not based on oldBase at all. + origLayers, err := orig.Layers() + if err != nil { + return nil, fmt.Errorf("failed to get layers for original: %v", err) + } + oldBaseLayers, err := oldBase.Layers() + if err != nil { + return nil, err + } + if len(oldBaseLayers) > len(origLayers) { + return nil, fmt.Errorf("image %q is not based on %q (too few layers)", orig, oldBase) + } + for i, l := range oldBaseLayers { + oldLayerDigest, err := l.Digest() + if err != nil { + return nil, fmt.Errorf("failed to get digest of layer %d of %q: %v", i, oldBase, err) + } + origLayerDigest, err := origLayers[i].Digest() + if err != nil { + return nil, fmt.Errorf("failed to get digest of layer %d of %q: %v", i, orig, err) + } + if oldLayerDigest != origLayerDigest { + return nil, fmt.Errorf("image %q is not based on %q (layer %d mismatch)", orig, oldBase, i) + } + } + + origConfig, err := orig.ConfigFile() + if err != nil { + return nil, fmt.Errorf("failed to get config for original: %v", err) + } + + // Stitch together an image that contains: + // - original image's config + // - new base image's layers + top of original image's layers + // - new base image's history + top of original image's history + rebasedImage, err := Config(empty.Image, *origConfig.Config.DeepCopy()) + if err != nil { + return nil, fmt.Errorf("failed to create empty image with original config: %v", err) + } + // Get new base layers and config for history. + newBaseLayers, err := newBase.Layers() + if err != nil { + return nil, fmt.Errorf("could not get new base layers for new base: %v", err) + } + newConfig, err := newBase.ConfigFile() + if err != nil { + return nil, fmt.Errorf("could not get config for new base: %v", err) + } + // Add new base layers. + for i := range newBaseLayers { + rebasedImage, err = Append(rebasedImage, Addendum{ + Layer: newBaseLayers[i], + History: newConfig.History[i], + }) + if err != nil { + return nil, fmt.Errorf("failed to append layer %d of new base layers", i) + } + } + // Add original layers above the old base. + start := len(oldBaseLayers) + for i := range origLayers[start:] { + rebasedImage, err = Append(rebasedImage, Addendum{ + Layer: origLayers[start+i], + History: origConfig.History[start+i], + }) + if err != nil { + return nil, fmt.Errorf("failed to append layer %d of original layers", i) + } + } + return rebasedImage, nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/doc.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/doc.go new file mode 100644 index 00000000000..4eb79bb4e5a --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/doc.go @@ -0,0 +1,17 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tarball provides facilities for reading/writing v1.Images from/to +// a tarball on-disk. +package tarball diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go new file mode 100644 index 00000000000..ced18735c85 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go @@ -0,0 +1,340 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarball + +import ( + "archive/tar" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "sync" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/v1util" +) + +type image struct { + opener Opener + td *tarDescriptor + config []byte + imgDescriptor *singleImageTarDescriptor + + tag *name.Tag +} + +type uncompressedImage struct { + *image +} + +type compressedImage struct { + *image + manifestLock sync.Mutex // Protects manifest + manifest *v1.Manifest +} + +var _ partial.UncompressedImageCore = (*uncompressedImage)(nil) +var _ partial.CompressedImageCore = (*compressedImage)(nil) + +// Opener is a thunk for opening a tar file. +type Opener func() (io.ReadCloser, error) + +func pathOpener(path string) Opener { + return func() (io.ReadCloser, error) { + return os.Open(path) + } +} + +// ImageFromPath returns a v1.Image from a tarball located on path. +func ImageFromPath(path string, tag *name.Tag) (v1.Image, error) { + return Image(pathOpener(path), tag) +} + +// Image exposes an image from the tarball at the provided path. +func Image(opener Opener, tag *name.Tag) (v1.Image, error) { + img := &image{ + opener: opener, + tag: tag, + } + if err := img.loadTarDescriptorAndConfig(); err != nil { + return nil, err + } + + // Peek at the first layer and see if it's compressed. + compressed, err := img.areLayersCompressed() + if err != nil { + return nil, err + } + if compressed { + c := compressedImage{ + image: img, + } + return partial.CompressedToImage(&c) + } + + uc := uncompressedImage{ + image: img, + } + return partial.UncompressedToImage(&uc) +} + +func (i *image) MediaType() (types.MediaType, error) { + return types.DockerManifestSchema2, nil +} + +// singleImageTarDescriptor is the struct used to represent a single image inside a `docker save` tarball. +type singleImageTarDescriptor struct { + Config string + RepoTags []string + Layers []string +} + +// tarDescriptor is the struct used inside the `manifest.json` file of a `docker save` tarball. +type tarDescriptor []singleImageTarDescriptor + +func (td tarDescriptor) findSpecifiedImageDescriptor(tag *name.Tag) (*singleImageTarDescriptor, error) { + if tag == nil { + if len(td) != 1 { + return nil, errors.New("tarball must contain only a single image to be used with tarball.Image") + } + return &(td)[0], nil + } + for _, img := range td { + for _, tagStr := range img.RepoTags { + repoTag, err := name.NewTag(tagStr, name.WeakValidation) + if err != nil { + return nil, err + } + + // Compare the resolved names, since there are several ways to specify the same tag. + if repoTag.Name() == tag.Name() { + return &img, nil + } + } + } + return nil, fmt.Errorf("tag %s not found in tarball", tag) +} + +func (i *image) areLayersCompressed() (bool, error) { + if len(i.imgDescriptor.Layers) == 0 { + return false, errors.New("0 layers found in image") + } + layer := i.imgDescriptor.Layers[0] + blob, err := extractFileFromTar(i.opener, layer) + if err != nil { + return false, err + } + defer blob.Close() + return v1util.IsGzipped(blob) +} + +func (i *image) loadTarDescriptorAndConfig() error { + td, err := extractFileFromTar(i.opener, "manifest.json") + if err != nil { + return err + } + defer td.Close() + + if err := json.NewDecoder(td).Decode(&i.td); err != nil { + return err + } + + i.imgDescriptor, err = i.td.findSpecifiedImageDescriptor(i.tag) + if err != nil { + return err + } + + cfg, err := extractFileFromTar(i.opener, i.imgDescriptor.Config) + if err != nil { + return err + } + defer cfg.Close() + + i.config, err = ioutil.ReadAll(cfg) + if err != nil { + return err + } + return nil +} + +func (i *image) RawConfigFile() ([]byte, error) { + return i.config, nil +} + +// tarFile represents a single file inside a tar. Closing it closes the tar itself. +type tarFile struct { + io.Reader + io.Closer +} + +func extractFileFromTar(opener Opener, filePath string) (io.ReadCloser, error) { + f, err := opener() + if err != nil { + return nil, err + } + tf := tar.NewReader(f) + for { + hdr, err := tf.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if hdr.Name == filePath { + return tarFile{ + Reader: tf, + Closer: f, + }, nil + } + } + return nil, fmt.Errorf("file %s not found in tar", filePath) +} + +// uncompressedLayerFromTarball implements partial.UncompressedLayer +type uncompressedLayerFromTarball struct { + diffID v1.Hash + opener Opener + filePath string +} + +// DiffID implements partial.UncompressedLayer +func (ulft *uncompressedLayerFromTarball) DiffID() (v1.Hash, error) { + return ulft.diffID, nil +} + +// Uncompressed implements partial.UncompressedLayer +func (ulft *uncompressedLayerFromTarball) Uncompressed() (io.ReadCloser, error) { + return extractFileFromTar(ulft.opener, ulft.filePath) +} + +func (i *uncompressedImage) LayerByDiffID(h v1.Hash) (partial.UncompressedLayer, error) { + cfg, err := partial.ConfigFile(i) + if err != nil { + return nil, err + } + for idx, diffID := range cfg.RootFS.DiffIDs { + if diffID == h { + return &uncompressedLayerFromTarball{ + diffID: diffID, + opener: i.opener, + filePath: i.imgDescriptor.Layers[idx], + }, nil + } + } + return nil, fmt.Errorf("diff id %q not found", h) +} + +func (c *compressedImage) Manifest() (*v1.Manifest, error) { + c.manifestLock.Lock() + defer c.manifestLock.Unlock() + if c.manifest != nil { + return c.manifest, nil + } + + b, err := c.RawConfigFile() + if err != nil { + return nil, err + } + + cfgHash, cfgSize, err := v1.SHA256(bytes.NewReader(b)) + if err != nil { + return nil, err + } + + c.manifest = &v1.Manifest{ + SchemaVersion: 2, + MediaType: types.DockerManifestSchema2, + Config: v1.Descriptor{ + MediaType: types.DockerConfigJSON, + Size: cfgSize, + Digest: cfgHash, + }, + } + + for _, p := range c.imgDescriptor.Layers { + l, err := extractFileFromTar(c.opener, p) + if err != nil { + return nil, err + } + defer l.Close() + sha, size, err := v1.SHA256(l) + if err != nil { + return nil, err + } + c.manifest.Layers = append(c.manifest.Layers, v1.Descriptor{ + MediaType: types.DockerLayer, + Size: size, + Digest: sha, + }) + } + return c.manifest, nil +} + +func (c *compressedImage) RawManifest() ([]byte, error) { + return partial.RawManifest(c) +} + +// compressedLayerFromTarball implements partial.CompressedLayer +type compressedLayerFromTarball struct { + digest v1.Hash + opener Opener + filePath string +} + +// Digest implements partial.CompressedLayer +func (clft *compressedLayerFromTarball) Digest() (v1.Hash, error) { + return clft.digest, nil +} + +// Compressed implements partial.CompressedLayer +func (clft *compressedLayerFromTarball) Compressed() (io.ReadCloser, error) { + return extractFileFromTar(clft.opener, clft.filePath) +} + +// Size implements partial.CompressedLayer +func (clft *compressedLayerFromTarball) Size() (int64, error) { + r, err := clft.Compressed() + if err != nil { + return -1, err + } + defer r.Close() + _, i, err := v1.SHA256(r) + return i, err +} + +func (c *compressedImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) { + m, err := c.Manifest() + if err != nil { + return nil, err + } + for i, l := range m.Layers { + if l.Digest == h { + fp := c.imgDescriptor.Layers[i] + return &compressedLayerFromTarball{ + digest: h, + opener: c.opener, + filePath: fp, + }, nil + } + } + return nil, fmt.Errorf("blob %v not found", h) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go new file mode 100644 index 00000000000..d1a7e2c8cbf --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go @@ -0,0 +1,157 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarball + +import ( + "bytes" + "compress/gzip" + "io" + "io/ioutil" + "os" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/v1util" +) + +type layer struct { + digest v1.Hash + diffID v1.Hash + size int64 + opener Opener + compressed bool +} + +func (l *layer) Digest() (v1.Hash, error) { + return l.digest, nil +} + +func (l *layer) DiffID() (v1.Hash, error) { + return l.diffID, nil +} + +func (l *layer) Compressed() (io.ReadCloser, error) { + rc, err := l.opener() + if err == nil && !l.compressed { + return v1util.GzipReadCloser(rc) + } + + return rc, err +} + +func (l *layer) Uncompressed() (io.ReadCloser, error) { + rc, err := l.opener() + if err == nil && l.compressed { + return v1util.GunzipReadCloser(rc) + } + + return rc, err +} + +func (l *layer) Size() (int64, error) { + return l.size, nil +} + +// LayerFromFile returns a v1.Layer given a tarball +func LayerFromFile(path string) (v1.Layer, error) { + opener := func() (io.ReadCloser, error) { + return os.Open(path) + } + return LayerFromOpener(opener) +} + +// LayerFromOpener returns a v1.Layer given an Opener function +func LayerFromOpener(opener Opener) (v1.Layer, error) { + rc, err := opener() + if err != nil { + return nil, err + } + defer rc.Close() + + compressed, err := v1util.IsGzipped(rc) + if err != nil { + return nil, err + } + + var digest v1.Hash + var size int64 + if digest, size, err = computeDigest(opener, compressed); err != nil { + return nil, err + } + + diffID, err := computeDiffID(opener, compressed) + if err != nil { + return nil, err + } + + return &layer{ + digest: digest, + diffID: diffID, + size: size, + compressed: compressed, + opener: opener, + }, nil +} + +// LayerFromReader returns a v1.Layer given a io.Reader. +func LayerFromReader(reader io.Reader) (v1.Layer, error) { + // Buffering due to Opener requiring multiple calls. + a, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + return LayerFromOpener(func() (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewReader(a)), nil + }) +} + +func computeDigest(opener Opener, compressed bool) (v1.Hash, int64, error) { + rc, err := opener() + if err != nil { + return v1.Hash{}, 0, err + } + defer rc.Close() + + if compressed { + return v1.SHA256(rc) + } + + reader, err := v1util.GzipReadCloser(ioutil.NopCloser(rc)) + if err != nil { + return v1.Hash{}, 0, err + } + + return v1.SHA256(reader) +} + +func computeDiffID(opener Opener, compressed bool) (v1.Hash, error) { + rc, err := opener() + if err != nil { + return v1.Hash{}, err + } + defer rc.Close() + + if !compressed { + digest, _, err := v1.SHA256(rc) + return digest, err + } + + reader, err := gzip.NewReader(rc) + if err != nil { + return v1.Hash{}, err + } + + diffID, _, err := v1.SHA256(reader) + return diffID, err +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go new file mode 100644 index 00000000000..2ee81f0b803 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go @@ -0,0 +1,194 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarball + +import ( + "archive/tar" + "bytes" + "encoding/json" + "fmt" + "io" + "os" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// WriteToFile writes in the compressed format to a tarball, on disk. +// This is just syntactic sugar wrapping tarball.Write with a new file. +func WriteToFile(p string, ref name.Reference, img v1.Image) error { + w, err := os.Create(p) + if err != nil { + return err + } + defer w.Close() + + return Write(ref, img, w) +} + +// MultiWriteToFile writes in the compressed format to a tarball, on disk. +// This is just syntactic sugar wrapping tarball.MultiWrite with a new file. +func MultiWriteToFile(p string, tagToImage map[name.Tag]v1.Image) error { + var refToImage map[name.Reference]v1.Image = make(map[name.Reference]v1.Image, len(tagToImage)) + for i, d := range tagToImage { + refToImage[i] = d + } + return MultiRefWriteToFile(p, refToImage) +} + +// MultiRefWriteToFile writes in the compressed format to a tarball, on disk. +// This is just syntactic sugar wrapping tarball.MultiRefWrite with a new file. +func MultiRefWriteToFile(p string, refToImage map[name.Reference]v1.Image) error { + w, err := os.Create(p) + if err != nil { + return err + } + defer w.Close() + + return MultiRefWrite(refToImage, w) +} + +// Write is a wrapper to write a single image and tag to a tarball. +func Write(ref name.Reference, img v1.Image, w io.Writer) error { + return MultiRefWrite(map[name.Reference]v1.Image{ref: img}, w) +} + +// MultiWrite writes the contents of each image to the provided reader, in the compressed format. +// The contents are written in the following format: +// One manifest.json file at the top level containing information about several images. +// One file for each layer, named after the layer's SHA. +// One file for the config blob, named after its SHA. +func MultiWrite(tagToImage map[name.Tag]v1.Image, w io.Writer) error { + var refToImage map[name.Reference]v1.Image = make(map[name.Reference]v1.Image, len(tagToImage)) + for i, d := range tagToImage { + refToImage[i] = d + } + return MultiRefWrite(refToImage, w) +} + +// MultiRefWrite writes the contents of each image to the provided reader, in the compressed format. +// The contents are written in the following format: +// One manifest.json file at the top level containing information about several images. +// One file for each layer, named after the layer's SHA. +// One file for the config blob, named after its SHA. +func MultiRefWrite(refToImage map[name.Reference]v1.Image, w io.Writer) error { + tf := tar.NewWriter(w) + defer tf.Close() + + imageToTags := dedupRefToImage(refToImage) + var td tarDescriptor + + for img, tags := range imageToTags { + // Write the config. + cfgName, err := img.ConfigName() + if err != nil { + return err + } + cfgBlob, err := img.RawConfigFile() + if err != nil { + return err + } + if err := writeTarEntry(tf, cfgName.String(), bytes.NewReader(cfgBlob), int64(len(cfgBlob))); err != nil { + return err + } + + // Write the layers. + layers, err := img.Layers() + if err != nil { + return err + } + layerFiles := make([]string, len(layers)) + for i, l := range layers { + d, err := l.Digest() + if err != nil { + return err + } + + // Munge the file name to appease ancient technology. + // + // tar assumes anything with a colon is a remote tape drive: + // https://www.gnu.org/software/tar/manual/html_section/tar_45.html + // Drop the algorithm prefix, e.g. "sha256:" + hex := d.Hex + + // gunzip expects certain file extensions: + // https://www.gnu.org/software/gzip/manual/html_node/Overview.html + layerFiles[i] = fmt.Sprintf("%s.tar.gz", hex) + + r, err := l.Compressed() + if err != nil { + return err + } + blobSize, err := l.Size() + if err != nil { + return err + } + + if err := writeTarEntry(tf, layerFiles[i], r, blobSize); err != nil { + return err + } + } + + // Generate the tar descriptor and write it. + sitd := singleImageTarDescriptor{ + Config: cfgName.String(), + RepoTags: tags, + Layers: layerFiles, + } + + td = append(td, sitd) + } + + tdBytes, err := json.Marshal(td) + if err != nil { + return err + } + return writeTarEntry(tf, "manifest.json", bytes.NewReader(tdBytes), int64(len(tdBytes))) +} + +func dedupRefToImage(refToImage map[name.Reference]v1.Image) map[v1.Image][]string { + imageToTags := make(map[v1.Image][]string) + + for ref, img := range refToImage { + if tag, ok := ref.(name.Tag); ok { + if tags, ok := imageToTags[img]; ok && tags != nil { + imageToTags[img] = append(tags, tag.String()) + } else { + imageToTags[img] = []string{tag.String()} + } + } else { + if _, ok := imageToTags[img]; !ok { + imageToTags[img] = nil + } + } + } + + return imageToTags +} + +// write a file to the provided writer with a corresponding tar header +func writeTarEntry(tf *tar.Writer, path string, r io.Reader, size int64) error { + hdr := &tar.Header{ + Mode: 0644, + Typeflag: tar.TypeReg, + Size: size, + Name: path, + } + if err := tf.WriteHeader(hdr); err != nil { + return err + } + _, err := io.Copy(tf, r) + return err +}