From 42d3c577f2fcc5182cbdc7b67b50b451b6bd70ba Mon Sep 17 00:00:00 2001 From: Peter Wagner <1559510+thepwagner@users.noreply.github.com> Date: Tue, 22 Dec 2020 06:48:00 -0500 Subject: [PATCH 1/4] SHA_PINNING option --- action.yml | 5 +++++ docker/env.go | 3 ++- docker/updater.go | 11 +++++++++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/action.yml b/action.yml index 7d28c33..23ba8e1 100644 --- a/action.yml +++ b/action.yml @@ -39,6 +39,10 @@ inputs: pattern: /.*aws.*/ range: >= 2.0.0, <3.0.0 required: false + sha_pinning: + description: 'Replace images with pinned shasums' + required: false + default: false runs: using: "composite" steps: @@ -57,3 +61,4 @@ runs: INPUT_TOKEN: ${{ inputs.token }} INPUT_LOG_LEVEL: ${{ inputs.log_level }} INPUT_IGNORE: ${{ inputs.ignore }} + INPUT_SHA_PINNING: ${{ inputs.sha_pinning }} diff --git a/docker/env.go b/docker/env.go index 96ded80..3526650 100644 --- a/docker/env.go +++ b/docker/env.go @@ -7,10 +7,11 @@ import ( type Environment struct { updateaction.Environment + ShaPinning bool `env:"INPUT_SHA_PINNING" envDefault:"false"` } func (c *Environment) NewUpdater(root string) updater.Updater { - u := NewUpdater(root) + u := NewUpdater(root, WithShaPinning(c.ShaPinning)) u.pathFilter = c.Ignored return u } diff --git a/docker/updater.go b/docker/updater.go index 3280a16..7c142d3 100644 --- a/docker/updater.go +++ b/docker/updater.go @@ -5,8 +5,9 @@ import ( ) type Updater struct { - root string - pathFilter func(string) bool + root string + pathFilter func(string) bool + pinImageSha bool tags TagLister } @@ -31,3 +32,9 @@ func WithTagsLister(tags TagLister) UpdaterOpt { u.tags = tags } } + +func WithShaPinning(shaPinning bool) UpdaterOpt { + return func(u *Updater) { + u.pinImageSha = shaPinning + } +} From 790ec95f4009c5605001fa80353319ff12787e28 Mon Sep 17 00:00:00 2001 From: Peter Wagner <1559510+thepwagner@users.noreply.github.com> Date: Tue, 22 Dec 2020 06:48:06 -0500 Subject: [PATCH 2/4] rough impl: docker pinning --- docker/applyupdate.go | 39 ++++++++---- docker/applyupdate_test.go | 23 ++++++- docker/mockimagepinner_test.go | 35 +++++++++++ docker/registry.go | 107 +++++++++++++++++++++++++++++++++ docker/registry_test.go | 47 +++++++++++++++ docker/taglister.go | 49 --------------- docker/taglister_test.go | 28 --------- docker/updater.go | 15 ++++- docker/updater_test.go | 11 ++-- 9 files changed, 257 insertions(+), 97 deletions(-) create mode 100644 docker/mockimagepinner_test.go create mode 100644 docker/registry.go create mode 100644 docker/registry_test.go delete mode 100644 docker/taglister.go delete mode 100644 docker/taglister_test.go diff --git a/docker/applyupdate.go b/docker/applyupdate.go index 22cf2d6..c9ec72d 100644 --- a/docker/applyupdate.go +++ b/docker/applyupdate.go @@ -2,9 +2,10 @@ package docker import ( "context" - "io" + "fmt" "io/ioutil" "os" + "regexp" "strings" "github.com/moby/buildkit/frontend/dockerfile/command" @@ -13,7 +14,18 @@ import ( "golang.org/x/mod/semver" ) -func (u *Updater) ApplyUpdate(_ context.Context, update updater.Update) error { +func (u *Updater) ApplyUpdate(ctx context.Context, update updater.Update) error { + var nextVersion string + if u.pinImageSha { + pinned, err := u.pinner.Pin(ctx, fmt.Sprintf("%s:%s", update.Path, update.Next)) + if err != nil { + return fmt.Errorf("pinning image: %w", err) + } + nextVersion = pinned + } else { + nextVersion = update.Next + } + return WalkDockerfiles(u.root, u.pathFilter, func(path string, parsed *parser.Result) error { vars := NewInterpolation(parsed) @@ -28,8 +40,16 @@ func (u *Updater) ApplyUpdate(_ context.Context, update updater.Update) error { if dep == nil || dep.Path != update.Path { continue } + // Ignore FROM statements with a variable: if !strings.Contains(instruction.Original, "$") { - oldnew = append(oldnew, instruction.Original, strings.ReplaceAll(instruction.Original, update.Previous, update.Next)) + re := regexp.MustCompile(fmt.Sprintf(`%s[:@][^\s]*`, regexp.QuoteMeta(update.Path))) + var replacement string + if u.pinImageSha { + replacement = fmt.Sprintf("%s@%s", update.Path, nextVersion) + } else { + replacement = fmt.Sprintf("%s:%s", update.Path, nextVersion) + } + oldnew = append(oldnew, instruction.Original, re.ReplaceAllString(instruction.Original, replacement)) } case command.Arg: if seenFrom { @@ -43,7 +63,7 @@ func (u *Updater) ApplyUpdate(_ context.Context, update updater.Update) error { if varSplit[1] == update.Previous { // Variable is exact version, direct replace - oldnew = append(oldnew, instruction.Original, strings.ReplaceAll(instruction.Original, update.Previous, update.Next)) + oldnew = append(oldnew, instruction.Original, strings.ReplaceAll(instruction.Original, update.Previous, nextVersion)) continue } @@ -67,20 +87,17 @@ func (u *Updater) ApplyUpdate(_ context.Context, update updater.Update) error { } // Read file into memory: - f, err := os.OpenFile(path, os.O_RDWR, 0640) - if err != nil { - return err - } - defer f.Close() - b, err := ioutil.ReadAll(f) + b, err := ioutil.ReadFile(path) if err != nil { return err } // Rewrite contents through replacer: - if _, err := f.Seek(0, io.SeekStart); err != nil { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0640) + if err != nil { return err } + defer f.Close() if _, err := strings.NewReplacer(oldnew...).WriteString(f, string(b)); err != nil { return err } diff --git a/docker/applyupdate_test.go b/docker/applyupdate_test.go index 3047b49..92cc515 100644 --- a/docker/applyupdate_test.go +++ b/docker/applyupdate_test.go @@ -6,7 +6,9 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/thepwagner/action-update-docker/docker" "github.com/thepwagner/action-update/updater" "github.com/thepwagner/action-update/updatertest" ) @@ -22,6 +24,13 @@ func TestUpdater_ApplyUpdate_Simple(t *testing.T) { assert.NotContains(t, dockerfile, "alpine:3.11.0") } +func TestUpdater_ApplyUpdate_Simple_Pinned(t *testing.T) { + dockerfile := applyUpdateToFixture(t, "simple", alpine3120, withShaPinning("sha256:pinned")...) + assert.Contains(t, dockerfile, "alpine@sha256:pinned") + assert.NotContains(t, dockerfile, "alpine:3.12.0") + assert.NotContains(t, dockerfile, "alpine:3.11.0") +} + func TestUpdater_ApplyUpdate_BuildArg(t *testing.T) { dockerfile := applyUpdateToFixture(t, "buildarg", alpine3120) assert.Contains(t, dockerfile, "FROM alpine:$ALPINE_VERSION") @@ -43,10 +52,20 @@ func TestUpdater_ApplyUpdate_Comments(t *testing.T) { assert.Contains(t, dockerfile, "# check out this whitespace\n\n\n# intentional trailing spaces \n") } -func applyUpdateToFixture(t *testing.T, fixture string, update updater.Update) string { - tempDir := updatertest.ApplyUpdateToFixture(t, fixture, &testFactory{}, update) +func applyUpdateToFixture(t *testing.T, fixture string, update updater.Update, opts ...docker.UpdaterOpt) string { + tempDir := updatertest.ApplyUpdateToFixture(t, fixture, updaterFactory(opts...), update) b, err := ioutil.ReadFile(filepath.Join(tempDir, "Dockerfile")) require.NoError(t, err) dockerfile := string(b) + t.Log(dockerfile) return dockerfile } + +func withShaPinning(pinned string) []docker.UpdaterOpt { + mockPinner := &mockImagePinner{} + mockPinner.On("Pin", mock.Anything, mock.Anything).Return(pinned, nil) + return []docker.UpdaterOpt{ + docker.WithShaPinning(true), + docker.WithImagePinner(mockPinner), + } +} diff --git a/docker/mockimagepinner_test.go b/docker/mockimagepinner_test.go new file mode 100644 index 0000000..7c8a300 --- /dev/null +++ b/docker/mockimagepinner_test.go @@ -0,0 +1,35 @@ +// Code generated by mockery v0.0.0-dev. DO NOT EDIT. + +package docker_test + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// mockImagePinner is an autogenerated mock type for the ImagePinner type +type mockImagePinner struct { + mock.Mock +} + +// Pin provides a mock function with given fields: ctx, image +func (_m *mockImagePinner) Pin(ctx context.Context, image string) (string, error) { + ret := _m.Called(ctx, image) + + var r0 string + if rf, ok := ret.Get(0).(func(context.Context, string) string); ok { + r0 = rf(ctx, image) + } else { + r0 = ret.Get(0).(string) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, image) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/docker/registry.go b/docker/registry.go new file mode 100644 index 0000000..660ce8c --- /dev/null +++ b/docker/registry.go @@ -0,0 +1,107 @@ +package docker + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/docker/cli/cli/command" + "github.com/docker/cli/cli/flags" + "github.com/docker/cli/cli/manifest/types" + "github.com/docker/cli/cli/registry/client" + "github.com/docker/distribution/reference" + "github.com/sirupsen/logrus" +) + +type TagLister interface { + // Tags returns potential version tags given a updater.Dependency path + Tags(ctx context.Context, path string) ([]string, error) +} + +type ImagePinner interface { + // Pin normalizes Docker image name to sha256 pinned image. + Pin(ctx context.Context, image string) (string, error) +} + +type RemoteRegistries struct { + rt http.RoundTripper +} + +func NewRemoteRegistries() *RemoteRegistries { + return &RemoteRegistries{ + rt: http.DefaultTransport, + } +} + +func (r *RemoteRegistries) Tags(ctx context.Context, image string) ([]string, error) { + // Normalize image name: + normalized, err := reference.ParseNormalizedNamed(image) + if err != nil { + return nil, fmt.Errorf("invalid image name: %w", err) + } + logrus.WithField("image", normalized.String()).Debug("listing image tags") + + cli, err := command.NewDockerCli() + if err != nil { + return nil, err + } + if err := cli.Initialize(flags.NewClientOptions()); err != nil { + return nil, fmt.Errorf("initializing cli: %w", err) + } + + tags, err := cli.RegistryClient(false).GetTags(ctx, normalized) + if err != nil { + return nil, fmt.Errorf("listing tags: %w", err) + } + return tags, nil +} + +func (r *RemoteRegistries) Pin(ctx context.Context, image string) (string, error) { + // Normalize image name: + normalized, err := reference.ParseNormalizedNamed(image) + if err != nil { + return "", fmt.Errorf("invalid image name: %w", err) + } + logrus.WithField("image", normalized.String()).Debug("listing image tags") + + cli, err := command.NewDockerCli() + if err != nil { + return "", err + } + if err := cli.Initialize(flags.NewClientOptions()); err != nil { + return "", fmt.Errorf("initializing cli: %w", err) + } + + registryClient := cli.RegistryClient(false) + mf, err := r.getManifest(ctx, registryClient, normalized) + if err != nil { + return "", fmt.Errorf("getting manifest: %w", err) + } + return mf.Descriptor.Digest.String(), nil +} + +func (r *RemoteRegistries) getManifest(ctx context.Context, registryClient client.RegistryClient, normalized reference.Named) (*types.ImageManifest, error) { + // Assume this image is available for one platform: + mf, err := registryClient.GetManifest(ctx, normalized) + if err == nil { + return &mf, nil + } + if !strings.Contains(err.Error(), "is a manifest list") { + return nil, fmt.Errorf("getting manifest: %w", err) + } + + // Multi-platform images have a list of manifests, select the "right" one: + manifestList, err := registryClient.GetManifestList(ctx, normalized) + if err != nil { + return nil, fmt.Errorf("fetching manifest list: %w", err) + } + for _, mf := range manifestList { + pl := mf.Descriptor.Platform + if pl.Architecture != "amd64" && pl.OS != "linux" { + continue + } + return &mf, nil + } + return nil, fmt.Errorf("could not resolve %q", normalized.String()) +} diff --git a/docker/registry_test.go b/docker/registry_test.go new file mode 100644 index 0000000..e3ff3f5 --- /dev/null +++ b/docker/registry_test.go @@ -0,0 +1,47 @@ +package docker_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/thepwagner/action-update-docker/docker" +) + +func TestRemoteTagLister_Tags(t *testing.T) { + t.Skip("queries dockerhub") + cases := []string{ + "alpine", + "datadog/agent", + "ghcr.io/thepwagner-smurf/alpine", + } + + for _, tc := range cases { + tl := docker.NewRemoteRegistries() + t.Run(tc, func(t *testing.T) { + tags, err := tl.Tags(context.Background(), tc) + require.NoError(t, err) + assert.NotEmpty(t, tags) + }) + } +} + +func TestRemoteTagLister_Pin(t *testing.T) { + t.Skip("queries dockerhub") + cases := []string{ + "alpine:3.11.0", + "datadog/agent:7", + "ghcr.io/thepwagner-smurf/alpine:3.11.0", + } + + for _, tc := range cases { + tl := docker.NewRemoteRegistries() + t.Run(tc, func(t *testing.T) { + pinned, err := tl.Pin(context.Background(), tc) + require.NoError(t, err) + t.Log(pinned) + assert.Contains(t, pinned, "@sha256:") + }) + } +} diff --git a/docker/taglister.go b/docker/taglister.go deleted file mode 100644 index 0d30f31..0000000 --- a/docker/taglister.go +++ /dev/null @@ -1,49 +0,0 @@ -package docker - -import ( - "context" - "fmt" - "net/http" - - "github.com/docker/cli/cli/command" - "github.com/docker/cli/cli/flags" - "github.com/docker/distribution/reference" - "github.com/sirupsen/logrus" -) - -type TagLister interface { - Tags(ctx context.Context, path string) ([]string, error) -} - -type RemoteTagLister struct { - rt http.RoundTripper -} - -func NewRemoteTagLister() *RemoteTagLister { - return &RemoteTagLister{ - rt: http.DefaultTransport, - } -} - -func (r *RemoteTagLister) Tags(ctx context.Context, image string) ([]string, error) { - // Normalize image name: - normalized, err := reference.ParseNormalizedNamed(image) - if err != nil { - return nil, fmt.Errorf("invalid image name: %w", err) - } - logrus.WithField("image", normalized.String()).Debug("listing image tags") - - cli, err := command.NewDockerCli() - if err != nil { - return nil, err - } - if err := cli.Initialize(flags.NewClientOptions()); err != nil { - return nil, fmt.Errorf("initializing cli: %w", err) - } - - tags, err := cli.RegistryClient(false).GetTags(ctx, normalized) - if err != nil { - return nil, fmt.Errorf("listing tags: %w", err) - } - return tags, nil -} diff --git a/docker/taglister_test.go b/docker/taglister_test.go deleted file mode 100644 index 89b4dab..0000000 --- a/docker/taglister_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package docker_test - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/thepwagner/action-update-docker/docker" -) - -func TestRemoteTagLister_Tags(t *testing.T) { - t.Skip("queries dockerhub") - cases := []string{ - "alpine", - "datadog/agent", - "ghcr.io/thepwagner/alpine", - } - - for _, tc := range cases { - tl := docker.NewRemoteTagLister() - t.Run(tc, func(t *testing.T) { - tags, err := tl.Tags(context.Background(), tc) - require.NoError(t, err) - assert.NotEmpty(t, tags) - }) - } -} diff --git a/docker/updater.go b/docker/updater.go index 7c142d3..c10d8f8 100644 --- a/docker/updater.go +++ b/docker/updater.go @@ -9,15 +9,18 @@ type Updater struct { pathFilter func(string) bool pinImageSha bool - tags TagLister + tags TagLister + pinner ImagePinner } var _ updater.Updater = (*Updater)(nil) func NewUpdater(root string, opts ...UpdaterOpt) *Updater { + reg := NewRemoteRegistries() u := &Updater{ - root: root, - tags: NewRemoteTagLister(), + root: root, + tags: reg, + pinner: reg, } for _, opt := range opts { opt(u) @@ -33,6 +36,12 @@ func WithTagsLister(tags TagLister) UpdaterOpt { } } +func WithImagePinner(pinner ImagePinner) UpdaterOpt { + return func(u *Updater) { + u.pinner = pinner + } +} + func WithShaPinning(shaPinning bool) UpdaterOpt { return func(u *Updater) { u.pinImageSha = shaPinning diff --git a/docker/updater_test.go b/docker/updater_test.go index cc12327..58e0c5f 100644 --- a/docker/updater_test.go +++ b/docker/updater_test.go @@ -6,13 +6,16 @@ import ( ) //go:generate mockery --outpkg docker_test --output . --testonly --name TagLister --structname mockTagLister --filename mocktaglister_test.go +//go:generate mockery --outpkg docker_test --output . --testonly --name ImagePinner --structname mockImagePinner --filename mockimagepinner_test.go -type testFactory struct{} +type testFactory struct { + opts []docker.UpdaterOpt +} func (u *testFactory) NewUpdater(root string) updater.Updater { - return docker.NewUpdater(root) + return docker.NewUpdater(root, u.opts...) } -func updaterFactory() updater.Factory { - return &testFactory{} +func updaterFactory(opts ...docker.UpdaterOpt) updater.Factory { + return &testFactory{opts: opts} } From 1097877ee330d8ff3cb21ddf520cd97b6b7a4178 Mon Sep 17 00:00:00 2001 From: Peter Wagner <1559510+thepwagner@users.noreply.github.com> Date: Thu, 21 Jan 2021 16:31:04 -0500 Subject: [PATCH 3/4] pinning: update nearby comments --- docker/applyupdate.go | 17 ++++++++++++++++- docker/applyupdate_test.go | 21 +++++++++++++++++++-- docker/dependencies.go | 5 +++++ docker/dependencies_test.go | 3 +++ docker/testdata/pinned/Dockerfile | 2 ++ docker/walk_test.go | 2 +- 6 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 docker/testdata/pinned/Dockerfile diff --git a/docker/applyupdate.go b/docker/applyupdate.go index c9ec72d..f9a2514 100644 --- a/docker/applyupdate.go +++ b/docker/applyupdate.go @@ -49,7 +49,22 @@ func (u *Updater) ApplyUpdate(ctx context.Context, update updater.Update) error } else { replacement = fmt.Sprintf("%s:%s", update.Path, nextVersion) } - oldnew = append(oldnew, instruction.Original, re.ReplaceAllString(instruction.Original, replacement)) + newInstruction := re.ReplaceAllString(instruction.Original, replacement) + + oldVersion := fmt.Sprintf("%s:%s", update.Path, update.Previous) + newVersion := fmt.Sprintf("%s:%s", update.Path, update.Next) + var commentFound bool + for _, comment := range instruction.PrevComment { + if strings.Contains(comment, oldVersion) { + comment = fmt.Sprintf("# %s", comment) + oldnew = append(oldnew, comment, re.ReplaceAllString(comment, newVersion)) + commentFound = true + } + } + if u.pinImageSha && !commentFound { + newInstruction = fmt.Sprintf("# %s\n%s", newVersion, newInstruction) + } + oldnew = append(oldnew, instruction.Original, newInstruction) } case command.Arg: if seenFrom { diff --git a/docker/applyupdate_test.go b/docker/applyupdate_test.go index 92cc515..c8bfffb 100644 --- a/docker/applyupdate_test.go +++ b/docker/applyupdate_test.go @@ -15,19 +15,22 @@ import ( var ( alpine3120 = updater.Update{Path: "alpine", Previous: "3.11.0", Next: "3.12.0"} - redis608 = updater.Update{Path: "redis", Previous: "6.0.0-alpine", Next: "6.0.8-alpine"} + //alpine3120Pinned = updater.Update{Path: "alpine", Previous: "sha256:7c92a2c6bbcb6b6beff92d0a940779769c2477b807c202954c537e2e0deb9bed", Next: "3.12.0"} + redis608 = updater.Update{Path: "redis", Previous: "6.0.0-alpine", Next: "6.0.8-alpine"} ) func TestUpdater_ApplyUpdate_Simple(t *testing.T) { dockerfile := applyUpdateToFixture(t, "simple", alpine3120) assert.Contains(t, dockerfile, "alpine:3.12.0") + assert.NotContains(t, dockerfile, "# alpine:3.12.0\n") assert.NotContains(t, dockerfile, "alpine:3.11.0") } func TestUpdater_ApplyUpdate_Simple_Pinned(t *testing.T) { dockerfile := applyUpdateToFixture(t, "simple", alpine3120, withShaPinning("sha256:pinned")...) assert.Contains(t, dockerfile, "alpine@sha256:pinned") - assert.NotContains(t, dockerfile, "alpine:3.12.0") + assert.Contains(t, dockerfile, "# alpine:3.12.0\n") + assert.NotContains(t, dockerfile, "FROM alpine:3.12.0") assert.NotContains(t, dockerfile, "alpine:3.11.0") } @@ -52,6 +55,20 @@ func TestUpdater_ApplyUpdate_Comments(t *testing.T) { assert.Contains(t, dockerfile, "# check out this whitespace\n\n\n# intentional trailing spaces \n") } +func TestUpdater_ApplyUpdate_Pinned(t *testing.T) { + dockerfile := applyUpdateToFixture(t, "pinned", alpine3120) + assert.Contains(t, dockerfile, "FROM alpine:3.12.0") + assert.Contains(t, dockerfile, "# alpine:3.12.0\n") + assert.NotContains(t, dockerfile, "alpine:3.11.0") +} +func TestUpdater_ApplyUpdate_Pinned_Pinned(t *testing.T) { + dockerfile := applyUpdateToFixture(t, "pinned", alpine3120, withShaPinning("sha256:pinned")...) + assert.Contains(t, dockerfile, "alpine@sha256:pinned") + assert.Contains(t, dockerfile, "# alpine:3.12.0\n") + assert.NotContains(t, dockerfile, "FROM alpine:3.12.0") + assert.NotContains(t, dockerfile, "alpine:3.11.0") +} + func applyUpdateToFixture(t *testing.T, fixture string, update updater.Update, opts ...docker.UpdaterOpt) string { tempDir := updatertest.ApplyUpdateToFixture(t, fixture, updaterFactory(opts...), update) b, err := ioutil.ReadFile(filepath.Join(tempDir, "Dockerfile")) diff --git a/docker/dependencies.go b/docker/dependencies.go index 3a68d50..8b5859e 100644 --- a/docker/dependencies.go +++ b/docker/dependencies.go @@ -3,6 +3,7 @@ package docker import ( "context" "fmt" + "regexp" "strings" "github.com/moby/buildkit/frontend/dockerfile/command" @@ -46,6 +47,8 @@ func extractImages(parsed *parser.Result) ([]updater.Dependency, error) { return deps, nil } +var sha256RE = regexp.MustCompile("[a-f0-9]{64}") + func parseDependency(vars *Interpolation, image string) *updater.Dependency { imageSplit := strings.SplitN(image, ":", 2) if len(imageSplit) == 1 { @@ -62,6 +65,8 @@ func parseDependency(vars *Interpolation, image string) *updater.Dependency { } else if semverIsh(imageSplit[1]) != "" { // Image tag is valid semver: return &updater.Dependency{Path: imageSplit[0], Version: imageSplit[1]} + } else if strings.HasSuffix(imageSplit[0], "@sha256") && sha256RE.MatchString(imageSplit[1]) { + return &updater.Dependency{Path: imageSplit[0][:len(imageSplit[0])-7], Version: fmt.Sprintf("sha256:%s", imageSplit[1])} } return nil } diff --git a/docker/dependencies_test.go b/docker/dependencies_test.go index 60337f2..e2eda73 100644 --- a/docker/dependencies_test.go +++ b/docker/dependencies_test.go @@ -21,6 +21,9 @@ func TestUpdater_Dependencies(t *testing.T) { {Path: "redis", Version: "6.0.0-alpine"}, {Path: "alpine", Version: "3.11.0"}, }, + "pinned": { + {Path: "alpine", Version: "sha256:7c92a2c6bbcb6b6beff92d0a940779769c2477b807c202954c537e2e0deb9bed"}, + }, } updatertest.DependenciesFixtures(t, updaterFactory(), cases) } diff --git a/docker/testdata/pinned/Dockerfile b/docker/testdata/pinned/Dockerfile new file mode 100644 index 0000000..ff8a970 --- /dev/null +++ b/docker/testdata/pinned/Dockerfile @@ -0,0 +1,2 @@ +# alpine:3.11.0 +FROM alpine@sha256:7c92a2c6bbcb6b6beff92d0a940779769c2477b807c202954c537e2e0deb9bed diff --git a/docker/walk_test.go b/docker/walk_test.go index 988e2c0..85c26f8 100644 --- a/docker/walk_test.go +++ b/docker/walk_test.go @@ -12,7 +12,7 @@ import ( "github.com/thepwagner/action-update-docker/docker" ) -const fixtureCount = 3 +const fixtureCount = 4 func TestWalkDockerfiles(t *testing.T) { var cnt int64 From 0bc30447f51bae70c4ec9a398b39772df377b7a0 Mon Sep 17 00:00:00 2001 From: Peter Wagner <1559510+thepwagner@users.noreply.github.com> Date: Thu, 21 Jan 2021 17:34:21 -0500 Subject: [PATCH 4/4] resolve pinned shas --- docker/check.go | 43 +++++++++++++++------- docker/check_test.go | 8 +++++ docker/mockimagepinner_test.go | 21 +++++++++++ docker/registry.go | 66 ++++++++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 12 deletions(-) diff --git a/docker/check.go b/docker/check.go index 67c2c8b..63de2f1 100644 --- a/docker/check.go +++ b/docker/check.go @@ -3,6 +3,7 @@ package docker import ( "context" "fmt" + "regexp" "sort" "strings" @@ -14,8 +15,15 @@ import ( func (u *Updater) Check(ctx context.Context, dependency updater.Dependency, filter func(string) bool) (*updater.Update, error) { previous := semverIsh(dependency.Version) if previous == "" { - logrus.WithFields(logrus.Fields{"path": dependency.Path, "version": dependency.Version}).Debug("ignoring non-semver dependency") - return nil, nil + if !sha256Ish(dependency.Version) { + logrus.WithFields(logrus.Fields{"path": dependency.Path, "version": dependency.Version}).Debug("ignoring non-semver dependency") + return nil, nil + } + tag, err := u.pinner.Unpin(ctx, dependency.Path, dependency.Version) + if err != nil { + return nil, fmt.Errorf("unpinning %q: %w", dependency.Path, err) + } + previous = tag } suffix := semver.Prerelease(previous) @@ -50,16 +58,7 @@ func (u *Updater) Check(ctx context.Context, dependency updater.Dependency, filt return nil, nil } - sort.Slice(versions, func(i, j int) bool { - // Prefer strict semver ordering: - if c := semver.Compare(versions[i], versions[j]); c > 0 { - return true - } else if c < 0 { - return false - } - // Failing that, prefer the most specific version: - return strings.Count(versions[i], ".") > strings.Count(versions[j], ".") - }) + versions = semverSort(versions) latest := versions[0] if semver.Compare(previous, latest) >= 0 { return nil, nil @@ -72,6 +71,20 @@ func (u *Updater) Check(ctx context.Context, dependency updater.Dependency, filt }, nil } +func semverSort(versions []string) []string { + sort.Slice(versions, func(i, j int) bool { + // Prefer strict semver ordering: + if c := semver.Compare(semverIsh(versions[i]), semverIsh(versions[j])); c > 0 { + return true + } else if c < 0 { + return false + } + // Failing that, prefer the most specific version: + return strings.Count(versions[i], ".") > strings.Count(versions[j], ".") + }) + return versions +} + func semverIsh(s string) string { if semver.IsValid(s) { return s @@ -82,3 +95,9 @@ func semverIsh(s string) string { } return "" } + +var sha256VersionRE = regexp.MustCompile("sha256:[a-f0-9]{64}") + +func sha256Ish(s string) bool { + return sha256VersionRE.MatchString(s) +} diff --git a/docker/check_test.go b/docker/check_test.go index 05f128d..3df47f7 100644 --- a/docker/check_test.go +++ b/docker/check_test.go @@ -29,6 +29,14 @@ func TestUpdater_CheckAuth(t *testing.T) { assert.Equal(t, "3.12.0", u.Next) } +func TestUpdater_CheckAuthPinned(t *testing.T) { + t.Skip("early integration test") + var privateImage = updater.Dependency{Path: "ghcr.io/thepwagner/alpine", Version: "sha256:d371657a4f661a854ff050898003f4cb6c7f36d968a943c1d5cde0952bd93c80"} + u := updatertest.CheckInFixture(t, "pinned", updaterFactory(), privateImage, nil) + assert.NotNil(t, u) + assert.Equal(t, "3.12.0", u.Next) +} + func TestUpdater_Check(t *testing.T) { cases := map[string]struct { dep updater.Dependency diff --git a/docker/mockimagepinner_test.go b/docker/mockimagepinner_test.go index 7c8a300..c071653 100644 --- a/docker/mockimagepinner_test.go +++ b/docker/mockimagepinner_test.go @@ -33,3 +33,24 @@ func (_m *mockImagePinner) Pin(ctx context.Context, image string) (string, error return r0, r1 } + +// Unpin provides a mock function with given fields: ctx, image, hash +func (_m *mockImagePinner) Unpin(ctx context.Context, image string, hash string) (string, error) { + ret := _m.Called(ctx, image, hash) + + var r0 string + if rf, ok := ret.Get(0).(func(context.Context, string, string) string); ok { + r0 = rf(ctx, image, hash) + } else { + r0 = ret.Get(0).(string) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, image, hash) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/docker/registry.go b/docker/registry.go index 660ce8c..a5e7c71 100644 --- a/docker/registry.go +++ b/docker/registry.go @@ -22,6 +22,7 @@ type TagLister interface { type ImagePinner interface { // Pin normalizes Docker image name to sha256 pinned image. Pin(ctx context.Context, image string) (string, error) + Unpin(ctx context.Context, image, hash string) (string, error) } type RemoteRegistries struct { @@ -81,6 +82,71 @@ func (r *RemoteRegistries) Pin(ctx context.Context, image string) (string, error return mf.Descriptor.Digest.String(), nil } +func (r *RemoteRegistries) Unpin(ctx context.Context, image, hash string) (string, error) { + normalized, err := reference.ParseNormalizedNamed(fmt.Sprintf("%s@%s", image, hash)) + if err != nil { + return "", fmt.Errorf("invalid image name: %w", err) + } + + cli, err := command.NewDockerCli() + if err != nil { + return "", err + } + if err := cli.Initialize(flags.NewClientOptions()); err != nil { + return "", fmt.Errorf("initializing cli: %w", err) + } + + registryClient := cli.RegistryClient(false) + tags, err := registryClient.GetTags(ctx, normalized) + if err != nil { + return "", err + } + if len(tags) == 0 { + return "", fmt.Errorf("tag not found") + } + + // Filter semver tags, work backwards (assuming the pinned sha is a near-latest version) + semverTags := make([]string, 0) + for _, tag := range tags { + if semverIsh(tag) == "" { + continue + } + semverTags = append(semverTags, tag) + } + semverTags = semverSort(semverTags) + + logrus.WithFields(logrus.Fields{ + "image": normalized.String(), + "hash": hash, + "tags": len(semverTags), + }).Info("listing tags to identify SHA") + + for _, tag := range semverTags { + normalizedTag, err := reference.ParseNormalizedNamed(fmt.Sprintf("%s:%s", normalized.Name(), tag)) + if err != nil { + continue + } + mf, err := r.getManifest(ctx, registryClient, normalizedTag) + if err != nil { + continue + } + digest := mf.Descriptor.Digest.String() + logrus.WithFields(logrus.Fields{ + "tag": tag, + "digest": digest, + }).Debug("fetched image details") + + if digest == hash { + logrus.WithFields(logrus.Fields{ + "digest": digest, + "tag": tag, + }).Info("resolved pinned image to tag") + return tag, nil + } + } + return "", fmt.Errorf("manifest not found") +} + func (r *RemoteRegistries) getManifest(ctx context.Context, registryClient client.RegistryClient, normalized reference.Named) (*types.ImageManifest, error) { // Assume this image is available for one platform: mf, err := registryClient.GetManifest(ctx, normalized)