dockerfile: implement ADD --checksum=<checksum> <http src> <dest>

e.g.,

  ADD --checksum=sha256:24454f830cdb571e2c4ad15481119c43b3cafd48dd869a9b2945d1036d1dc68d https://mirrors.edge.kernel.org/pub/linux/kernel/Historic/linux-0.01.tar.gz /

Fix issue 975

Signed-off-by: Akihiro Suda <[email protected]>

.golangci.yml

@@ -8,6 +8,7 @@ run:
   build-tags:
     - dfrunsecurity
     - dfaddgit
+    - dfaddchecksum
 
 linters:
   enable:

frontend/dockerfile/dockerfile2llb/convert.go

@@ -34,6 +34,7 @@ import (
 	"github.com/moby/buildkit/util/suggest"
 	"github.com/moby/buildkit/util/system"
 	"github.com/moby/sys/signal"
+	digest "github.com/opencontainers/go-digest"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
@@ -724,6 +725,7 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
 			chmod:        c.Chmod,
 			link:         c.Link,
 			keepGitDir:   c.KeepGitDir,
+			checksum:     c.Checksum,
 			location:     c.Location(),
 			opt:          opt,
 		})
@@ -1069,6 +1071,21 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 		}
 	}
 
+	if cfg.checksum != "" {
+		if !cfg.isAddCommand {
+			return errors.New("checksum can't be specified for COPY")
+		}
+		if !addChecksumEnabled {
+			return errors.New("instruction 'ADD --checksum=<CHECKSUM>' requires the labs channel")
+		}
+		if len(cfg.params.SourcePaths) != 1 {
+			return errors.New("checksum can't be specified for multiple sources")
+		}
+		if !isHTTPSource(cfg.params.SourcePaths[0]) {
+			return errors.New("checksum can't be specified for non-HTTP sources")
+		}
+	}
+
 	commitMessage := bytes.NewBufferString("")
 	if cfg.isAddCommand {
 		commitMessage.WriteString("ADD")
@@ -1107,7 +1124,7 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 			} else {
 				a = a.Copy(st, "/", dest, opts...)
 			}
-		} else if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
+		} else if isHTTPSource(src) {
 			if !cfg.isAddCommand {
 				return errors.New("source can't be a URL for COPY")
 			}
@@ -1125,7 +1142,7 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 				}
 			}
 
-			st := llb.HTTP(src, llb.Filename(f), dfCmd(cfg.params))
+			st := llb.HTTP(src, llb.Filename(f), llb.Checksum(cfg.checksum), dfCmd(cfg.params))
 
 			opts := append([]llb.CopyOption{&llb.CopyInfo{
 				Mode:           mode,
@@ -1231,6 +1248,7 @@ type copyConfig struct {
 	chmod        string
 	link         bool
 	keepGitDir   bool
+	checksum     digest.Digest
 	location     []parser.Range
 	opt          dispatchOpt
 }
@@ -1732,3 +1750,7 @@ func commonImageNames() []string {
 	}
 	return out
 }
+
+func isHTTPSource(src string) bool {
+	return strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://")
+}

frontend/dockerfile/dockerfile2llb/convert_addchecksum.go

@@ -0,0 +1,6 @@
+//go:build dfaddchecksum
+// +build dfaddchecksum
+
+package dockerfile2llb
+
+const addChecksumEnabled = true

frontend/dockerfile/dockerfile2llb/convert_noaddchecksum.go

@@ -0,0 +1,6 @@
+//go:build !dfaddchecksum
+// +build !dfaddchecksum
+
+package dockerfile2llb
+
+const addChecksumEnabled = false

frontend/dockerfile/dockerfile_addchecksum_test.go

@@ -0,0 +1,155 @@
+//go:build dfaddchecksum
+// +build dfaddchecksum
+
+package dockerfile
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/containerd/continuity/fs/fstest"
+	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/frontend/dockerfile/builder"
+	"github.com/moby/buildkit/identity"
+	"github.com/moby/buildkit/util/testutil/httpserver"
+	"github.com/moby/buildkit/util/testutil/integration"
+	digest "github.com/opencontainers/go-digest"
+	"github.com/stretchr/testify/require"
+)
+
+var addChecksumTests = integration.TestFuncs(
+	testAddChecksum,
+)
+
+func init() {
+	allTests = append(allTests, addChecksumTests...)
+}
+
+func testAddChecksum(t *testing.T, sb integration.Sandbox) {
+	f := getFrontend(t, sb)
+	f.RequiresBuildctl(t)
+
+	resp := httpserver.Response{
+		Etag:    identity.NewID(),
+		Content: []byte("content1"),
+	}
+	server := httpserver.NewTestServer(map[string]httpserver.Response{
+		"/foo": resp,
+	})
+	defer server.Close()
+
+	c, err := client.New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	t.Run("Valid", func(t *testing.T) {
+		dockerfile := []byte(fmt.Sprintf(`
+FROM scratch
+ADD --checksum=%s %s /tmp/foo
+`, digest.FromBytes(resp.Content).String(), server.URL+"/foo"))
+		dir, err := integration.Tmpdir(
+			t,
+			fstest.CreateFile("Dockerfile", dockerfile, 0600),
+		)
+		require.NoError(t, err)
+		_, err = f.Solve(sb.Context(), c, client.SolveOpt{
+			LocalDirs: map[string]string{
+				builder.DefaultLocalNameDockerfile: dir,
+				builder.DefaultLocalNameContext:    dir,
+			},
+		}, nil)
+		require.NoError(t, err)
+	})
+	t.Run("DigestMismatch", func(t *testing.T) {
+		dockerfile := []byte(fmt.Sprintf(`
+FROM scratch
+ADD --checksum=%s %s /tmp/foo
+`, digest.FromBytes(nil).String(), server.URL+"/foo"))
+		dir, err := integration.Tmpdir(
+			t,
+			fstest.CreateFile("Dockerfile", dockerfile, 0600),
+		)
+		require.NoError(t, err)
+		_, err = f.Solve(sb.Context(), c, client.SolveOpt{
+			LocalDirs: map[string]string{
+				builder.DefaultLocalNameDockerfile: dir,
+				builder.DefaultLocalNameContext:    dir,
+			},
+		}, nil)
+		require.Error(t, err, "digest mismatch")
+	})
+	t.Run("DigestWithKnownButUnsupportedAlgoName", func(t *testing.T) {
+		dockerfile := []byte(fmt.Sprintf(`
+FROM scratch
+ADD --checksum=md5:7e55db001d319a94b0b713529a756623 %s /tmp/foo
+`, server.URL+"/foo"))
+		dir, err := integration.Tmpdir(
+			t,
+			fstest.CreateFile("Dockerfile", dockerfile, 0600),
+		)
+		require.NoError(t, err)
+		_, err = f.Solve(sb.Context(), c, client.SolveOpt{
+			LocalDirs: map[string]string{
+				builder.DefaultLocalNameDockerfile: dir,
+				builder.DefaultLocalNameContext:    dir,
+			},
+		}, nil)
+		require.Error(t, err, "unsupported digest algorithm")
+	})
+	t.Run("DigestWithUnknownAlgoName", func(t *testing.T) {
+		dockerfile := []byte(fmt.Sprintf(`
+FROM scratch
+ADD --checksum=unknown:%s %s /tmp/foo
+`, digest.FromBytes(resp.Content).Encoded(), server.URL+"/foo"))
+		dir, err := integration.Tmpdir(
+			t,
+			fstest.CreateFile("Dockerfile", dockerfile, 0600),
+		)
+		require.NoError(t, err)
+		_, err = f.Solve(sb.Context(), c, client.SolveOpt{
+			LocalDirs: map[string]string{
+				builder.DefaultLocalNameDockerfile: dir,
+				builder.DefaultLocalNameContext:    dir,
+			},
+		}, nil)
+		require.Error(t, err, "unsupported digest algorithm")
+	})
+	t.Run("DigestWithoutAlgoName", func(t *testing.T) {
+		dockerfile := []byte(fmt.Sprintf(`
+FROM scratch
+ADD --checksum=%s %s /tmp/foo
+`, digest.FromBytes(resp.Content).Encoded(), server.URL+"/foo"))
+		dir, err := integration.Tmpdir(
+			t,
+			fstest.CreateFile("Dockerfile", dockerfile, 0600),
+		)
+		require.NoError(t, err)
+		_, err = f.Solve(sb.Context(), c, client.SolveOpt{
+			LocalDirs: map[string]string{
+				builder.DefaultLocalNameDockerfile: dir,
+				builder.DefaultLocalNameContext:    dir,
+			},
+		}, nil)
+		require.Error(t, err, "invalid checksum digest format")
+	})
+	t.Run("NonHTTPSource", func(t *testing.T) {
+		foo := []byte("local file")
+		dockerfile := []byte(fmt.Sprintf(`
+FROM scratch
+ADD --checksum=%s foo /tmp/foo
+`, digest.FromBytes(foo).String()))
+		dir, err := integration.Tmpdir(
+			t,
+			fstest.CreateFile("foo", foo, 0600),
+			fstest.CreateFile("Dockerfile", dockerfile, 0600),
+		)
+		require.NoError(t, err)
+		_, err = f.Solve(sb.Context(), c, client.SolveOpt{
+			LocalDirs: map[string]string{
+				builder.DefaultLocalNameDockerfile: dir,
+				builder.DefaultLocalNameContext:    dir,
+			},
+		}, nil)
+		require.Error(t, err, "checksum can't be specified for non-HTTP sources")
+	})
+}

frontend/dockerfile/docs/reference.md

@@ -1336,7 +1336,7 @@ RUN apt-get update && apt-get install -y ...
 ADD has two forms:
 
 ```dockerfile
-ADD [--chown=<user>:<group>] <src>... <dest>
+ADD [--chown=<user>:<group>] [--checksum=<checksum>] <src>... <dest>
 ADD [--chown=<user>:<group>] ["<src>",... "<dest>"]
 ```
 
@@ -1507,6 +1507,20 @@ guide – Leverage build cache](https://docs.docker.com/develop/develop-images/d
 - If `<dest>` doesn't exist, it is created along with all missing directories
   in its path.
 
+### Verifying a remote file checksum `ADD --checksum=<checksum> <http src> <dest>`
+> **Note**
+>
+> Available in [`docker/dockerfile-upstream:master-labs`](#syntax).
+> Will be included in `docker/dockerfile:1.5-labs`.
+
+The checksum of a remote file can be verified with the `--checksum` flag:
+
+```dockerfile
+ADD --checksum=sha256:24454f830cdb571e2c4ad15481119c43b3cafd48dd869a9b2945d1036d1dc68d https://mirrors.edge.kernel.org/pub/linux/kernel/Historic/linux-0.01.tar.gz /
+```
+
+The `--checksum` flag only supports HTTP sources currently.
+
 ### Adding a git repository `ADD <git ref> <dir>`
 
 > **Note**

frontend/dockerfile/instructions/commands.go

@@ -6,6 +6,7 @@ import (
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/strslice"
 	"github.com/moby/buildkit/frontend/dockerfile/parser"
+	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
 )
 
@@ -226,6 +227,7 @@ type AddCommand struct {
 	Chmod      string
 	Link       bool
 	KeepGitDir bool // whether to keep .git dir, only meaningful for git sources
+	Checksum   digest.Digest
 }
 
 // Expand variables

frontend/dockerfile/instructions/parse.go

@@ -13,6 +13,7 @@ import (
 	"github.com/moby/buildkit/frontend/dockerfile/command"
 	"github.com/moby/buildkit/frontend/dockerfile/parser"
 	"github.com/moby/buildkit/util/suggest"
+	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
 )
 
@@ -282,6 +283,7 @@ func parseAdd(req parseRequest) (*AddCommand, error) {
 	flChmod := req.flags.AddString("chmod", "")
 	flLink := req.flags.AddBool("link", false)
 	flKeepGitDir := req.flags.AddBool("keep-git-dir", false)
+	flChecksum := req.flags.AddString("checksum", "")
 	if err := req.flags.Parse(); err != nil {
 		return nil, err
 	}
@@ -291,13 +293,22 @@ func parseAdd(req parseRequest) (*AddCommand, error) {
 		return nil, err
 	}
 
+	var checksum digest.Digest
+	if flChecksum.Value != "" {
+		checksum, err = digest.Parse(flChecksum.Value)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	return &AddCommand{
 		withNameAndCode: newWithNameAndCode(req),
 		SourcesAndDest:  *sourcesAndDest,
 		Chown:           flChown.Value,
 		Chmod:           flChmod.Value,
 		Link:            flLink.Value == "true",
 		KeepGitDir:      flKeepGitDir.Value == "true",
+		Checksum:        checksum,
 	}, nil
 }
 

frontend/dockerfile/release/labs/tags

@@ -1 +1 @@
-dfrunsecurity dfaddgit
+dfrunsecurity dfaddgit dfaddchecksum