diff --git a/README.md b/README.md index 09302889a6..c6d327acf7 100644 --- a/README.md +++ b/README.md @@ -386,7 +386,7 @@ as a remote image destination: ### Caching #### Caching Layers -kaniko can cache layers created by `RUN` commands in a remote repository. +kaniko can cache layers created by `RUN` and `Copy` (configured by flag `--cache-copy-layers`) commands in a remote repository. Before executing a command, kaniko checks the cache for the layer. If it exists, kaniko will pull and extract the cached layer instead of executing the command. If not, kaniko will execute the command and then push the newly created layer to the cache. diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index dc11de3d61..f98a0d4a32 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -179,6 +179,7 @@ func addKanikoOptionsFlags() { RootCmd.PersistentFlags().BoolVarP(&opts.SkipUnusedStages, "skip-unused-stages", "", false, "Build only used stages if defined to true. Otherwise it builds by default all stages, even the unnecessaries ones until it reaches the target stage / end of Dockerfile") RootCmd.PersistentFlags().BoolVarP(&opts.RunV2, "use-new-run", "", false, "Use the experimental run implementation for detecting changes without requiring file system snapshots.") RootCmd.PersistentFlags().Var(&opts.Git, "git", "Branch to clone if build context is a git repository") + RootCmd.PersistentFlags().BoolVarP(&opts.CacheCopyLayers, "cache-copy-layers", "", false, "Caches copy layers") } // addHiddenFlags marks certain flags as hidden from the executor help text diff --git a/integration/images.go b/integration/images.go index 3c3f0ccd70..36ec390c55 100644 --- a/integration/images.go +++ b/integration/images.go @@ -308,7 +308,7 @@ func populateVolumeCache() error { } // buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built -func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cacheRepo, dockerfilesPath string, version int) error { +func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cacheRepo, dockerfilesPath string, version int, args []string) error { imageRepo, serviceAccount := config.imageRepo, config.serviceAccount _, ex, _, _ := runtime.Caller(0) cwd := filepath.Dir(ex) @@ -334,6 +334,9 @@ func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cac cacheFlag, "--cache-repo", cacheRepo, "--cache-dir", cacheDir) + for _, v := range args { + dockerRunFlags = append(dockerRunFlags, v) + } kanikoCmd := exec.Command("docker", dockerRunFlags...) _, err := RunCommandWithoutTest(kanikoCmd) diff --git a/integration/integration_test.go b/integration/integration_test.go index 1ccce6a4e8..ad8d7ab49f 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -464,17 +464,21 @@ func buildImage(t *testing.T, dockerfile string, imageBuilder *DockerFileBuilder func TestCache(t *testing.T) { populateVolumeCache() for dockerfile := range imageBuilder.TestCacheDockerfiles { + args := []string{} + if dockerfile == "Dockerfile_test_cache_copy" { + args = append(args, "--cache-copy-layers=true") + } t.Run("test_cache_"+dockerfile, func(t *testing.T) { dockerfile := dockerfile t.Parallel() cache := filepath.Join(config.imageRepo, "cache", fmt.Sprintf("%v", time.Now().UnixNano())) // Build the initial image which will cache layers - if err := imageBuilder.buildCachedImages(config, cache, dockerfilesPath, 0); err != nil { + if err := imageBuilder.buildCachedImages(config, cache, dockerfilesPath, 0, args); err != nil { t.Fatalf("error building cached image for the first time: %v", err) } // Build the second image which should pull from the cache - if err := imageBuilder.buildCachedImages(config, cache, dockerfilesPath, 1); err != nil { + if err := imageBuilder.buildCachedImages(config, cache, dockerfilesPath, 1, args); err != nil { t.Fatalf("error building cached image for the first time: %v", err) } // Make sure both images are the same diff --git a/pkg/commands/commands.go b/pkg/commands/commands.go index 61e6fd497a..2683c98f56 100644 --- a/pkg/commands/commands.go +++ b/pkg/commands/commands.go @@ -60,7 +60,7 @@ type DockerCommand interface { ShouldDetectDeletedFiles() bool } -func GetCommand(cmd instructions.Command, fileContext util.FileContext, useNewRun bool) (DockerCommand, error) { +func GetCommand(cmd instructions.Command, fileContext util.FileContext, useNewRun bool, cacheCopy bool) (DockerCommand, error) { switch c := cmd.(type) { case *instructions.RunCommand: if useNewRun { @@ -68,7 +68,7 @@ func GetCommand(cmd instructions.Command, fileContext util.FileContext, useNewRu } return &RunCommand{cmd: c}, nil case *instructions.CopyCommand: - return &CopyCommand{cmd: c, fileContext: fileContext}, nil + return &CopyCommand{cmd: c, fileContext: fileContext, shdCache: cacheCopy}, nil case *instructions.ExposeCommand: return &ExposeCommand{cmd: c}, nil case *instructions.EnvCommand: diff --git a/pkg/commands/copy.go b/pkg/commands/copy.go index b4ae40956d..86671b2b41 100644 --- a/pkg/commands/copy.go +++ b/pkg/commands/copy.go @@ -17,6 +17,7 @@ limitations under the License. package commands import ( + "fmt" "os" "path/filepath" "strings" @@ -41,6 +42,7 @@ type CopyCommand struct { cmd *instructions.CopyCommand fileContext util.FileContext snapshotFiles []string + shdCache bool } func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { @@ -145,6 +147,85 @@ func (c *CopyCommand) From() string { return c.cmd.From } +func (c *CopyCommand) ShouldCacheOutput() bool { + return c.shdCache +} + +// CacheCommand returns true since this command should be cached +func (c *CopyCommand) CacheCommand(img v1.Image) DockerCommand { + return &CachingCopyCommand{ + img: img, + cmd: c.cmd, + fileContext: c.fileContext, + extractFn: util.ExtractFile, + } +} + +type CachingCopyCommand struct { + BaseCommand + caching + img v1.Image + extractedFiles []string + cmd *instructions.CopyCommand + fileContext util.FileContext + extractFn util.ExtractFunction +} + +func (cr *CachingCopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { + logrus.Infof("Found cached layer, extracting to filesystem") + var err error + + if cr.img == nil { + return errors.New(fmt.Sprintf("cached command image is nil %v", cr.String())) + } + + layers, err := cr.img.Layers() + if err != nil { + return errors.Wrapf(err, "retrieve image layers") + } + + if len(layers) != 1 { + return errors.New(fmt.Sprintf("expected %d layers but got %d", 1, len(layers))) + } + + cr.layer = layers[0] + cr.extractedFiles, err = util.GetFSFromLayers(kConfig.RootDir, layers, util.ExtractFunc(cr.extractFn), util.IncludeWhiteout()) + + logrus.Debugf("extractedFiles: %s", cr.extractedFiles) + if err != nil { + return errors.Wrap(err, "extracting fs from image") + } + + return nil +} + +func (cr *CachingCopyCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerfile.BuildArgs) ([]string, error) { + return copyCmdFilesUsedFromContext(config, buildArgs, cr.cmd, cr.fileContext) +} + +func (cr *CachingCopyCommand) FilesToSnapshot() []string { + f := cr.extractedFiles + logrus.Debugf("%d files extracted by caching copy command", len(f)) + logrus.Tracef("Extracted files: %s", f) + + return f +} + +func (cr *CachingCopyCommand) MetadataOnly() bool { + return false +} + +func (cr *CachingCopyCommand) String() string { + if cr.cmd == nil { + return "nil command" + } + return cr.cmd.String() +} + +func (cr *CachingCopyCommand) From() string { + return cr.cmd.From +} + func resolveIfSymlink(destPath string) (string, error) { if !filepath.IsAbs(destPath) { return "", errors.New("dest path must be abs") diff --git a/pkg/commands/copy_test.go b/pkg/commands/copy_test.go index d189df2609..1f72bf1efd 100755 --- a/pkg/commands/copy_test.go +++ b/pkg/commands/copy_test.go @@ -16,6 +16,7 @@ limitations under the License. package commands import ( + "archive/tar" "fmt" "io" "io/ioutil" @@ -105,6 +106,160 @@ func setupTestTemp() string { return tempDir } + +func Test_CachingCopyCommand_ExecuteCommand(t *testing.T) { + tempDir := setupTestTemp() + + tarContent, err := prepareTarFixture([]string{"foo.txt"}) + if err != nil { + t.Errorf("couldn't prepare tar fixture %v", err) + } + + config := &v1.Config{} + buildArgs := &dockerfile.BuildArgs{} + + type testCase struct { + desctiption string + expectLayer bool + expectErr bool + count *int + expectedCount int + command *CachingCopyCommand + extractedFiles []string + contextFiles []string + } + testCases := []testCase{ + func() testCase { + err = ioutil.WriteFile(filepath.Join(tempDir, "foo.txt"), []byte("meow"), 0644) + if err != nil { + t.Errorf("couldn't write tempfile %v", err) + t.FailNow() + } + + c := &CachingCopyCommand{ + img: fakeImage{ + ImageLayers: []v1.Layer{ + fakeLayer{TarContent: tarContent}, + }, + }, + fileContext: util.FileContext{Root: tempDir}, + cmd: &instructions.CopyCommand{ + SourcesAndDest: []string{ + "foo.txt", "foo.txt", + }, + }, + } + count := 0 + tc := testCase{ + desctiption: "with valid image and valid layer", + count: &count, + expectedCount: 1, + expectLayer: true, + extractedFiles: []string{"/foo.txt"}, + contextFiles: []string{"foo.txt"}, + } + c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error { + *tc.count++ + return nil + } + tc.command = c + return tc + }(), + func() testCase { + c := &CachingCopyCommand{} + tc := testCase{ + desctiption: "with no image", + expectErr: true, + } + c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error { + return nil + } + tc.command = c + return tc + }(), + func() testCase { + c := &CachingCopyCommand{ + img: fakeImage{}, + } + c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error { + return nil + } + return testCase{ + desctiption: "with image containing no layers", + expectErr: true, + command: c, + } + }(), + func() testCase { + c := &CachingCopyCommand{ + img: fakeImage{ + ImageLayers: []v1.Layer{ + fakeLayer{}, + }, + }, + } + c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error { + return nil + } + tc := testCase{ + desctiption: "with image one layer which has no tar content", + expectErr: false, // this one probably should fail but doesn't because of how ExecuteCommand and util.GetFSFromLayers are implemented - cvgw- 2019-11-25 + expectLayer: true, + } + tc.command = c + return tc + }(), + } + + for _, tc := range testCases { + t.Run(tc.desctiption, func(t *testing.T) { + c := tc.command + err := c.ExecuteCommand(config, buildArgs) + if !tc.expectErr && err != nil { + t.Errorf("Expected err to be nil but was %v", err) + } else if tc.expectErr && err == nil { + t.Error("Expected err but was nil") + } + + if tc.count != nil { + if *tc.count != tc.expectedCount { + t.Errorf("Expected extractFn to be called %v times but was called %v times", tc.expectedCount, *tc.count) + } + for _, file := range tc.extractedFiles { + match := false + cFiles := c.FilesToSnapshot() + for _, cFile := range cFiles { + if file == cFile { + match = true + break + } + } + if !match { + t.Errorf("Expected extracted files to include %v but did not %v", file, cFiles) + } + } + + cmdFiles, err := c.FilesUsedFromContext( + config, buildArgs, + ) + if err != nil { + t.Errorf("failed to get files used from context from command %v", err) + } + + if len(cmdFiles) != len(tc.contextFiles) { + t.Errorf("expected files used from context to equal %v but was %v", tc.contextFiles, cmdFiles) + } + } + + if c.layer == nil && tc.expectLayer { + t.Error("expected the command to have a layer set but instead was nil") + } else if c.layer != nil && !tc.expectLayer { + t.Error("expected the command to have no layer set but instead found a layer") + } + }) + } +} + func TestCopyExecuteCmd(t *testing.T) { tempDir := setupTestTemp() defer os.RemoveAll(tempDir) diff --git a/pkg/config/options.go b/pkg/config/options.go index ca36a435e2..07b5391889 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -68,6 +68,7 @@ type KanikoOptions struct { IgnoreVarRun bool SkipUnusedStages bool RunV2 bool + CacheCopyLayers bool Git KanikoGitOptions } diff --git a/pkg/executor/build.go b/pkg/executor/build.go index ca2b038917..723180a152 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -127,7 +127,7 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, cross } for _, cmd := range s.stage.Commands { - command, err := commands.GetCommand(cmd, fileContext, opts.RunV2) + command, err := commands.GetCommand(cmd, fileContext, opts.RunV2, opts.CacheCopyLayers) if err != nil { return nil, err } @@ -184,6 +184,7 @@ func (s *stageBuilder) populateCompositeKey(command fmt.Stringer, files []string compositeKey.AddKey(resolvedCmd) switch v := command.(type) { case *commands.CopyCommand: + case *commands.CachingCopyCommand: compositeKey = s.populateCopyCmdCompositeKey(command, v.From(), compositeKey) } diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index 351755a87c..4cb872c4da 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -795,6 +795,135 @@ func Test_stageBuilder_build(t *testing.T) { retrieve: true, }, }, + func() testcase { + dir, filenames := tempDirAndFile(t) + filename := filenames[0] + filepath := filepath.Join(dir, filename) + + tarContent := generateTar(t, dir, filename) + + ch := NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename)) + ch.AddPath(filepath, util.FileContext{}) + + hash, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + copyCommandCacheKey := hash + dockerFile := fmt.Sprintf(` +FROM ubuntu:16.04 +COPY %s foo.txt +`, filename) + f, _ := ioutil.TempFile("", "") + ioutil.WriteFile(f.Name(), []byte(dockerFile), 0755) + opts := &config.KanikoOptions{ + DockerfilePath: f.Name(), + Cache: true, + CacheCopyLayers: true, + } + + testStages, metaArgs, err := dockerfile.ParseStages(opts) + if err != nil { + t.Errorf("Failed to parse test dockerfile to stages: %s", err) + } + + kanikoStages, err := dockerfile.MakeKanikoStages(opts, testStages, metaArgs) + if err != nil { + t.Errorf("Failed to parse stages to Kaniko Stages: %s", err) + } + _ = ResolveCrossStageInstructions(kanikoStages) + stage := kanikoStages[0] + + cmds := stage.Commands + + return testcase{ + description: "copy command cache enabled and key in cache", + opts: opts, + image: fakeImage{ + ImageLayers: []v1.Layer{ + fakeLayer{ + TarContent: tarContent, + }, + }, + }, + layerCache: &fakeLayerCache{ + retrieve: true, + img: fakeImage{ + ImageLayers: []v1.Layer{ + fakeLayer{ + TarContent: tarContent, + }, + }, + }, + }, + rootDir: dir, + expectedCacheKeys: []string{copyCommandCacheKey}, + // CachingCopyCommand is not pushed to the cache + pushedCacheKeys: []string{}, + commands: getCommands(util.FileContext{Root: dir}, cmds, true), + fileName: filename, + } + }(), + func() testcase { + dir, filenames := tempDirAndFile(t) + filename := filenames[0] + tarContent := []byte{} + destDir, err := ioutil.TempDir("", "baz") + if err != nil { + t.Errorf("could not create temp dir %v", err) + } + filePath := filepath.Join(dir, filename) + ch := NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename)) + ch.AddPath(filePath, util.FileContext{}) + + hash, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + dockerFile := fmt.Sprintf(` +FROM ubuntu:16.04 +COPY %s foo.txt +`, filename) + f, _ := ioutil.TempFile("", "") + ioutil.WriteFile(f.Name(), []byte(dockerFile), 0755) + opts := &config.KanikoOptions{ + DockerfilePath: f.Name(), + Cache: true, + CacheCopyLayers: true, + } + + testStages, metaArgs, err := dockerfile.ParseStages(opts) + if err != nil { + t.Errorf("Failed to parse test dockerfile to stages: %s", err) + } + + kanikoStages, err := dockerfile.MakeKanikoStages(opts, testStages, metaArgs) + if err != nil { + t.Errorf("Failed to parse stages to Kaniko Stages: %s", err) + } + _ = ResolveCrossStageInstructions(kanikoStages) + stage := kanikoStages[0] + + cmds := stage.Commands + return testcase{ + description: "copy command cache enabled and key is not in cache", + opts: opts, + config: &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}}, + layerCache: &fakeLayerCache{}, + image: fakeImage{ + ImageLayers: []v1.Layer{ + fakeLayer{ + TarContent: tarContent, + }, + }, + }, + rootDir: dir, + expectedCacheKeys: []string{hash}, + pushedCacheKeys: []string{hash}, + commands: getCommands(util.FileContext{Root: dir}, cmds, true), + fileName: filename, + } + }(), func() testcase { dir, filenames := tempDirAndFile(t) filename := filenames[0] @@ -804,14 +933,26 @@ func Test_stageBuilder_build(t *testing.T) { if err != nil { t.Errorf("could not create temp dir %v", err) } + filePath := filepath.Join(dir, filename) - ch := NewCompositeCache("", fmt.Sprintf("RUN foobar")) + ch := NewCompositeCache("", "RUN foobar") hash1, err := ch.Hash() if err != nil { t.Errorf("couldn't create hash %v", err) } + ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename)) + ch.AddPath(filePath, util.FileContext{}) + + hash2, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + ch = NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename)) + ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename)) + ch.AddPath(filePath, util.FileContext{}) + image := fakeImage{ ImageLayers: []v1.Layer{ fakeLayer{ @@ -845,8 +986,8 @@ COPY %s bar.txt cmds := stage.Commands return testcase{ - description: "cached run command followed by copy command results in consistent read and write hashes", - opts: &config.KanikoOptions{Cache: true}, + description: "cached run command followed by uncached copy command results in consistent read and write hashes", + opts: &config.KanikoOptions{Cache: true, CacheCopyLayers: true}, rootDir: dir, config: &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}}, layerCache: &fakeLayerCache{ @@ -855,9 +996,9 @@ COPY %s bar.txt }, image: image, // hash1 is the read cachekey for the first layer - expectedCacheKeys: []string{hash1}, - pushedCacheKeys: []string{}, - commands: getCommands(util.FileContext{Root: dir}, cmds), + expectedCacheKeys: []string{hash1, hash2}, + pushedCacheKeys: []string{hash2}, + commands: getCommands(util.FileContext{Root: dir}, cmds, true), } }(), func() testcase { @@ -933,7 +1074,7 @@ RUN foobar image: image, expectedCacheKeys: []string{runHash}, pushedCacheKeys: []string{}, - commands: getCommands(util.FileContext{Root: dir}, cmds), + commands: getCommands(util.FileContext{Root: dir}, cmds, false), } }(), func() testcase { @@ -1107,7 +1248,7 @@ RUN foobar if err != nil { t.Errorf("Expected error to be nil but was %v", err) } - + fmt.Println(lc.receivedKeys) assertCacheKeys(t, tc.expectedCacheKeys, lc.receivedKeys, "receive") assertCacheKeys(t, tc.pushedCacheKeys, keys, "push") @@ -1140,13 +1281,14 @@ func assertCacheKeys(t *testing.T, expectedCacheKeys, actualCacheKeys []string, } } -func getCommands(fileContext util.FileContext, cmds []instructions.Command) []commands.DockerCommand { +func getCommands(fileContext util.FileContext, cmds []instructions.Command, cacheCopy bool) []commands.DockerCommand { outCommands := make([]commands.DockerCommand, 0) for _, c := range cmds { cmd, err := commands.GetCommand( c, fileContext, false, + cacheCopy, ) if err != nil { panic(err) diff --git a/pkg/executor/push.go b/pkg/executor/push.go index 874c724d8f..d876677896 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -252,6 +252,7 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { } } timing.DefaultRun.Stop(t) + logrus.Infof("Pushed images to %d destinations", len(destRefs)) return writeImageOutputs(image, destRefs) }