From d6d6fef34f0451bd470ab609ad178f5a33f12b1f Mon Sep 17 00:00:00 2001 From: dlorenc Date: Mon, 1 Oct 2018 12:42:35 -0700 Subject: [PATCH] Refactor command interface a bit, only cache the context for commands that use it. --- pkg/commands/add.go | 6 +++--- pkg/commands/arg.go | 11 +---------- pkg/commands/base_command.go | 34 +++++++++++++++++++++++++++++++++ pkg/commands/cmd.go | 11 +---------- pkg/commands/cmd_test.go | 2 +- pkg/commands/commands.go | 3 +++ pkg/commands/copy.go | 6 +++--- pkg/commands/entrypoint.go | 11 +---------- pkg/commands/entrypoint_test.go | 2 +- pkg/commands/env.go | 11 +---------- pkg/commands/env_test.go | 2 +- pkg/commands/expose.go | 10 +--------- pkg/commands/expose_test.go | 2 +- pkg/commands/healthcheck.go | 11 +---------- pkg/commands/label.go | 11 +---------- pkg/commands/onbuild.go | 11 +---------- pkg/commands/run.go | 7 +------ pkg/commands/shell.go | 11 +---------- pkg/commands/stopsignal.go | 11 +---------- pkg/commands/user.go | 10 +--------- pkg/commands/volume.go | 6 +----- pkg/commands/workdir.go | 6 +----- pkg/executor/build.go | 6 +++++- pkg/executor/composite_cache.go | 29 ++++++++++++++-------------- 24 files changed, 80 insertions(+), 150 deletions(-) create mode 100644 pkg/commands/base_command.go diff --git a/pkg/commands/add.go b/pkg/commands/add.go index dfc7f63f8d..b0f07a5996 100644 --- a/pkg/commands/add.go +++ b/pkg/commands/add.go @@ -29,6 +29,7 @@ import ( ) type AddCommand struct { + BaseCommand cmd *instructions.AddCommand buildcontext string snapshotFiles []string @@ -110,7 +111,6 @@ func (a *AddCommand) String() string { return a.cmd.String() } -// CacheCommand returns false since this command shouldn't be cached -func (a *AddCommand) CacheCommand() bool { - return false +func (a *AddCommand) UsesContext() bool { + return true } diff --git a/pkg/commands/arg.go b/pkg/commands/arg.go index 6174826b42..9f5533fcfc 100644 --- a/pkg/commands/arg.go +++ b/pkg/commands/arg.go @@ -24,6 +24,7 @@ import ( ) type ArgCommand struct { + BaseCommand cmd *instructions.ArgCommand } @@ -46,17 +47,7 @@ func (r *ArgCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui return nil } -// FilesToSnapshot returns an empty array since this command only touches metadata. -func (r *ArgCommand) FilesToSnapshot() []string { - return []string{} -} - // String returns some information about the command for the image config history func (r *ArgCommand) String() string { return r.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (r *ArgCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/base_command.go b/pkg/commands/base_command.go new file mode 100644 index 0000000000..bcb6448c5f --- /dev/null +++ b/pkg/commands/base_command.go @@ -0,0 +1,34 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +type BaseCommand struct { + cache bool + usesContext bool +} + +func (b *BaseCommand) CacheCommand() bool { + return b.cache +} + +func (b *BaseCommand) UsesContext() bool { + return b.usesContext +} + +func (b *BaseCommand) FilesToSnapshot() []string { + return []string{} +} diff --git a/pkg/commands/cmd.go b/pkg/commands/cmd.go index 3087552002..87714430ba 100644 --- a/pkg/commands/cmd.go +++ b/pkg/commands/cmd.go @@ -26,6 +26,7 @@ import ( ) type CmdCommand struct { + BaseCommand cmd *instructions.CmdCommand } @@ -52,17 +53,7 @@ func (c *CmdCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui return nil } -// FilesToSnapshot returns an empty array since this is a metadata command -func (c *CmdCommand) FilesToSnapshot() []string { - return []string{} -} - // String returns some information about the command for the image config history func (c *CmdCommand) String() string { return c.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (c *CmdCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/cmd_test.go b/pkg/commands/cmd_test.go index 1b8d2a455f..b56786ad19 100644 --- a/pkg/commands/cmd_test.go +++ b/pkg/commands/cmd_test.go @@ -48,7 +48,7 @@ func TestExecuteCmd(t *testing.T) { for _, test := range cmdTests { cmd := CmdCommand{ - &instructions.CmdCommand{ + cmd: &instructions.CmdCommand{ ShellDependantCmdLine: instructions.ShellDependantCmdLine{ PrependShell: test.prependShell, CmdLine: test.cmdLine, diff --git a/pkg/commands/commands.go b/pkg/commands/commands.go index 63cb388a20..4222727652 100644 --- a/pkg/commands/commands.go +++ b/pkg/commands/commands.go @@ -37,6 +37,9 @@ type DockerCommand interface { // Return true if this command should be true // Currently only true for RUN CacheCommand() bool + + // Return true if this command depends on the build context. + UsesContext() bool } func GetCommand(cmd instructions.Command, buildcontext string) (DockerCommand, error) { diff --git a/pkg/commands/copy.go b/pkg/commands/copy.go index 8b34d1b6b2..7bb582ea63 100644 --- a/pkg/commands/copy.go +++ b/pkg/commands/copy.go @@ -29,6 +29,7 @@ import ( ) type CopyCommand struct { + BaseCommand cmd *instructions.CopyCommand buildcontext string snapshotFiles []string @@ -103,7 +104,6 @@ func (c *CopyCommand) String() string { return c.cmd.String() } -// CacheCommand returns true since this command should be cached -func (c *CopyCommand) CacheCommand() bool { - return false +func (c *CopyCommand) UsesContext() bool { + return true } diff --git a/pkg/commands/entrypoint.go b/pkg/commands/entrypoint.go index 5e403e23d2..23f349347d 100644 --- a/pkg/commands/entrypoint.go +++ b/pkg/commands/entrypoint.go @@ -26,6 +26,7 @@ import ( ) type EntrypointCommand struct { + BaseCommand cmd *instructions.EntrypointCommand } @@ -50,17 +51,7 @@ func (e *EntrypointCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerf return nil } -// FilesToSnapshot returns an empty array since this is a metadata command -func (e *EntrypointCommand) FilesToSnapshot() []string { - return []string{} -} - // String returns some information about the command for the image config history func (e *EntrypointCommand) String() string { return e.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (e *EntrypointCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/entrypoint_test.go b/pkg/commands/entrypoint_test.go index 6508c70604..5343d67fe7 100644 --- a/pkg/commands/entrypoint_test.go +++ b/pkg/commands/entrypoint_test.go @@ -48,7 +48,7 @@ func TestEntrypointExecuteCmd(t *testing.T) { for _, test := range entrypointTests { cmd := EntrypointCommand{ - &instructions.EntrypointCommand{ + cmd: &instructions.EntrypointCommand{ ShellDependantCmdLine: instructions.ShellDependantCmdLine{ PrependShell: test.prependShell, CmdLine: test.cmdLine, diff --git a/pkg/commands/env.go b/pkg/commands/env.go index 249d438edd..5286f72608 100644 --- a/pkg/commands/env.go +++ b/pkg/commands/env.go @@ -25,6 +25,7 @@ import ( ) type EnvCommand struct { + BaseCommand cmd *instructions.EnvCommand } @@ -34,17 +35,7 @@ func (e *EnvCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui return util.UpdateConfigEnv(newEnvs, config, replacementEnvs) } -// We know that no files have changed, so return an empty array -func (e *EnvCommand) FilesToSnapshot() []string { - return []string{} -} - // String returns some information about the command for the image config history func (e *EnvCommand) String() string { return e.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (e *EnvCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/env_test.go b/pkg/commands/env_test.go index d75993bce8..d4870bb1ee 100644 --- a/pkg/commands/env_test.go +++ b/pkg/commands/env_test.go @@ -33,7 +33,7 @@ func Test_EnvExecute(t *testing.T) { } envCmd := &EnvCommand{ - &instructions.EnvCommand{ + cmd: &instructions.EnvCommand{ Env: []instructions.KeyValuePair{ { Key: "path", diff --git a/pkg/commands/expose.go b/pkg/commands/expose.go index 1797b50103..fa497f17d4 100644 --- a/pkg/commands/expose.go +++ b/pkg/commands/expose.go @@ -29,6 +29,7 @@ import ( ) type ExposeCommand struct { + BaseCommand cmd *instructions.ExposeCommand } @@ -72,15 +73,6 @@ func validProtocol(protocol string) bool { return false } -func (r *ExposeCommand) FilesToSnapshot() []string { - return []string{} -} - func (r *ExposeCommand) String() string { return r.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (r *ExposeCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/expose_test.go b/pkg/commands/expose_test.go index bb5d71eccd..7cfb108082 100644 --- a/pkg/commands/expose_test.go +++ b/pkg/commands/expose_test.go @@ -77,7 +77,7 @@ func TestInvalidProtocol(t *testing.T) { } exposeCmd := &ExposeCommand{ - &instructions.ExposeCommand{ + cmd: &instructions.ExposeCommand{ Ports: ports, }, } diff --git a/pkg/commands/healthcheck.go b/pkg/commands/healthcheck.go index 380953e73f..610d9c935a 100644 --- a/pkg/commands/healthcheck.go +++ b/pkg/commands/healthcheck.go @@ -23,6 +23,7 @@ import ( ) type HealthCheckCommand struct { + BaseCommand cmd *instructions.HealthCheckCommand } @@ -34,17 +35,7 @@ func (h *HealthCheckCommand) ExecuteCommand(config *v1.Config, buildArgs *docker return nil } -// FilesToSnapshot returns an empty array since this is a metadata command -func (h *HealthCheckCommand) FilesToSnapshot() []string { - return []string{} -} - // String returns some information about the command for the image config history func (h *HealthCheckCommand) String() string { return h.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (h *HealthCheckCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/label.go b/pkg/commands/label.go index dae43546b9..c8790824bf 100644 --- a/pkg/commands/label.go +++ b/pkg/commands/label.go @@ -26,6 +26,7 @@ import ( ) type LabelCommand struct { + BaseCommand cmd *instructions.LabelCommand } @@ -64,17 +65,7 @@ func updateLabels(labels []instructions.KeyValuePair, config *v1.Config, buildAr } -// No files have changed, this command only touches metadata. -func (r *LabelCommand) FilesToSnapshot() []string { - return []string{} -} - // String returns some information about the command for the image config history func (r *LabelCommand) String() string { return r.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (r *LabelCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/onbuild.go b/pkg/commands/onbuild.go index b7b8728d32..0d36afe4b4 100644 --- a/pkg/commands/onbuild.go +++ b/pkg/commands/onbuild.go @@ -24,6 +24,7 @@ import ( ) type OnBuildCommand struct { + BaseCommand cmd *instructions.OnbuildCommand } @@ -39,17 +40,7 @@ func (o *OnBuildCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile return nil } -// FilesToSnapshot returns that no files have changed, this command only touches metadata. -func (o *OnBuildCommand) FilesToSnapshot() []string { - return []string{} -} - // String returns some information about the command for the image config history func (o *OnBuildCommand) String() string { return o.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (o *OnBuildCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/run.go b/pkg/commands/run.go index 7cfe7b4864..f7bb78f7a6 100644 --- a/pkg/commands/run.go +++ b/pkg/commands/run.go @@ -35,6 +35,7 @@ import ( ) type RunCommand struct { + BaseCommand cmd *instructions.RunCommand } @@ -142,12 +143,6 @@ func addDefaultHOME(u string, envs []string) []string { return append(envs, home) } -// FilesToSnapshot returns nil for this command because we don't know which files -// have changed, so we snapshot the entire system. -func (r *RunCommand) FilesToSnapshot() []string { - return nil -} - // String returns some information about the command for the image config func (r *RunCommand) String() string { return r.cmd.String() diff --git a/pkg/commands/shell.go b/pkg/commands/shell.go index f1bc1ab0bb..88b7310d9e 100644 --- a/pkg/commands/shell.go +++ b/pkg/commands/shell.go @@ -23,6 +23,7 @@ import ( ) type ShellCommand struct { + BaseCommand cmd *instructions.ShellCommand } @@ -32,17 +33,7 @@ func (s *ShellCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.B return nil } -// FilesToSnapshot returns an empty array since this is a metadata command -func (s *ShellCommand) FilesToSnapshot() []string { - return []string{} -} - // String returns some information about the command for the image config history func (s *ShellCommand) String() string { return s.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (s *ShellCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/stopsignal.go b/pkg/commands/stopsignal.go index 87341f57fd..476376c87e 100644 --- a/pkg/commands/stopsignal.go +++ b/pkg/commands/stopsignal.go @@ -26,6 +26,7 @@ import ( ) type StopSignalCommand struct { + BaseCommand cmd *instructions.StopSignalCommand } @@ -52,17 +53,7 @@ func (s *StopSignalCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerf return nil } -// FilesToSnapshot returns an empty array since this is a metadata command -func (s *StopSignalCommand) FilesToSnapshot() []string { - return []string{} -} - // String returns some information about the command for the image config history func (s *StopSignalCommand) String() string { return s.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (s *StopSignalCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/user.go b/pkg/commands/user.go index d957ddfa30..9bc2228367 100644 --- a/pkg/commands/user.go +++ b/pkg/commands/user.go @@ -27,6 +27,7 @@ import ( ) type UserCommand struct { + BaseCommand cmd *instructions.UserCommand } @@ -59,15 +60,6 @@ func (r *UserCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bu return nil } -func (r *UserCommand) FilesToSnapshot() []string { - return []string{} -} - func (r *UserCommand) String() string { return r.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (r *UserCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/volume.go b/pkg/commands/volume.go index 5c9ecd99c2..7a01255609 100644 --- a/pkg/commands/volume.go +++ b/pkg/commands/volume.go @@ -29,6 +29,7 @@ import ( ) type VolumeCommand struct { + BaseCommand cmd *instructions.VolumeCommand snapshotFiles []string } @@ -74,8 +75,3 @@ func (v *VolumeCommand) FilesToSnapshot() []string { func (v *VolumeCommand) String() string { return v.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (v *VolumeCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/workdir.go b/pkg/commands/workdir.go index 1eaeb072e1..12a2f44d8e 100644 --- a/pkg/commands/workdir.go +++ b/pkg/commands/workdir.go @@ -29,6 +29,7 @@ import ( ) type WorkdirCommand struct { + BaseCommand cmd *instructions.WorkdirCommand snapshotFiles []string } @@ -66,8 +67,3 @@ func (w *WorkdirCommand) FilesToSnapshot() []string { func (w *WorkdirCommand) String() string { return w.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (w *WorkdirCommand) CacheCommand() bool { - return false -} diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 9931cae572..0e3278392a 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -123,7 +123,8 @@ func (s *stageBuilder) build(opts *config.KanikoOptions) error { // Set the initial cache key to be the base image digest, the build args and the SrcContext. compositeKey := NewCompositeCache(s.baseImageDigest) - if err := compositeKey.AddDir(opts.SrcContext); err != nil { + contextHash, err := HashDir(opts.SrcContext) + if err != nil { return err } compositeKey.AddKey(opts.BuildArgs...) @@ -141,6 +142,9 @@ func (s *stageBuilder) build(opts *config.KanikoOptions) error { // Add the next command to the cache key. compositeKey.AddKey(command.String()) + if command.UsesContext() { + compositeKey.AddKey(contextHash) + } logrus.Info(command.String()) ck, err := compositeKey.Hash() diff --git a/pkg/executor/composite_cache.go b/pkg/executor/composite_cache.go index b2762ee484..0a832e6369 100644 --- a/pkg/executor/composite_cache.go +++ b/pkg/executor/composite_cache.go @@ -43,8 +43,18 @@ func (s *CompositeCache) AddKey(k ...string) { s.keys = append(s.keys, k...) } -// AddDir adds the contents of a directory to the composite key. -func (s *CompositeCache) AddDir(p string) error { +// Key returns the human readable composite key as a string. +func (s *CompositeCache) Key() string { + return strings.Join(s.keys, "-") +} + +// Hash returns the composite key in a string SHA256 format. +func (s *CompositeCache) Hash() (string, error) { + return util.SHA256(strings.NewReader(s.Key())) +} + +// HashDir returns a hash of the directory. +func HashDir(p string) (string, error) { sha := sha256.New() if err := filepath.Walk(p, func(path string, fi os.FileInfo, err error) error { if err != nil { @@ -59,19 +69,8 @@ func (s *CompositeCache) AddDir(p string) error { } return nil }); err != nil { - return err + return "", err } - s.AddKey(string(sha.Sum(nil))) - return nil -} - -// Key returns the human readable composite key as a string. -func (s *CompositeCache) Key() string { - return strings.Join(s.keys, "-") -} - -// Hash returns the composite key in a string SHA256 format. -func (s *CompositeCache) Hash() (string, error) { - return util.SHA256(strings.NewReader(s.Key())) + return string(sha.Sum(nil)), nil }