diff --git a/pkg/commands/add.go b/pkg/commands/add.go index dfc7f63f8d..b0f07a5996 100644 --- a/pkg/commands/add.go +++ b/pkg/commands/add.go @@ -29,6 +29,7 @@ import ( ) type AddCommand struct { + BaseCommand cmd *instructions.AddCommand buildcontext string snapshotFiles []string @@ -110,7 +111,6 @@ func (a *AddCommand) String() string { return a.cmd.String() } -// CacheCommand returns false since this command shouldn't be cached -func (a *AddCommand) CacheCommand() bool { - return false +func (a *AddCommand) UsesContext() bool { + return true } diff --git a/pkg/commands/arg.go b/pkg/commands/arg.go index 6174826b42..9f5533fcfc 100644 --- a/pkg/commands/arg.go +++ b/pkg/commands/arg.go @@ -24,6 +24,7 @@ import ( ) type ArgCommand struct { + BaseCommand cmd *instructions.ArgCommand } @@ -46,17 +47,7 @@ func (r *ArgCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui return nil } -// FilesToSnapshot returns an empty array since this command only touches metadata. -func (r *ArgCommand) FilesToSnapshot() []string { - return []string{} -} - // String returns some information about the command for the image config history func (r *ArgCommand) String() string { return r.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (r *ArgCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/base_command.go b/pkg/commands/base_command.go new file mode 100644 index 0000000000..bcb6448c5f --- /dev/null +++ b/pkg/commands/base_command.go @@ -0,0 +1,34 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +type BaseCommand struct { + cache bool + usesContext bool +} + +func (b *BaseCommand) CacheCommand() bool { + return b.cache +} + +func (b *BaseCommand) UsesContext() bool { + return b.usesContext +} + +func (b *BaseCommand) FilesToSnapshot() []string { + return []string{} +} diff --git a/pkg/commands/cmd.go b/pkg/commands/cmd.go index 3087552002..87714430ba 100644 --- a/pkg/commands/cmd.go +++ b/pkg/commands/cmd.go @@ -26,6 +26,7 @@ import ( ) type CmdCommand struct { + BaseCommand cmd *instructions.CmdCommand } @@ -52,17 +53,7 @@ func (c *CmdCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui return nil } -// FilesToSnapshot returns an empty array since this is a metadata command -func (c *CmdCommand) FilesToSnapshot() []string { - return []string{} -} - // String returns some information about the command for the image config history func (c *CmdCommand) String() string { return c.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (c *CmdCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/cmd_test.go b/pkg/commands/cmd_test.go index 1b8d2a455f..b56786ad19 100644 --- a/pkg/commands/cmd_test.go +++ b/pkg/commands/cmd_test.go @@ -48,7 +48,7 @@ func TestExecuteCmd(t *testing.T) { for _, test := range cmdTests { cmd := CmdCommand{ - &instructions.CmdCommand{ + cmd: &instructions.CmdCommand{ ShellDependantCmdLine: instructions.ShellDependantCmdLine{ PrependShell: test.prependShell, CmdLine: test.cmdLine, diff --git a/pkg/commands/commands.go b/pkg/commands/commands.go index 63cb388a20..4222727652 100644 --- a/pkg/commands/commands.go +++ b/pkg/commands/commands.go @@ -37,6 +37,9 @@ type DockerCommand interface { // Return true if this command should be true // Currently only true for RUN CacheCommand() bool + + // Return true if this command depends on the build context. + UsesContext() bool } func GetCommand(cmd instructions.Command, buildcontext string) (DockerCommand, error) { diff --git a/pkg/commands/copy.go b/pkg/commands/copy.go index 8b34d1b6b2..7bb582ea63 100644 --- a/pkg/commands/copy.go +++ b/pkg/commands/copy.go @@ -29,6 +29,7 @@ import ( ) type CopyCommand struct { + BaseCommand cmd *instructions.CopyCommand buildcontext string snapshotFiles []string @@ -103,7 +104,6 @@ func (c *CopyCommand) String() string { return c.cmd.String() } -// CacheCommand returns true since this command should be cached -func (c *CopyCommand) CacheCommand() bool { - return false +func (c *CopyCommand) UsesContext() bool { + return true } diff --git a/pkg/commands/entrypoint.go b/pkg/commands/entrypoint.go index 5e403e23d2..23f349347d 100644 --- a/pkg/commands/entrypoint.go +++ b/pkg/commands/entrypoint.go @@ -26,6 +26,7 @@ import ( ) type EntrypointCommand struct { + BaseCommand cmd *instructions.EntrypointCommand } @@ -50,17 +51,7 @@ func (e *EntrypointCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerf return nil } -// FilesToSnapshot returns an empty array since this is a metadata command -func (e *EntrypointCommand) FilesToSnapshot() []string { - return []string{} -} - // String returns some information about the command for the image config history func (e *EntrypointCommand) String() string { return e.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (e *EntrypointCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/entrypoint_test.go b/pkg/commands/entrypoint_test.go index 6508c70604..5343d67fe7 100644 --- a/pkg/commands/entrypoint_test.go +++ b/pkg/commands/entrypoint_test.go @@ -48,7 +48,7 @@ func TestEntrypointExecuteCmd(t *testing.T) { for _, test := range entrypointTests { cmd := EntrypointCommand{ - &instructions.EntrypointCommand{ + cmd: &instructions.EntrypointCommand{ ShellDependantCmdLine: instructions.ShellDependantCmdLine{ PrependShell: test.prependShell, CmdLine: test.cmdLine, diff --git a/pkg/commands/env.go b/pkg/commands/env.go index 249d438edd..5286f72608 100644 --- a/pkg/commands/env.go +++ b/pkg/commands/env.go @@ -25,6 +25,7 @@ import ( ) type EnvCommand struct { + BaseCommand cmd *instructions.EnvCommand } @@ -34,17 +35,7 @@ func (e *EnvCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui return util.UpdateConfigEnv(newEnvs, config, replacementEnvs) } -// We know that no files have changed, so return an empty array -func (e *EnvCommand) FilesToSnapshot() []string { - return []string{} -} - // String returns some information about the command for the image config history func (e *EnvCommand) String() string { return e.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (e *EnvCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/env_test.go b/pkg/commands/env_test.go index d75993bce8..d4870bb1ee 100644 --- a/pkg/commands/env_test.go +++ b/pkg/commands/env_test.go @@ -33,7 +33,7 @@ func Test_EnvExecute(t *testing.T) { } envCmd := &EnvCommand{ - &instructions.EnvCommand{ + cmd: &instructions.EnvCommand{ Env: []instructions.KeyValuePair{ { Key: "path", diff --git a/pkg/commands/expose.go b/pkg/commands/expose.go index 1797b50103..fa497f17d4 100644 --- a/pkg/commands/expose.go +++ b/pkg/commands/expose.go @@ -29,6 +29,7 @@ import ( ) type ExposeCommand struct { + BaseCommand cmd *instructions.ExposeCommand } @@ -72,15 +73,6 @@ func validProtocol(protocol string) bool { return false } -func (r *ExposeCommand) FilesToSnapshot() []string { - return []string{} -} - func (r *ExposeCommand) String() string { return r.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (r *ExposeCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/expose_test.go b/pkg/commands/expose_test.go index bb5d71eccd..02ad175763 100644 --- a/pkg/commands/expose_test.go +++ b/pkg/commands/expose_test.go @@ -48,7 +48,7 @@ func TestUpdateExposedPorts(t *testing.T) { } exposeCmd := &ExposeCommand{ - &instructions.ExposeCommand{ + cmd: &instructions.ExposeCommand{ Ports: ports, }, } @@ -77,7 +77,7 @@ func TestInvalidProtocol(t *testing.T) { } exposeCmd := &ExposeCommand{ - &instructions.ExposeCommand{ + cmd: &instructions.ExposeCommand{ Ports: ports, }, } diff --git a/pkg/commands/healthcheck.go b/pkg/commands/healthcheck.go index 380953e73f..610d9c935a 100644 --- a/pkg/commands/healthcheck.go +++ b/pkg/commands/healthcheck.go @@ -23,6 +23,7 @@ import ( ) type HealthCheckCommand struct { + BaseCommand cmd *instructions.HealthCheckCommand } @@ -34,17 +35,7 @@ func (h *HealthCheckCommand) ExecuteCommand(config *v1.Config, buildArgs *docker return nil } -// FilesToSnapshot returns an empty array since this is a metadata command -func (h *HealthCheckCommand) FilesToSnapshot() []string { - return []string{} -} - // String returns some information about the command for the image config history func (h *HealthCheckCommand) String() string { return h.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (h *HealthCheckCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/label.go b/pkg/commands/label.go index dae43546b9..c8790824bf 100644 --- a/pkg/commands/label.go +++ b/pkg/commands/label.go @@ -26,6 +26,7 @@ import ( ) type LabelCommand struct { + BaseCommand cmd *instructions.LabelCommand } @@ -64,17 +65,7 @@ func updateLabels(labels []instructions.KeyValuePair, config *v1.Config, buildAr } -// No files have changed, this command only touches metadata. -func (r *LabelCommand) FilesToSnapshot() []string { - return []string{} -} - // String returns some information about the command for the image config history func (r *LabelCommand) String() string { return r.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (r *LabelCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/onbuild.go b/pkg/commands/onbuild.go index b7b8728d32..0d36afe4b4 100644 --- a/pkg/commands/onbuild.go +++ b/pkg/commands/onbuild.go @@ -24,6 +24,7 @@ import ( ) type OnBuildCommand struct { + BaseCommand cmd *instructions.OnbuildCommand } @@ -39,17 +40,7 @@ func (o *OnBuildCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile return nil } -// FilesToSnapshot returns that no files have changed, this command only touches metadata. -func (o *OnBuildCommand) FilesToSnapshot() []string { - return []string{} -} - // String returns some information about the command for the image config history func (o *OnBuildCommand) String() string { return o.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (o *OnBuildCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/onbuild_test.go b/pkg/commands/onbuild_test.go index dd1d166d2c..10ee9ba260 100644 --- a/pkg/commands/onbuild_test.go +++ b/pkg/commands/onbuild_test.go @@ -60,7 +60,7 @@ func TestExecuteOnbuild(t *testing.T) { } onbuildCmd := &OnBuildCommand{ - &instructions.OnbuildCommand{ + cmd: &instructions.OnbuildCommand{ Expression: test.expression, }, } diff --git a/pkg/commands/run.go b/pkg/commands/run.go index 7cfe7b4864..dba6196804 100644 --- a/pkg/commands/run.go +++ b/pkg/commands/run.go @@ -35,6 +35,7 @@ import ( ) type RunCommand struct { + BaseCommand cmd *instructions.RunCommand } @@ -142,17 +143,15 @@ func addDefaultHOME(u string, envs []string) []string { return append(envs, home) } -// FilesToSnapshot returns nil for this command because we don't know which files -// have changed, so we snapshot the entire system. -func (r *RunCommand) FilesToSnapshot() []string { - return nil -} - // String returns some information about the command for the image config func (r *RunCommand) String() string { return r.cmd.String() } +func (r *RunCommand) FilesToSnapshot() []string { + return nil +} + // CacheCommand returns true since this command should be cached func (r *RunCommand) CacheCommand() bool { return true diff --git a/pkg/commands/shell.go b/pkg/commands/shell.go index f1bc1ab0bb..88b7310d9e 100644 --- a/pkg/commands/shell.go +++ b/pkg/commands/shell.go @@ -23,6 +23,7 @@ import ( ) type ShellCommand struct { + BaseCommand cmd *instructions.ShellCommand } @@ -32,17 +33,7 @@ func (s *ShellCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.B return nil } -// FilesToSnapshot returns an empty array since this is a metadata command -func (s *ShellCommand) FilesToSnapshot() []string { - return []string{} -} - // String returns some information about the command for the image config history func (s *ShellCommand) String() string { return s.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (s *ShellCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/shell_test.go b/pkg/commands/shell_test.go index 2e604c90e0..75e9eaa7b5 100644 --- a/pkg/commands/shell_test.go +++ b/pkg/commands/shell_test.go @@ -45,7 +45,7 @@ func TestShellExecuteCmd(t *testing.T) { for _, test := range shellTests { cmd := ShellCommand{ - &instructions.ShellCommand{ + cmd: &instructions.ShellCommand{ Shell: test.cmdLine, }, } diff --git a/pkg/commands/stopsignal.go b/pkg/commands/stopsignal.go index 87341f57fd..476376c87e 100644 --- a/pkg/commands/stopsignal.go +++ b/pkg/commands/stopsignal.go @@ -26,6 +26,7 @@ import ( ) type StopSignalCommand struct { + BaseCommand cmd *instructions.StopSignalCommand } @@ -52,17 +53,7 @@ func (s *StopSignalCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerf return nil } -// FilesToSnapshot returns an empty array since this is a metadata command -func (s *StopSignalCommand) FilesToSnapshot() []string { - return []string{} -} - // String returns some information about the command for the image config history func (s *StopSignalCommand) String() string { return s.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (s *StopSignalCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/stopsignal_test.go b/pkg/commands/stopsignal_test.go index 3c1a8f777b..66d5de3dd2 100644 --- a/pkg/commands/stopsignal_test.go +++ b/pkg/commands/stopsignal_test.go @@ -51,7 +51,7 @@ func TestStopsignalExecuteCmd(t *testing.T) { for _, test := range stopsignalTests { cmd := StopSignalCommand{ - &instructions.StopSignalCommand{ + cmd: &instructions.StopSignalCommand{ Signal: test.signal, }, } diff --git a/pkg/commands/user.go b/pkg/commands/user.go index d957ddfa30..9bc2228367 100644 --- a/pkg/commands/user.go +++ b/pkg/commands/user.go @@ -27,6 +27,7 @@ import ( ) type UserCommand struct { + BaseCommand cmd *instructions.UserCommand } @@ -59,15 +60,6 @@ func (r *UserCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bu return nil } -func (r *UserCommand) FilesToSnapshot() []string { - return []string{} -} - func (r *UserCommand) String() string { return r.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (r *UserCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/user_test.go b/pkg/commands/user_test.go index 5e04b135a8..6770d127bb 100644 --- a/pkg/commands/user_test.go +++ b/pkg/commands/user_test.go @@ -91,7 +91,7 @@ func TestUpdateUser(t *testing.T) { }, } cmd := UserCommand{ - &instructions.UserCommand{ + cmd: &instructions.UserCommand{ User: test.user, }, } diff --git a/pkg/commands/volume.go b/pkg/commands/volume.go index 01c9fdab28..af2d956752 100644 --- a/pkg/commands/volume.go +++ b/pkg/commands/volume.go @@ -29,6 +29,7 @@ import ( ) type VolumeCommand struct { + BaseCommand cmd *instructions.VolumeCommand snapshotFiles []string } @@ -74,8 +75,3 @@ func (v *VolumeCommand) FilesToSnapshot() []string { func (v *VolumeCommand) String() string { return v.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (v *VolumeCommand) CacheCommand() bool { - return false -} diff --git a/pkg/commands/workdir.go b/pkg/commands/workdir.go index 1eaeb072e1..12a2f44d8e 100644 --- a/pkg/commands/workdir.go +++ b/pkg/commands/workdir.go @@ -29,6 +29,7 @@ import ( ) type WorkdirCommand struct { + BaseCommand cmd *instructions.WorkdirCommand snapshotFiles []string } @@ -66,8 +67,3 @@ func (w *WorkdirCommand) FilesToSnapshot() []string { func (w *WorkdirCommand) String() string { return w.cmd.String() } - -// CacheCommand returns false since this command shouldn't be cached -func (w *WorkdirCommand) CacheCommand() bool { - return false -} diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 50baad886e..e517dafa09 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -18,7 +18,6 @@ package executor import ( "bytes" - "encoding/json" "fmt" "io" "io/ioutil" @@ -85,23 +84,6 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage) (*sta }, nil } -// key will return a string representation of the build at the cmd -func (s *stageBuilder) key(cmd string) (string, error) { - fsKey, err := s.snapshotter.Key() - if err != nil { - return "", err - } - c := bytes.NewBuffer([]byte{}) - enc := json.NewEncoder(c) - enc.Encode(s.cf) - cf, err := util.SHA256(c) - if err != nil { - return "", err - } - logrus.Debugf("%s\n%s\n%s\n%s\n", s.baseImageDigest, fsKey, cf, cmd) - return util.SHA256(bytes.NewReader([]byte(s.baseImageDigest + fsKey + cf + cmd))) -} - // extractCachedLayer will extract the cached layer and append it to the config file func (s *stageBuilder) extractCachedLayer(layer v1.Image, createdBy string) error { logrus.Infof("Found cached layer, extracting to filesystem") @@ -139,6 +121,15 @@ func (s *stageBuilder) build(opts *config.KanikoOptions) error { return err } var volumes []string + + // Set the initial cache key to be the base image digest, the build args and the SrcContext. + compositeKey := NewCompositeCache(s.baseImageDigest) + contextHash, err := HashDir(opts.SrcContext) + if err != nil { + return err + } + compositeKey.AddKey(opts.BuildArgs...) + args := dockerfile.NewBuildArgs(opts.BuildArgs) for index, cmd := range s.stage.Commands { finalCmd := index == len(s.stage.Commands)-1 @@ -149,13 +140,21 @@ func (s *stageBuilder) build(opts *config.KanikoOptions) error { if command == nil { continue } + + // Add the next command to the cache key. + compositeKey.AddKey(command.String()) + if command.UsesContext() { + compositeKey.AddKey(contextHash) + } logrus.Info(command.String()) - cacheKey, err := s.key(command.String()) + + ck, err := compositeKey.Hash() if err != nil { - return errors.Wrap(err, "getting key") + return err } + if command.CacheCommand() && opts.Cache { - image, err := cache.RetrieveLayer(opts, cacheKey) + image, err := cache.RetrieveLayer(opts, ck) if err == nil { if err := s.extractCachedLayer(image, command.String()); err != nil { return errors.Wrap(err, "extracting cached layer") @@ -222,7 +221,7 @@ func (s *stageBuilder) build(opts *config.KanikoOptions) error { } // Push layer to cache now along with new config file if command.CacheCommand() && opts.Cache { - if err := pushLayerToCache(opts, cacheKey, layer, command.String()); err != nil { + if err := pushLayerToCache(opts, ck, layer, command.String()); err != nil { return err } } diff --git a/pkg/executor/composite_cache.go b/pkg/executor/composite_cache.go new file mode 100644 index 0000000000..0a832e6369 --- /dev/null +++ b/pkg/executor/composite_cache.go @@ -0,0 +1,76 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package executor + +import ( + "crypto/sha256" + "os" + "path/filepath" + "strings" + + "github.com/GoogleContainerTools/kaniko/pkg/util" +) + +// NewCompositeCache returns an initialized composite cache object. +func NewCompositeCache(initial ...string) *CompositeCache { + c := CompositeCache{ + keys: initial, + } + return &c +} + +// CompositeCache is a type that generates a cache key from a series of keys. +type CompositeCache struct { + keys []string +} + +// AddKey adds the specified key to the sequence. +func (s *CompositeCache) AddKey(k ...string) { + s.keys = append(s.keys, k...) +} + +// Key returns the human readable composite key as a string. +func (s *CompositeCache) Key() string { + return strings.Join(s.keys, "-") +} + +// Hash returns the composite key in a string SHA256 format. +func (s *CompositeCache) Hash() (string, error) { + return util.SHA256(strings.NewReader(s.Key())) +} + +// HashDir returns a hash of the directory. +func HashDir(p string) (string, error) { + sha := sha256.New() + if err := filepath.Walk(p, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + fileHash, err := util.CacheHasher()(path) + if err != nil { + return err + } + if _, err := sha.Write([]byte(fileHash)); err != nil { + return err + } + return nil + }); err != nil { + return "", err + } + + return string(sha.Sum(nil)), nil +}