From cb71b1729e0cc72e5068c1a1e9cdb3cc701e7c6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20N=C3=BCtzi?= Date: Thu, 12 May 2022 22:53:58 +0200 Subject: [PATCH 01/10] fix: Correct flatten function in layers - Added a test. - Cache current image, track deletes in `whiteouts` as well as normal adds in `layers`. - Fix ugly delete behavior of `layerHashCache`. Delete it when crerating a new snapshot. - Slight cleanup in `snapshot.go`. - Format ugly `WalkFS` function. --- pkg/snapshot/layered_map.go | 116 +++++++++++++++++++------------ pkg/snapshot/layered_map_test.go | 64 +++++++++++++++++ pkg/snapshot/snapshot.go | 85 ++++++++++++---------- pkg/util/fs_util.go | 80 ++++++++++++--------- 4 files changed, 233 insertions(+), 112 deletions(-) diff --git a/pkg/snapshot/layered_map.go b/pkg/snapshot/layered_map.go index 60e802b7b1..7e973793a0 100644 --- a/pkg/snapshot/layered_map.go +++ b/pkg/snapshot/layered_map.go @@ -20,19 +20,18 @@ import ( "bytes" "encoding/json" "fmt" - "os" - "path/filepath" - "strings" "github.com/GoogleContainerTools/kaniko/pkg/timing" "github.com/GoogleContainerTools/kaniko/pkg/util" - "github.com/docker/docker/pkg/archive" - "github.com/sirupsen/logrus" ) type LayeredMap struct { - layers []map[string]string - whiteouts []map[string]struct{} + layers []map[string]string // All layers with added files. + whiteouts []map[string]struct{} // All layers with deleted files. + + currentImage map[string]string // All files and hashes in the current image (up to the last layer). + isCurrentImageValid bool // If the currentImage is not out-of-date. + layerHashCache map[string]string hasher func(string) (string, error) // cacheHasher doesn't include mtime in it's hash so that filesystem cache keys are stable @@ -44,39 +43,57 @@ func NewLayeredMap(h func(string) (string, error), c func(string) (string, error hasher: h, cacheHasher: c, } - l.layers = []map[string]string{} + + l.currentImage = map[string]string{} l.layerHashCache = map[string]string{} return &l } func (l *LayeredMap) Snapshot() { + + // Save current state of image + l.UpdateCurrentImage() + l.whiteouts = append(l.whiteouts, map[string]struct{}{}) l.layers = append(l.layers, map[string]string{}) + l.layerHashCache = map[string]string{} // Erase the hash cache for this new layer. } // Key returns a hash for added files func (l *LayeredMap) Key() (string, error) { c := bytes.NewBuffer([]byte{}) enc := json.NewEncoder(c) - enc.Encode(l.layers) + err := enc.Encode(l.layers) + if err != nil { + return "", err + } return util.SHA256(c) } -// getFlattenedPaths returns all existing paths in the current FS -func (l *LayeredMap) getFlattenedPaths() map[string]struct{} { - paths := map[string]struct{}{} - for _, l := range l.layers { - for p := range l { - basename := filepath.Base(p) - if strings.HasPrefix(basename, archive.WhiteoutPrefix) { - deletedFile := filepath.Join(filepath.Dir(p), strings.TrimPrefix(basename, archive.WhiteoutPrefix)) - delete(paths, deletedFile) - } else { - paths[p] = struct{}{} - } +// UpdateCurrentImage computes the current image by +// flattening all layers and stores the result in currentImage. +func (l *LayeredMap) UpdateCurrentImage() { + if l.isCurrentImageValid { + return + } + + l.currentImage = map[string]string{} + + // Adding and deleting files over all layers. + for i := 0; i < len(l.layers); i++ { + addedFiles := l.layers[i] + deletedFiles := l.whiteouts[i] + + for add, hash := range addedFiles { + l.currentImage[add] = hash + } + + for del := range deletedFiles { + delete(l.currentImage, del) } } - return paths + + l.isCurrentImageValid = true } func (l *LayeredMap) Get(s string) (string, bool) { @@ -88,61 +105,70 @@ func (l *LayeredMap) Get(s string) (string, bool) { return "", false } -func (l *LayeredMap) GetWhiteout(s string) bool { - for i := len(l.whiteouts) - 1; i >= 0; i-- { - if _, ok := l.whiteouts[i][s]; ok { - return ok - } +// GetCurrentPaths returns all existing paths in the actual current image +// cached by FlattenLayers. +func (l *LayeredMap) GetCurrentPaths() map[string]struct{} { + l.UpdateCurrentImage() + + paths := map[string]struct{}{} + for f := range l.currentImage { + paths[f] = struct{}{} } - return false + return paths } -func (l *LayeredMap) MaybeAddWhiteout(s string) bool { - ok := l.GetWhiteout(s) - if ok { - return false - } +// AddWhiteout will delete the specific files in the current layer. +func (l *LayeredMap) AddWhiteout(s string) error { + l.isCurrentImageValid = false + l.whiteouts[len(l.whiteouts)-1][s] = struct{}{} - return true + return nil } -// Add will add the specified file s to the layered map. +// Add will add the specified file s to the current layer. func (l *LayeredMap) Add(s string) error { + l.isCurrentImageValid = false + // Use hash function and add to layers newV, err := func(s string) (string, error) { if v, ok := l.layerHashCache[s]; ok { - // clear it cache for next layer. - delete(l.layerHashCache, s) return v, nil } return l.hasher(s) }(s) + if err != nil { - return fmt.Errorf("error creating hash for %s: %v", s, err) + return fmt.Errorf("Error creating hash for %s: %w", s, err) } + l.layers[len(l.layers)-1][s] = newV return nil } -// CheckFileChange checks whether a given file changed +// CheckFileChange checks whether a given file (needs to exist) changed // from the current layered map by its hashing function. +// If the file does not exist, an error is returned. // Returns true if the file is changed. func (l *LayeredMap) CheckFileChange(s string) (bool, error) { t := timing.Start("Hashing files") defer timing.DefaultRun.Stop(t) + newV, err := l.hasher(s) if err != nil { - // if this file does not exist in the new layer return. - if os.IsNotExist(err) { - logrus.Tracef("%s detected as changed but does not exist", s) - return false, nil - } return false, err } + + // Save hash to not recompute it when + // adding the file. l.layerHashCache[s] = newV - oldV, ok := l.Get(s) + + oldV, ok := l.currentImage[s] if ok && newV == oldV { + // File hash did not change => Unchanged. return false, nil } + + // File does not exist in current image, + // or it did change => Changed. return true, nil } diff --git a/pkg/snapshot/layered_map_test.go b/pkg/snapshot/layered_map_test.go index bedfbd10a5..d42003d0bd 100644 --- a/pkg/snapshot/layered_map_test.go +++ b/pkg/snapshot/layered_map_test.go @@ -77,3 +77,67 @@ func Test_CacheKey(t *testing.T) { }) } } + +func Test_FlattenPaths(t *testing.T) { + layers := []map[string]string{ + { + "a": "2", + "b": "3", + }, + { + "b": "5", + "c": "6", + }, + { + "a": "8", + }, + } + + whiteouts := []map[string]struct{}{ + { + "a": {}, // delete a + }, + { + "b": {}, // delete b + }, + { + "c": {}, // delete c + }, + } + + lm := LayeredMap{ + layers: []map[string]string{layers[0]}, + whiteouts: []map[string]struct{}{whiteouts[0]}} + + paths := lm.GetCurrentPaths() + + assertPath := func(f string, exists bool) { + _, ok := paths[f] + if exists && !ok { + t.Fatalf("expected path '%s' to be present.", f) + } else if !exists && ok { + t.Fatalf("expected path '%s' not to be present.", f) + } + } + + assertPath("a", false) + assertPath("b", true) + + lm = LayeredMap{ + layers: []map[string]string{layers[0], layers[1]}, + whiteouts: []map[string]struct{}{whiteouts[0], whiteouts[1]}} + paths = lm.GetCurrentPaths() + + assertPath("a", false) + assertPath("b", false) + assertPath("c", true) + + lm = LayeredMap{ + layers: []map[string]string{layers[0], layers[1], layers[2]}, + whiteouts: []map[string]struct{}{whiteouts[0], whiteouts[1], whiteouts[2]}} + paths = lm.GetCurrentPaths() + + assertPath("a", true) + assertPath("b", false) + assertPath("c", false) +} diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 50fa803a02..4d82f48ba0 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -49,6 +49,7 @@ func NewSnapshotter(l *LayeredMap, d string) *Snapshotter { // Init initializes a new snapshotter func (s *Snapshotter) Init() error { + logrus.Info("Initializing snapshotter ...") _, _, err := s.scanFullFilesystem() return err } @@ -75,41 +76,39 @@ func (s *Snapshotter) TakeSnapshot(files []string, shdCheckDelete bool, forceBui filesToAdd, err := filesystem.ResolvePaths(files, s.ignorelist) if err != nil { - return "", nil + return "", err } logrus.Info("Taking snapshot of files...") - logrus.Debugf("Taking snapshot of files %v", filesToAdd) sort.Strings(filesToAdd) + logrus.Debugf("Adding to layer: %v", filesToAdd) - // Add files to the layered map + // Add files to current layer. for _, file := range filesToAdd { if err := s.l.Add(file); err != nil { - return "", fmt.Errorf("unable to add file %s to layered map: %s", file, err) + return "", fmt.Errorf("Unable to add file %s to layered map: %w", file, err) } } // Get whiteout paths - filesToWhiteout := []string{} + var filesToWhiteout []string if shdCheckDelete { - _, deletedFiles := util.WalkFS(s.directory, s.l.getFlattenedPaths(), func(s string) (bool, error) { + _, deletedFiles := util.WalkFS(s.directory, s.l.GetCurrentPaths(), func(s string) (bool, error) { return true, nil }) - // The paths left here are the ones that have been deleted in this layer. - for path := range deletedFiles { - // Only add the whiteout if the directory for the file still exists. - dir := filepath.Dir(path) - if _, ok := deletedFiles[dir]; !ok { - if s.l.MaybeAddWhiteout(path) { - logrus.Debugf("Adding whiteout for %s", path) - filesToWhiteout = append(filesToWhiteout, path) - } + + logrus.Debugf("Deleting in layer: %v", deletedFiles) + // Whiteout files in current layer. + for file := range deletedFiles { + if err := s.l.AddWhiteout(file); err != nil { + return "", fmt.Errorf("Unable to whiteout file %s in layered map: %w", file, err) } } - } - sort.Strings(filesToWhiteout) + filesToWhiteout = removeObsoleteWhiteouts(deletedFiles) + sort.Strings(filesToWhiteout) + } t := util.NewTar(f) defer t.Close() @@ -159,7 +158,9 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { s.l.Snapshot() - changedPaths, deletedPaths := util.WalkFS(s.directory, s.l.getFlattenedPaths(), s.l.CheckFileChange) + logrus.Debugf("Current image filesystem: %v", s.l.currentImage) + + changedPaths, deletedPaths := util.WalkFS(s.directory, s.l.GetCurrentPaths(), s.l.CheckFileChange) timer := timing.Start("Resolving Paths") filesToAdd := []string{} @@ -169,41 +170,55 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { } for _, path := range resolvedFiles { if util.CheckIgnoreList(path) { - logrus.Tracef("Not adding %s to layer, as it's ignored", path) + logrus.Debugf("Not adding %s to layer, as it's ignored", path) continue } filesToAdd = append(filesToAdd, path) } - // The paths left here are the ones that have been deleted in this layer. - filesToWhiteOut := []string{} - for path := range deletedPaths { - // Only add the whiteout if the directory for the file still exists. - dir := filepath.Dir(path) - if _, ok := deletedPaths[dir]; !ok { - if s.l.MaybeAddWhiteout(path) { - logrus.Debugf("Adding whiteout for %s", path) - filesToWhiteOut = append(filesToWhiteOut, path) - } + logrus.Debugf("Adding to layer: %v", filesToAdd) + logrus.Debugf("Deleting in layer: %v", deletedPaths) + + // Add files to the layered map + for _, file := range filesToAdd { + if err := s.l.Add(file); err != nil { + return nil, nil, fmt.Errorf("Unable to add file %s to layered map: %w", file, err) } } + for file := range deletedPaths { + if err := s.l.AddWhiteout(file); err != nil { + return nil, nil, fmt.Errorf("Unable to whiteout file %s in layered map: %w", file, err) + } + } + + filesToWhiteout := removeObsoleteWhiteouts(deletedPaths) timing.DefaultRun.Stop(timer) sort.Strings(filesToAdd) - sort.Strings(filesToWhiteOut) + sort.Strings(filesToWhiteout) - // Add files to the layered map - for _, file := range filesToAdd { - if err := s.l.Add(file); err != nil { - return nil, nil, fmt.Errorf("unable to add file %s to layered map: %s", file, err) + return filesToAdd, filesToWhiteout, nil +} + +// removeObsoleteWhiteouts filters deleted files according to their parents delete status. +func removeObsoleteWhiteouts(deletedFiles map[string]struct{}) (filesToWhiteout []string) { + + for path := range deletedFiles { + // Only add the whiteout if the directory for the file still exists. + dir := filepath.Dir(path) + if _, ok := deletedFiles[dir]; !ok { + logrus.Tracef("Adding whiteout for %s", path) + filesToWhiteout = append(filesToWhiteout, path) } } - return filesToAdd, filesToWhiteOut, nil + + return filesToWhiteout } func writeToTar(t util.Tar, files, whiteouts []string) error { timer := timing.Start("Writing tar file") defer timing.DefaultRun.Stop(timer) + // Now create the tar. for _, path := range whiteouts { if err := t.Whiteout(path); err != nil { diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 426aeaf768..396830dfc9 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -178,17 +178,17 @@ func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, e dir := filepath.Dir(path) if strings.HasPrefix(base, archive.WhiteoutPrefix) { - logrus.Debugf("Whiting out %s", path) + logrus.Tracef("Whiting out %s", path) name := strings.TrimPrefix(base, archive.WhiteoutPrefix) path := filepath.Join(dir, name) if CheckIgnoreList(path) { - logrus.Debugf("Not deleting %s, as it's ignored", path) + logrus.Tracef("Not deleting %s, as it's ignored", path) continue } if childDirInIgnoreList(path) { - logrus.Debugf("Not deleting %s, as it contains a ignored path", path) + logrus.Tracef("Not deleting %s, as it contains a ignored path", path) continue } @@ -197,7 +197,7 @@ func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, e } if !cfg.includeWhiteout { - logrus.Debug("not including whiteout files") + logrus.Trace("not including whiteout files") continue } @@ -357,7 +357,7 @@ func ExtractFile(dest string, hdr *tar.Header, tr io.Reader) error { return err } if CheckIgnoreList(abs) { - logrus.Tracef("skipping symlink from %s to %s because %s is ignored", hdr.Linkname, path, hdr.Linkname) + logrus.Tracef("skipping link from %s to %s because %s is ignored", hdr.Linkname, path, hdr.Linkname) return nil } // The base directory for a link may not exist before it is created. @@ -995,22 +995,28 @@ type walkFSResult struct { existingPaths map[string]struct{} } -// WalkFS given a directory and list of existing files, -// returns a list of changed filed determined by changeFunc and a list -// of deleted files. -// It timesout after 90 mins. Can be configured via setting an environment variable +// WalkFS given a directory dir and list of existing files existingPaths, +// returns a list of changed files determined by `changeFunc` and a list +// of deleted files. Input existingPaths is changed inside this function and +// returned as deleted files map. +// It timesout after 90 mins which can be configured via setting an environment variable // SNAPSHOT_TIMEOUT in the kaniko pod definition. -func WalkFS(dir string, existingPaths map[string]struct{}, changeFunc func(string) (bool, error)) ([]string, map[string]struct{}) { +func WalkFS( + dir string, + existingPaths map[string]struct{}, + changeFunc func(string) (bool, error)) ([]string, map[string]struct{}) { + timeOutStr := os.Getenv(snapshotTimeout) if timeOutStr == "" { - logrus.Tracef("%s environment not set. Using default snapshot timeout %s", snapshotTimeout, defaultTimeout) + logrus.Tracef("Environment '%s' not set. Using default snapshot timeout '%s'", snapshotTimeout, defaultTimeout) timeOutStr = defaultTimeout } timeOut, err := time.ParseDuration(timeOutStr) if err != nil { - logrus.Fatalf("could not parse duration %s", timeOutStr) + logrus.Fatalf("Could not parse duration '%s'", timeOutStr) } timer := timing.Start("Walking filesystem with timeout") + ch := make(chan walkFSResult, 1) go func() { @@ -1031,28 +1037,38 @@ func WalkFS(dir string, existingPaths map[string]struct{}, changeFunc func(strin func gowalkDir(dir string, existingPaths map[string]struct{}, changeFunc func(string) (bool, error)) walkFSResult { foundPaths := make([]string, 0) - godirwalk.Walk(dir, &godirwalk.Options{ - Callback: func(path string, ent *godirwalk.Dirent) error { - logrus.Tracef("Analyzing path %s", path) - if IsInIgnoreList(path) { - if IsDestDir(path) { - logrus.Tracef("Skipping paths under %s, as it is a ignored directory", path) - return filepath.SkipDir - } - return nil - } - delete(existingPaths, path) - if t, err := changeFunc(path); err != nil { - return err - } else if t { - foundPaths = append(foundPaths, path) + deletedFiles := existingPaths // Make a reference. + + callback := func(path string, ent *godirwalk.Dirent) error { + logrus.Tracef("Analyzing path '%s'", path) + + if IsInIgnoreList(path) { + if IsDestDir(path) { + logrus.Tracef("Skipping paths under '%s', as it is an ignored directory", path) + return filepath.SkipDir } return nil - }, - Unsorted: true, - }, - ) - return walkFSResult{foundPaths, existingPaths} + } + + // File is existing on disk, remove it from deleted files. + delete(deletedFiles, path) + + if isChanged, err := changeFunc(path); err != nil { + return err + } else if isChanged { + foundPaths = append(foundPaths, path) + } + + return nil + } + + godirwalk.Walk(dir, + &godirwalk.Options{ + Callback: callback, + Unsorted: true, + }) + + return walkFSResult{foundPaths, deletedFiles} } // GetFSInfoMap given a directory gets a map of FileInfo for all files From f58df666902c41fad9fed4e044993efcf684b807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20N=C3=BCtzi?= Date: Thu, 12 May 2022 23:34:23 +0200 Subject: [PATCH 02/10] fix: Add symbolic link changes to Hasher and CacheHasher --- pkg/util/util.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/util/util.go b/pkg/util/util.go index 9387dd029c..cafb1ed02c 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -71,6 +71,12 @@ func Hasher() func(string) (string, error) { if _, err := io.CopyBuffer(h, f, *buf); err != nil { return "", err } + } else if fi.Mode()&os.ModeSymlink == os.ModeSymlink { + linkPath, err := os.Readlink(p) + if err != nil { + return "", err + } + h.Write([]byte(linkPath)) } return hex.EncodeToString(h.Sum(nil)), nil @@ -101,6 +107,12 @@ func CacheHasher() func(string) (string, error) { if _, err := io.Copy(h, f); err != nil { return "", err } + } else if fi.Mode()&os.ModeSymlink == os.ModeSymlink { + linkPath, err := os.Readlink(p) + if err != nil { + return "", err + } + h.Write([]byte(linkPath)) } return hex.EncodeToString(h.Sum(nil)), nil From d2c02eb8c4d70b9c782720923b73099fdba00917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20N=C3=BCtzi?= Date: Thu, 12 May 2022 22:59:42 +0200 Subject: [PATCH 03/10] fix: Better log messages --- integration/integration_test.go | 2 ++ pkg/executor/build.go | 12 +++++++++++- pkg/filesystem/resolve.go | 4 ++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/integration/integration_test.go b/integration/integration_test.go index 58089f1693..c2c2664f02 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -531,6 +531,8 @@ func TestLayers(t *testing.T) { } func buildImage(t *testing.T, dockerfile string, imageBuilder *DockerFileBuilder) { + t.Logf("Building image '%v'...", dockerfile) + if err := imageBuilder.BuildImage(t, config, dockerfilesPath, dockerfile); err != nil { t.Errorf("Error building image: %s", err) t.FailNow() diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 1506f55539..358dfcb8fe 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -623,7 +623,17 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { for index, stage := range kanikoStages { - sb, err := newStageBuilder(args, opts, stage, crossStageDependencies, digestToCacheKey, stageIdxToDigest, stageNameToIdx, fileContext) + sb, err := newStageBuilder( + args, opts, stage, + crossStageDependencies, + digestToCacheKey, + stageIdxToDigest, + stageNameToIdx, + fileContext) + + logrus.Infof("Building stage '%v' [idx: '%v', base-idx: '%v']", + stage.BaseName, stage.Index, stage.BaseImageIndex) + args = sb.args if err != nil { diff --git a/pkg/filesystem/resolve.go b/pkg/filesystem/resolve.go index 9ab2819c9e..2df69213cb 100644 --- a/pkg/filesystem/resolve.go +++ b/pkg/filesystem/resolve.go @@ -71,11 +71,11 @@ func ResolvePaths(paths []string, wl []util.IgnoreListEntry) (pathsToAdd []strin return } - logrus.Debugf("symlink path %s, target does not exist", f) + logrus.Tracef("symlink path %s, target does not exist", f) continue } if f != evaled { - logrus.Debugf("resolved symlink %s to %s", f, evaled) + logrus.Tracef("resolved symlink %s to %s", f, evaled) } // If the given path is a symlink and the target is part of the ignorelist From 3cba8d0b2ffb6013aab58568d167ea7f9b1387e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20N=C3=BCtzi?= Date: Tue, 17 May 2022 18:42:10 +0200 Subject: [PATCH 04/10] fix(ci): Integration tests --- DEVELOPMENT.md | 10 ++++++---- integration/.gitignore | 2 ++ integration/dockerfiles/Dockerfile_test_issue_1837 | 7 ++++--- integration/images.go | 12 +++++++++--- integration/integration_test.go | 12 +++++++----- 5 files changed, 28 insertions(+), 15 deletions(-) create mode 100644 integration/.gitignore diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 015c311e6d..c446791e74 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -74,7 +74,7 @@ The helper script to install and run lint is placed here at the root of project. ```shell ./hack/linter.sh -``` +``` To fix any `gofmt` issues, you can simply run `gofmt` with `-w` flag like this @@ -99,6 +99,8 @@ To run integration tests with your GCloud Storage, you will also need the follow * A bucket in [GCS](https://cloud.google.com/storage/) which you have write access to via the user currently logged into `gcloud` * An image repo which you have write access to via the user currently logged into `gcloud` +* A docker account and a `~/.docker/config.json` with login credentials if you run + into rate limiting problems during tests. Once this step done, you must override the project using environment variables: @@ -162,9 +164,9 @@ These tests will be kicked off by [reviewers](#reviews) for submitted PRs using ### Benchmarking -The goal is for Kaniko to be at least as fast at building Dockerfiles as Docker is, and to that end, we've built +The goal is for Kaniko to be at least as fast at building Dockerfiles as Docker is, and to that end, we've built in benchmarking to check the speed of not only each full run, but also how long each step of each run takes. To turn -on benchmarking, just set the `BENCHMARK_FILE` environment variable, and kaniko will output all the benchmark info +on benchmarking, just set the `BENCHMARK_FILE` environment variable, and kaniko will output all the benchmark info of each run to that file location. ```shell @@ -174,7 +176,7 @@ gcr.io/kaniko-project/executor:latest \ --dockerfile= --context=/workspace \ --destination=gcr.io/my-repo/my-image ``` -Additionally, the integration tests can output benchmarking information to a `benchmarks` directory under the +Additionally, the integration tests can output benchmarking information to a `benchmarks` directory under the `integration` directory if the `BENCHMARK` environment variable is set to `true.` ```shell diff --git a/integration/.gitignore b/integration/.gitignore new file mode 100644 index 0000000000..c61fee769a --- /dev/null +++ b/integration/.gitignore @@ -0,0 +1,2 @@ +cache +config.json \ No newline at end of file diff --git a/integration/dockerfiles/Dockerfile_test_issue_1837 b/integration/dockerfiles/Dockerfile_test_issue_1837 index 1f6b066a23..661ff2dd30 100644 --- a/integration/dockerfiles/Dockerfile_test_issue_1837 +++ b/integration/dockerfiles/Dockerfile_test_issue_1837 @@ -1,6 +1,7 @@ -FROM registry.access.redhat.com/ubi8/ubi:8.2 AS BASE +FROM registry.access.redhat.com/ubi8/ubi:8.2 AS base # Install ping RUN yum --disableplugin=subscription-manager install -y iputils +RUN setcap cap_net_raw+ep /bin/ping || exit 1 -FROM BASE -RUN set -e && [ ! -z "$(getcap /bin/ping)" ] || exit 1 \ No newline at end of file +FROM base +RUN [ ! -z "$(getcap /bin/ping)" ] || exit 1 \ No newline at end of file diff --git a/integration/images.go b/integration/images.go index 0a18b5b66e..dc10fb3553 100644 --- a/integration/images.go +++ b/integration/images.go @@ -31,6 +31,7 @@ import ( "time" "github.com/GoogleContainerTools/kaniko/pkg/timing" + "github.com/GoogleContainerTools/kaniko/pkg/util" ) const ( @@ -82,7 +83,7 @@ var additionalKanikoFlagsMap = map[string][]string{ "Dockerfile_test_scratch": {"--single-snapshot"}, "Dockerfile_test_maintainer": {"--single-snapshot"}, "Dockerfile_test_target": {"--target=second"}, - "Dockerfile_test_snapshotter_ignorelist": {"--use-new-run=true", "-v=debug"}, + "Dockerfile_test_snapshotter_ignorelist": {"--use-new-run=true", "-v=trace"}, } // output check to do when building with kaniko @@ -98,8 +99,8 @@ var outputChecks = map[string]func(string, []byte) error{ } for _, s := range []string{ - "resolved symlink /hello to /dev/null", - "path /dev/null is ignored, ignoring it", + "Resolved symlink /hello to /dev/null", + "Path /dev/null is ignored, ignoring it", } { if !strings.Contains(string(out), s) { return fmt.Errorf("output must contain %s", s) @@ -219,6 +220,11 @@ func addServiceAccountFlags(flags []string, serviceAccount string) []string { "-v", filepath.Dir(serviceAccount)+":/secret/") } else { flags = append(flags, "-v", os.Getenv("HOME")+"/.config/gcloud:/root/.config/gcloud") + + dockerConfig := os.Getenv("HOME") + "/.docker/config.json" + if util.FilepathExists(dockerConfig) { + flags = append(flags, "-v", dockerConfig+":/root/.docker/config.json", "-e", "DOCKER_CONFIG=/root/.docker") + } } return flags } diff --git a/integration/integration_test.go b/integration/integration_test.go index c2c2664f02..c2d8bcc31a 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -496,14 +496,16 @@ func TestLayers(t *testing.T) { offset := map[string]int{ "Dockerfile_test_add": 12, "Dockerfile_test_scratch": 3, + } + if os.Getenv("CI") == "true" { // TODO: tejaldesai fix this! - "Dockerfile_test_meta_arg": 1, - "Dockerfile_test_copy_same_file_many_times": 47, - "Dockerfile_test_arg_multi_with_quotes": 1, - "Dockerfile_test_arg_multi": 1, - "Dockerfile_test_arg_blank_with_quotes": 1, + // This files build locally with difference 0, on CI docker + // produces a different amount of layers (?). + offset["Dockerfile_test_copy_same_file_many_times"] = 47 + offset["Dockerfile_test_meta_arg"] = 1 } + for _, dockerfile := range allDockerfiles { t.Run("test_layer_"+dockerfile, func(t *testing.T) { dockerfileTest := dockerfile From ff0fc51d86fd8135ea74e948584acaf2b95c01fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20N=C3=BCtzi?= Date: Tue, 17 May 2022 20:46:54 +0200 Subject: [PATCH 05/10] fix(ci): Add `--no-cache` to docker builds --- integration/images.go | 1 + 1 file changed, 1 insertion(+) diff --git a/integration/images.go b/integration/images.go index dc10fb3553..2424c6ddc7 100644 --- a/integration/images.go +++ b/integration/images.go @@ -244,6 +244,7 @@ func (d *DockerFileBuilder) BuildDockerImage(t *testing.T, imageRepo, dockerfile dockerArgs := []string{ "build", + "--no-cache", "-t", dockerImage, } From 4dcbda721abd87262b8f7881b91ad422e036ce6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20N=C3=BCtzi?= Date: Tue, 17 May 2022 23:19:56 +0200 Subject: [PATCH 06/10] fix(ci): Pass credentials for error integration test --- integration/integration_test.go | 48 ++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/integration/integration_test.go b/integration/integration_test.go index c2d8bcc31a..2ace5184e9 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -623,14 +623,15 @@ func TestExitCodePropagation(t *testing.T) { t.Run("test error code propagation", func(t *testing.T) { // building the image with docker should fail with exit code 42 dockerImage := GetDockerImage(config.imageRepo, "Dockerfile_exit_code_propagation") - dockerCmd := exec.Command("docker", - append([]string{"build", - "-t", dockerImage, - "-f", dockerfile, - context})...) - _, kanikoErr := RunCommandWithoutTest(dockerCmd) + dockerFlags := []string{ + "build", + "-t", dockerImage, + "-f", dockerfile} + dockerCmd := exec.Command("docker", append(dockerFlags, context)...) + + out, kanikoErr := RunCommandWithoutTest(dockerCmd) if kanikoErr == nil { - t.Fatal("docker build did not produce an error") + t.Fatalf("docker build did not produce an error:\n%s", out) } var dockerCmdExitErr *exec.ExitError var dockerExitCode int @@ -638,32 +639,43 @@ func TestExitCodePropagation(t *testing.T) { if errors.As(kanikoErr, &dockerCmdExitErr) { dockerExitCode = dockerCmdExitErr.ExitCode() testutil.CheckDeepEqual(t, 42, dockerExitCode) + if t.Failed() { + t.Fatalf("Output was:\n%s", out) + } } else { - t.Fatalf("did not produce the expected error") + t.Fatalf("did not produce the expected error:\n%s", out) } //try to build the same image with kaniko the error code should match with the one from the plain docker build contextVolume := fmt.Sprintf("%s:/workspace", context) - dockerCmdWithKaniko := exec.Command("docker", append([]string{ + + dockerFlags = []string{ "run", "-v", contextVolume, - ExecutorImage, + } + dockerFlags = addServiceAccountFlags(dockerFlags, "") + dockerFlags = append(dockerFlags, ExecutorImage, "-c", "dir:///workspace/", "-f", "./Dockerfile_exit_code_propagation", "--no-push", "--force", // TODO: detection of whether kaniko is being run inside a container might be broken? - })...) + ) - _, kanikoErr = RunCommandWithoutTest(dockerCmdWithKaniko) + dockerCmdWithKaniko := exec.Command("docker", dockerFlags...) + + out, kanikoErr = RunCommandWithoutTest(dockerCmdWithKaniko) if kanikoErr == nil { - t.Fatal("the kaniko build did not produce the expected error") + t.Fatalf("the kaniko build did not produce the expected error:\n%s", out) } var kanikoExitErr *exec.ExitError if errors.As(kanikoErr, &kanikoExitErr) { testutil.CheckDeepEqual(t, dockerExitCode, kanikoExitErr.ExitCode()) + if t.Failed() { + t.Fatalf("Output was:\n%s", out) + } } else { - t.Fatalf("did not produce the expected error") + t.Fatalf("did not produce the expected error:\n%s", out) } }) } @@ -802,19 +814,19 @@ func checkLayers(t *testing.T, image1, image2 string, offset int) { func getImageDetails(image string) (*imageDetails, error) { ref, err := name.ParseReference(image, name.WeakValidation) if err != nil { - return nil, fmt.Errorf("Couldn't parse referance to image %s: %s", image, err) + return nil, fmt.Errorf("Couldn't parse referance to image %s: %w", image, err) } imgRef, err := daemon.Image(ref) if err != nil { - return nil, fmt.Errorf("Couldn't get reference to image %s from daemon: %s", image, err) + return nil, fmt.Errorf("Couldn't get reference to image %s from daemon: %w", image, err) } layers, err := imgRef.Layers() if err != nil { - return nil, fmt.Errorf("Error getting layers for image %s: %s", image, err) + return nil, fmt.Errorf("Error getting layers for image %s: %w", image, err) } digest, err := imgRef.Digest() if err != nil { - return nil, fmt.Errorf("Error getting digest for image %s: %s", image, err) + return nil, fmt.Errorf("Error getting digest for image %s: %w", image, err) } return &imageDetails{ name: image, From d0cbec868bc7a7842c82d9044aa9c82419224051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20N=C3=BCtzi?= Date: Tue, 17 May 2022 19:10:31 +0200 Subject: [PATCH 07/10] np: Missing .gitignore in `hack` --- hack/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 hack/.gitignore diff --git a/hack/.gitignore b/hack/.gitignore new file mode 100644 index 0000000000..c5e82d7458 --- /dev/null +++ b/hack/.gitignore @@ -0,0 +1 @@ +bin \ No newline at end of file From 62747f656e1320c1371598b182ece2510bd89d79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20N=C3=BCtzi?= Date: Thu, 12 May 2022 23:00:52 +0200 Subject: [PATCH 08/10] np: Capitalize every log message - Correct some linting. --- cmd/executor/cmd/root.go | 8 ++--- integration/benchmark_test.go | 2 +- integration/gcs.go | 10 +++--- integration/images.go | 21 ++++++----- pkg/commands/copy.go | 2 +- pkg/commands/copy_test.go | 6 ++-- pkg/commands/expose.go | 2 +- pkg/commands/onbuild.go | 4 +-- pkg/commands/run.go | 4 +-- pkg/commands/run_marker.go | 4 +-- pkg/commands/stopsignal.go | 2 +- pkg/commands/user.go | 2 +- pkg/commands/volume.go | 2 +- pkg/commands/workdir.go | 2 +- pkg/config/args.go | 2 +- pkg/dockerfile/dockerfile.go | 2 +- pkg/executor/build.go | 20 +++++------ pkg/executor/push.go | 2 +- pkg/filesystem/resolve.go | 12 +++---- pkg/util/fs_util.go | 36 +++++++++---------- pkg/util/groupids_fallback.go | 2 +- pkg/util/syscall_credentials.go | 2 +- pkg/util/tar_util.go | 2 +- .../github.com/containerd/cgroups/v2/utils.go | 2 +- .../docker/builder/remotecontext/detect.go | 2 +- .../docker/docker/container/container.go | 2 +- .../docker/docker/container/container_unix.go | 3 +- .../docker/docker/container/stream/attach.go | 10 +++--- .../docker/docker/container/stream/streams.go | 4 +-- .../docker/docker/container/view.go | 4 +-- .../docker/docker/daemon/exec/exec.go | 2 +- .../docker/daemon/graphdriver/driver.go | 2 +- .../daemon/logger/loggerutils/follow.go | 2 +- vendor/github.com/docker/docker/image/fs.go | 2 +- .../github.com/docker/docker/image/store.go | 10 +++--- .../docker/docker/layer/filestore.go | 2 +- .../docker/docker/layer/layer_store.go | 4 +-- .../docker/docker/pkg/archive/archive.go | 2 +- .../docker/docker/pkg/archive/changes.go | 2 +- .../docker/docker/pkg/archive/copy.go | 2 +- .../docker/docker/pkg/filenotify/poller.go | 2 +- .../docker/pkg/parsers/kernel/kernel_unix.go | 3 +- .../runc/libcontainer/configs/config.go | 2 +- 43 files changed, 110 insertions(+), 105 deletions(-) diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index a5867578f9..9c4ab55eb7 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -122,7 +122,7 @@ var RootCmd = &cobra.Command{ if !force { exit(errors.New("kaniko should only be run inside of a container, run with the --force flag if you are sure you want to continue")) } - logrus.Warn("kaniko is being run outside of a container. This can have dangerous effects on your system") + logrus.Warn("Kaniko is being run outside of a container. This can have dangerous effects on your system") } if !opts.NoPush || opts.CacheRepo != "" { if err := executor.CheckPushPermissions(opts); err != nil { @@ -152,11 +152,11 @@ var RootCmd = &cobra.Command{ return } if strings.HasPrefix(benchmarkFile, "gs://") { - logrus.Info("uploading to gcs") + logrus.Info("Uploading to gcs") if err := buildcontext.UploadToBucket(strings.NewReader(s), benchmarkFile); err != nil { logrus.Infof("Unable to upload %s due to %v", benchmarkFile, err) } - logrus.Infof("benchmark file written at %s", benchmarkFile) + logrus.Infof("Benchmark file written at %s", benchmarkFile) } else { f, err := os.Create(benchmarkFile) if err != nil { @@ -165,7 +165,7 @@ var RootCmd = &cobra.Command{ } defer f.Close() f.WriteString(s) - logrus.Infof("benchmark file written at %s", benchmarkFile) + logrus.Infof("Benchmark file written at %s", benchmarkFile) } } }, diff --git a/integration/benchmark_test.go b/integration/benchmark_test.go index 4451f54ebc..92efc4f426 100644 --- a/integration/benchmark_test.go +++ b/integration/benchmark_test.go @@ -169,7 +169,7 @@ func runInGcloud(dir string, num int) (string, error) { copyCommand := exec.Command("gsutil", "cp", src, dest) _, err = RunCommandWithoutTest(copyCommand) if err != nil { - return "", fmt.Errorf("failed to download file to GCS bucket %s: %s", src, err) + return "", fmt.Errorf("failed to download file to GCS bucket %s: %w", src, err) } return tmpDir, nil } diff --git a/integration/gcs.go b/integration/gcs.go index 0268d58f66..5680f1e7d0 100644 --- a/integration/gcs.go +++ b/integration/gcs.go @@ -31,17 +31,17 @@ func CreateIntegrationTarball() (string, error) { log.Println("Creating tarball of integration test files to use as build context") dir, err := os.Getwd() if err != nil { - return "", fmt.Errorf("Failed find path to integration dir: %s", err) + return "", fmt.Errorf("Failed find path to integration dir: %w", err) } tempDir, err := ioutil.TempDir("", "") if err != nil { - return "", fmt.Errorf("Failed to create temporary directory to hold tarball: %s", err) + return "", fmt.Errorf("Failed to create temporary directory to hold tarball: %w", err) } contextFile := fmt.Sprintf("%s/context_%d.tar.gz", tempDir, time.Now().UnixNano()) cmd := exec.Command("tar", "-C", dir, "-zcvf", contextFile, ".") _, err = RunCommandWithoutTest(cmd) if err != nil { - return "", fmt.Errorf("Failed to create build context tarball from integration dir: %s", err) + return "", fmt.Errorf("Failed to create build context tarball from integration dir: %w", err) } return contextFile, err } @@ -57,7 +57,7 @@ func UploadFileToBucket(gcsBucket string, filePath string, gcsPath string) (stri if err != nil { log.Printf("Error uploading file %s to GCS at %s: %s", filePath, dst, err) log.Println(string(out)) - return "", fmt.Errorf("Failed to copy tarball to GCS bucket %s: %s", gcsBucket, err) + return "", fmt.Errorf("Failed to copy tarball to GCS bucket %s: %w", gcsBucket, err) } return dst, nil @@ -69,7 +69,7 @@ func DeleteFromBucket(path string) error { cmd := exec.Command("gsutil", "rm", path) _, err := RunCommandWithoutTest(cmd) if err != nil { - return fmt.Errorf("Failed to delete file %s from GCS: %s", path, err) + return fmt.Errorf("Failed to delete file %s from GCS: %w", path, err) } return err } diff --git a/integration/images.go b/integration/images.go index 2424c6ddc7..cece599df0 100644 --- a/integration/images.go +++ b/integration/images.go @@ -163,7 +163,7 @@ func GetVersionedKanikoImage(imageRepo, dockerfile string, version int) string { func FindDockerFiles(dockerfilesPath string) ([]string, error) { allDockerfiles, err := filepath.Glob(path.Join(dockerfilesPath, "Dockerfile_test*")) if err != nil { - return []string{}, fmt.Errorf("Failed to find docker files at %s: %s", dockerfilesPath, err) + return []string{}, fmt.Errorf("Failed to find docker files at %s: %w", dockerfilesPath, err) } var dockerfiles []string @@ -219,7 +219,10 @@ func addServiceAccountFlags(flags []string, serviceAccount string) []string { "GOOGLE_APPLICATION_CREDENTIALS=/secret/"+filepath.Base(serviceAccount), "-v", filepath.Dir(serviceAccount)+":/secret/") } else { - flags = append(flags, "-v", os.Getenv("HOME")+"/.config/gcloud:/root/.config/gcloud") + gcloudConfig := os.Getenv("HOME") + "/.config/gcloud" + if util.FilepathExists(gcloudConfig) { + flags = append(flags, "-v", gcloudConfig+":/root/.config/gcloud") + } dockerConfig := os.Getenv("HOME") + "/.docker/config.json" if util.FilepathExists(dockerConfig) { @@ -262,7 +265,7 @@ func (d *DockerFileBuilder) BuildDockerImage(t *testing.T, imageRepo, dockerfile out, err := RunCommandWithoutTest(dockerCmd) if err != nil { - return fmt.Errorf("Failed to build image %s with docker command \"%s\": %s %s", dockerImage, dockerCmd.Args, err, string(out)) + return fmt.Errorf("Failed to build image %s with docker command \"%s\": %w %s", dockerImage, dockerCmd.Args, err, string(out)) } t.Logf("Build image for Dockerfile %s as %s. docker build output: %s \n", dockerfile, dockerImage, out) return nil @@ -340,7 +343,7 @@ func populateVolumeCache() error { ) if _, err := RunCommandWithoutTest(warmerCmd); err != nil { - return fmt.Errorf("Failed to warm kaniko cache: %s", err) + return fmt.Errorf("Failed to warm kaniko cache: %w", err) } return nil @@ -380,7 +383,7 @@ func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cac _, err := RunCommandWithoutTest(kanikoCmd) if err != nil { - return fmt.Errorf("Failed to build cached image %s with kaniko command \"%s\": %s", kanikoImage, kanikoCmd.Args, err) + return fmt.Errorf("Failed to build cached image %s with kaniko command \"%s\": %w", kanikoImage, kanikoCmd.Args, err) } } return nil @@ -406,7 +409,7 @@ func (d *DockerFileBuilder) buildRelativePathsImage(imageRepo, dockerfile, servi out, err := RunCommandWithoutTest(dockerCmd) timing.DefaultRun.Stop(timer) if err != nil { - return fmt.Errorf("Failed to build image %s with docker command \"%s\": %s %s", dockerImage, dockerCmd.Args, err, string(out)) + return fmt.Errorf("Failed to build image %s with docker command \"%s\": %w %s", dockerImage, dockerCmd.Args, err, string(out)) } dockerRunFlags := []string{"run", "--net=host", "-v", cwd + ":/workspace"} @@ -424,7 +427,7 @@ func (d *DockerFileBuilder) buildRelativePathsImage(imageRepo, dockerfile, servi if err != nil { return fmt.Errorf( - "Failed to build relative path image %s with kaniko command \"%s\": %s\n%s", + "Failed to build relative path image %s with kaniko command \"%s\": %w\n%s", kanikoImage, kanikoCmd.Args, err, string(out)) } @@ -492,11 +495,11 @@ func buildKanikoImage( out, err := RunCommandWithoutTest(kanikoCmd) if err != nil { - return "", fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s\n%s", kanikoImage, kanikoCmd.Args, err, string(out)) + return "", fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %w\n%s", kanikoImage, kanikoCmd.Args, err, string(out)) } if outputCheck := outputChecks[dockerfile]; outputCheck != nil { if err := outputCheck(dockerfile, out); err != nil { - return "", fmt.Errorf("Output check failed for image %s with kaniko command : %s\n%s", kanikoImage, err, string(out)) + return "", fmt.Errorf("Output check failed for image %s with kaniko command : %w\n%s", kanikoImage, err, string(out)) } } return benchmarkDir, nil diff --git a/pkg/commands/copy.go b/pkg/commands/copy.go index 977af66731..2e572794a0 100644 --- a/pkg/commands/copy.go +++ b/pkg/commands/copy.go @@ -191,7 +191,7 @@ func (cr *CachingCopyCommand) ExecuteCommand(config *v1.Config, buildArgs *docke cr.layer = layers[0] cr.extractedFiles, err = util.GetFSFromLayers(kConfig.RootDir, layers, util.ExtractFunc(cr.extractFn), util.IncludeWhiteout()) - logrus.Debugf("extractedFiles: %s", cr.extractedFiles) + logrus.Debugf("ExtractedFiles: %s", cr.extractedFiles) if err != nil { return errors.Wrap(err, "extracting fs from image") } diff --git a/pkg/commands/copy_test.go b/pkg/commands/copy_test.go index 79e41d1ae6..f80ac1d845 100755 --- a/pkg/commands/copy_test.go +++ b/pkg/commands/copy_test.go @@ -58,7 +58,7 @@ func setupTestTemp(t *testing.T) string { srcPath, err := filepath.Abs("../../integration/context") if err != nil { - logrus.Fatalf("error getting abs path %s", srcPath) + logrus.Fatalf("Error getting abs path %s", srcPath) } cperr := filepath.Walk(srcPath, func(path string, info os.FileInfo, err error) error { @@ -98,7 +98,7 @@ func setupTestTemp(t *testing.T) string { return nil }) if cperr != nil { - logrus.Fatalf("error populating temp dir %s", cperr) + logrus.Fatalf("Error populating temp dir %s", cperr) } return tempDir @@ -301,7 +301,7 @@ func TestCopyExecuteCmd(t *testing.T) { t.Error() } for _, file := range files { - logrus.Debugf("file: %v", file.Name()) + logrus.Debugf("File: %v", file.Name()) dirList = append(dirList, file.Name()) } } else { diff --git a/pkg/commands/expose.go b/pkg/commands/expose.go index 0aacbb6cfc..7346160ac8 100644 --- a/pkg/commands/expose.go +++ b/pkg/commands/expose.go @@ -34,7 +34,7 @@ type ExposeCommand struct { } func (r *ExposeCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { - logrus.Info("cmd: EXPOSE") + logrus.Info("Cmd: EXPOSE") // Grab the currently exposed ports existingPorts := config.ExposedPorts if existingPorts == nil { diff --git a/pkg/commands/onbuild.go b/pkg/commands/onbuild.go index 74c6ad01e5..87a2804ac8 100644 --- a/pkg/commands/onbuild.go +++ b/pkg/commands/onbuild.go @@ -30,8 +30,8 @@ type OnBuildCommand struct { //ExecuteCommand adds the specified expression in Onbuild to the config func (o *OnBuildCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { - logrus.Info("cmd: ONBUILD") - logrus.Infof("args: %s", o.cmd.Expression) + logrus.Info("Cmd: ONBUILD") + logrus.Infof("Args: %s", o.cmd.Expression) if config.OnBuild == nil { config.OnBuild = []string{o.cmd.Expression} } else { diff --git a/pkg/commands/run.go b/pkg/commands/run.go index 27b05ed45d..36a87dc1eb 100644 --- a/pkg/commands/run.go +++ b/pkg/commands/run.go @@ -80,8 +80,8 @@ func runCommandInExec(config *v1.Config, buildArgs *dockerfile.BuildArgs, cmdRun } } - logrus.Infof("cmd: %s", newCommand[0]) - logrus.Infof("args: %s", newCommand[1:]) + logrus.Infof("Cmd: %s", newCommand[0]) + logrus.Infof("Args: %s", newCommand[1:]) cmd := exec.Command(newCommand[0], newCommand[1:]...) diff --git a/pkg/commands/run_marker.go b/pkg/commands/run_marker.go index 0a446cc156..bc7482f088 100644 --- a/pkg/commands/run_marker.go +++ b/pkg/commands/run_marker.go @@ -34,14 +34,14 @@ type RunMarkerCommand struct { func (r *RunMarkerCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { // run command `touch filemarker` - logrus.Debugf("using new RunMarker command") + logrus.Debugf("Using new RunMarker command") prevFilesMap, _ := util.GetFSInfoMap("/", map[string]os.FileInfo{}) if err := runCommandInExec(config, buildArgs, r.cmd); err != nil { return err } _, r.Files = util.GetFSInfoMap("/", prevFilesMap) - logrus.Debugf("files changed %s", r.Files) + logrus.Debugf("Files changed %s", r.Files) return nil } diff --git a/pkg/commands/stopsignal.go b/pkg/commands/stopsignal.go index b28d8eba34..f73a70f49d 100644 --- a/pkg/commands/stopsignal.go +++ b/pkg/commands/stopsignal.go @@ -32,7 +32,7 @@ type StopSignalCommand struct { // ExecuteCommand handles command processing similar to CMD and RUN, func (s *StopSignalCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { - logrus.Info("cmd: STOPSIGNAL") + logrus.Info("Cmd: STOPSIGNAL") // resolve possible environment variables replacementEnvs := buildArgs.ReplacementEnvs(config.Env) diff --git a/pkg/commands/user.go b/pkg/commands/user.go index 410bb78401..ed31c7b560 100644 --- a/pkg/commands/user.go +++ b/pkg/commands/user.go @@ -39,7 +39,7 @@ type UserCommand struct { } func (r *UserCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { - logrus.Info("cmd: USER") + logrus.Info("Cmd: USER") u := r.cmd.User userAndGroup := strings.Split(u, ":") replacementEnvs := buildArgs.ReplacementEnvs(config.Env) diff --git a/pkg/commands/volume.go b/pkg/commands/volume.go index 94acd09e12..19c6194886 100644 --- a/pkg/commands/volume.go +++ b/pkg/commands/volume.go @@ -34,7 +34,7 @@ type VolumeCommand struct { } func (v *VolumeCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { - logrus.Info("cmd: VOLUME") + logrus.Info("Cmd: VOLUME") volumes := v.cmd.Volumes replacementEnvs := buildArgs.ReplacementEnvs(config.Env) resolvedVolumes, err := util.ResolveEnvironmentReplacementList(volumes, replacementEnvs, true) diff --git a/pkg/commands/workdir.go b/pkg/commands/workdir.go index 153f27c929..4a95424ec7 100644 --- a/pkg/commands/workdir.go +++ b/pkg/commands/workdir.go @@ -38,7 +38,7 @@ type WorkdirCommand struct { var mkdir = os.MkdirAll func (w *WorkdirCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { - logrus.Info("cmd: workdir") + logrus.Info("Cmd: workdir") workdirPath := w.cmd.Path replacementEnvs := buildArgs.ReplacementEnvs(config.Env) resolvedWorkingDir, err := util.ResolveEnvironmentReplacement(workdirPath, replacementEnvs, true) diff --git a/pkg/config/args.go b/pkg/config/args.go index 46ec51f7cd..7bf52494eb 100644 --- a/pkg/config/args.go +++ b/pkg/config/args.go @@ -35,7 +35,7 @@ func (b *multiArg) String() string { // The second method is Set(value string) error func (b *multiArg) Set(value string) error { - logrus.Debugf("appending to multi args %s", value) + logrus.Debugf("Appending to multi args %s", value) *b = append(*b, value) return nil } diff --git a/pkg/dockerfile/dockerfile.go b/pkg/dockerfile/dockerfile.go index 3dd88cab38..9dd56a7900 100644 --- a/pkg/dockerfile/dockerfile.go +++ b/pkg/dockerfile/dockerfile.go @@ -176,7 +176,7 @@ func extractValFromQuotes(val string) (string, error) { } if leader != tail { - logrus.Infof("leader %s tail %s", leader, tail) + logrus.Infof("Leader %s tail %s", leader, tail) return "", errors.New("quotes wrapping arg values must be matched") } diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 358dfcb8fe..1dbcd65a01 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -222,7 +222,7 @@ func (s *stageBuilder) populateCopyCmdCompositeKey(command fmt.Stringer, from st ds := digest cacheKey, ok := s.digestToCacheKey[ds] if ok { - logrus.Debugf("adding digest %v from previous stage to composite key for %v", ds, command.String()) + logrus.Debugf("Adding digest %v from previous stage to composite key for %v", ds, command.String()) compositeKey.AddKey(cacheKey) } } @@ -260,13 +260,13 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro return err } - logrus.Debugf("optimize: composite key for command %v %v", command.String(), compositeKey) + logrus.Debugf("Optimize: composite key for command %v %v", command.String(), compositeKey) ck, err := compositeKey.Hash() if err != nil { return errors.Wrap(err, "failed to hash composite key") } - logrus.Debugf("optimize: cache key for command %v %v", command.String(), ck) + logrus.Debugf("Optimize: cache key for command %v %v", command.String(), ck) s.finalCacheKey = ck if command.ShouldCacheOutput() && !stopCache { @@ -395,7 +395,7 @@ func (s *stageBuilder) build() error { timing.DefaultRun.Stop(t) if !s.shouldTakeSnapshot(index, command.MetadataOnly()) && !s.opts.ForceBuildMetadata { - logrus.Debugf("build: skipping snapshot for [%v]", command.String()) + logrus.Debugf("Build: skipping snapshot for [%v]", command.String()) continue } if isCacheCommand { @@ -411,13 +411,13 @@ func (s *stageBuilder) build() error { } if s.opts.Cache { - logrus.Debugf("build: composite key for command %v %v", command.String(), compositeKey) + logrus.Debugf("Build: composite key for command %v %v", command.String(), compositeKey) ck, err := compositeKey.Hash() if err != nil { return errors.Wrap(err, "failed to hash composite key") } - logrus.Debugf("build: cache key for command %v %v", command.String(), ck) + logrus.Debugf("Build: cache key for command %v %v", command.String(), ck) // Push layer to cache (in parallel) now along with new config file if command.ShouldCacheOutput() { @@ -433,7 +433,7 @@ func (s *stageBuilder) build() error { } if err := cacheGroup.Wait(); err != nil { - logrus.Warnf("error uploading layer to cache: %s", err) + logrus.Warnf("Error uploading layer to cache: %s", err) } return nil @@ -672,10 +672,10 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { } stageIdxToDigest[fmt.Sprintf("%d", sb.stage.Index)] = d.String() - logrus.Debugf("mapping stage idx %v to digest %v", sb.stage.Index, d.String()) + logrus.Debugf("Mapping stage idx %v to digest %v", sb.stage.Index, d.String()) digestToCacheKey[d.String()] = sb.finalCacheKey - logrus.Debugf("mapping digest %v to cachekey %v", d.String(), sb.finalCacheKey) + logrus.Debugf("Mapping digest %v to cachekey %v", d.String(), sb.finalCacheKey) if stage.Final { sourceImage, err = mutate.CreatedAt(sourceImage, v1.Time{Time: time.Now()}) @@ -827,7 +827,7 @@ func extractImageToDependencyDir(name string, image v1.Image) error { if err := os.MkdirAll(dependencyDir, 0755); err != nil { return err } - logrus.Debugf("trying to extract to %s", dependencyDir) + logrus.Debugf("Trying to extract to %s", dependencyDir) _, err := util.GetFSFromImage(dependencyDir, image, util.ExtractFile) return err } diff --git a/pkg/executor/push.go b/pkg/executor/push.go index 9f75305e5d..46da811b14 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -123,7 +123,7 @@ func writeDigestFile(path string, digestByteArray []byte) error { parentDir := filepath.Dir(path) if _, err := os.Stat(parentDir); os.IsNotExist(err) { if err := os.MkdirAll(parentDir, 0700); err != nil { - logrus.Debugf("error creating %s, %s", parentDir, err) + logrus.Debugf("Error creating %s, %s", parentDir, err) return err } logrus.Tracef("Created directory %v", parentDir) diff --git a/pkg/filesystem/resolve.go b/pkg/filesystem/resolve.go index 2df69213cb..648f13020e 100644 --- a/pkg/filesystem/resolve.go +++ b/pkg/filesystem/resolve.go @@ -42,7 +42,7 @@ func ResolvePaths(paths []string, wl []util.IgnoreListEntry) (pathsToAdd []strin for _, f := range paths { // If the given path is part of the ignorelist ignore it if util.IsInProvidedIgnoreList(f, wl) { - logrus.Debugf("path %s is in list to ignore, ignoring it", f) + logrus.Debugf("Path %s is in list to ignore, ignoring it", f) continue } @@ -52,7 +52,7 @@ func ResolvePaths(paths []string, wl []util.IgnoreListEntry) (pathsToAdd []strin } if f != link { - logrus.Tracef("updated link %s to %s", f, link) + logrus.Tracef("Updated link %s to %s", f, link) } if !fileSet[link] { @@ -67,21 +67,21 @@ func ResolvePaths(paths []string, wl []util.IgnoreListEntry) (pathsToAdd []strin evaled, e = filepath.EvalSymlinks(f) if e != nil { if !os.IsNotExist(e) { - logrus.Errorf("couldn't eval %s with link %s", f, link) + logrus.Errorf("Couldn't eval %s with link %s", f, link) return } - logrus.Tracef("symlink path %s, target does not exist", f) + logrus.Tracef("Symlink path %s, target does not exist", f) continue } if f != evaled { - logrus.Tracef("resolved symlink %s to %s", f, evaled) + logrus.Tracef("Resolved symlink %s to %s", f, evaled) } // If the given path is a symlink and the target is part of the ignorelist // ignore the target if util.CheckProvidedIgnoreList(evaled, wl) { - logrus.Debugf("path %s is ignored, ignoring it", evaled) + logrus.Debugf("Path %s is ignored, ignoring it", evaled) continue } diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 396830dfc9..347774b14a 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -197,7 +197,7 @@ func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, e } if !cfg.includeWhiteout { - logrus.Trace("not including whiteout files") + logrus.Trace("Not including whiteout files") continue } @@ -301,13 +301,13 @@ func ExtractFile(dest string, hdr *tar.Header, tr io.Reader) error { } switch hdr.Typeflag { case tar.TypeReg: - logrus.Tracef("creating file %s", path) + logrus.Tracef("Creating file %s", path) // It's possible a file is in the tar before its directory, // or a file was copied over a directory prior to now fi, err := os.Stat(dir) if os.IsNotExist(err) || !fi.IsDir() { - logrus.Debugf("base %s for file %s does not exist. Creating.", base, path) + logrus.Debugf("Base %s for file %s does not exist. Creating.", base, path) if err := os.MkdirAll(dir, 0755); err != nil { return err @@ -345,19 +345,19 @@ func ExtractFile(dest string, hdr *tar.Header, tr io.Reader) error { currFile.Close() case tar.TypeDir: - logrus.Tracef("creating dir %s", path) + logrus.Tracef("Creating dir %s", path) if err := mkdirAllWithPermissions(path, mode, int64(uid), int64(gid)); err != nil { return err } case tar.TypeLink: - logrus.Tracef("link from %s to %s", hdr.Linkname, path) + logrus.Tracef("Link from %s to %s", hdr.Linkname, path) abs, err := filepath.Abs(hdr.Linkname) if err != nil { return err } if CheckIgnoreList(abs) { - logrus.Tracef("skipping link from %s to %s because %s is ignored", hdr.Linkname, path, hdr.Linkname) + logrus.Tracef("Skipping link from %s to %s because %s is ignored", hdr.Linkname, path, hdr.Linkname) return nil } // The base directory for a link may not exist before it is created. @@ -377,7 +377,7 @@ func ExtractFile(dest string, hdr *tar.Header, tr io.Reader) error { } case tar.TypeSymlink: - logrus.Tracef("symlink from %s to %s", hdr.Linkname, path) + logrus.Tracef("Symlink from %s to %s", hdr.Linkname, path) // The base directory for a symlink may not exist before it is created. if err := os.MkdirAll(dir, 0755); err != nil { return err @@ -559,7 +559,7 @@ func CreateFile(path string, reader io.Reader, perm os.FileMode, uid uint32, gid // AddVolumePath adds the given path to the volume ignorelist. func AddVolumePathToIgnoreList(path string) { - logrus.Infof("adding volume %s to ignorelist", path) + logrus.Infof("Adding volume %s to ignorelist", path) ignorelist = append(ignorelist, IgnoreListEntry{ Path: path, PrefixMatchOnly: true, @@ -667,7 +667,7 @@ func CopySymlink(src, dest string, context FileContext) (bool, error) { } link, err := os.Readlink(src) if err != nil { - logrus.Debugf("could not read link for %s", src) + logrus.Debugf("Could not read link for %s", src) } return false, os.Symlink(link, dest) } @@ -733,13 +733,13 @@ func (c FileContext) ExcludesFile(path string) bool { var err error path, err = filepath.Rel(c.Root, path) if err != nil { - logrus.Errorf("unable to get relative path, including %s in build: %v", path, err) + logrus.Errorf("Unable to get relative path, including %s in build: %v", path, err) return false } } match, err := fileutils.Matches(path, c.ExcludedFiles) if err != nil { - logrus.Errorf("error matching, including %s in build: %v", path, err) + logrus.Errorf("Error matching, including %s in build: %v", path, err) return false } return match @@ -779,7 +779,7 @@ func mkdirAllWithPermissions(path string, mode os.FileMode, uid, gid int64) erro // Check if a file already exists on the path, if yes then delete it info, err := os.Stat(path) if err == nil && !info.IsDir() { - logrus.Tracef("removing file because it needs to be a directory %s", path) + logrus.Tracef("Removing file because it needs to be a directory %s", path) if err := os.Remove(path); err != nil { return errors.Wrapf(err, "error removing %s to make way for new directory.", path) } @@ -817,12 +817,12 @@ func setFileTimes(path string, aTime, mTime time.Time) error { // converted into a valid argument to the syscall that os.Chtimes uses. If mTime or // aTime are zero we convert them to the zero value for Unix Epoch. if mTime.IsZero() { - logrus.Tracef("mod time for %s is zero, converting to zero for epoch", path) + logrus.Tracef("Mod time for %s is zero, converting to zero for epoch", path) mTime = time.Unix(0, 0) } if aTime.IsZero() { - logrus.Tracef("access time for %s is zero, converting to zero for epoch", path) + logrus.Tracef("Access time for %s is zero, converting to zero for epoch", path) aTime = time.Unix(0, 0) } @@ -845,7 +845,7 @@ func setFileTimes(path string, aTime, mTime time.Time) error { func CreateTargetTarfile(tarpath string) (*os.File, error) { baseDir := filepath.Dir(tarpath) if _, err := os.Lstat(baseDir); os.IsNotExist(err) { - logrus.Debugf("baseDir %s for file %s does not exist. Creating.", baseDir, tarpath) + logrus.Debugf("BaseDir %s for file %s does not exist. Creating.", baseDir, tarpath) if err := os.MkdirAll(baseDir, 0755); err != nil { return nil, err } @@ -963,12 +963,12 @@ func CopyOwnership(src string, destDir string, root string) error { func createParentDirectory(path string) error { baseDir := filepath.Dir(path) if info, err := os.Lstat(baseDir); os.IsNotExist(err) { - logrus.Tracef("baseDir %s for file %s does not exist. Creating.", baseDir, path) + logrus.Tracef("BaseDir %s for file %s does not exist. Creating.", baseDir, path) if err := os.MkdirAll(baseDir, 0755); err != nil { return err } } else if IsSymlink(info) { - logrus.Infof("destination cannot be a symlink %v", baseDir) + logrus.Infof("Destination cannot be a symlink %v", baseDir) return errors.New("destination cannot be a symlink") } return nil @@ -1030,7 +1030,7 @@ func WalkFS( return res.filesAdded, res.existingPaths case <-time.After(timeOut): timing.DefaultRun.Stop(timer) - logrus.Fatalf("timed out snapshotting FS in %s", timeOutStr) + logrus.Fatalf("Timed out snapshotting FS in %s", timeOutStr) return nil, nil } } diff --git a/pkg/util/groupids_fallback.go b/pkg/util/groupids_fallback.go index 41552674d9..62d0760244 100644 --- a/pkg/util/groupids_fallback.go +++ b/pkg/util/groupids_fallback.go @@ -43,7 +43,7 @@ type group struct { // groupIDs returns all of the group ID's a user is a member of func groupIDs(u *user.User) ([]string, error) { - logrus.Infof("performing slow lookup of group ids for %s", u.Username) + logrus.Infof("Performing slow lookup of group ids for %s", u.Username) f, err := os.Open(groupFile) if err != nil { diff --git a/pkg/util/syscall_credentials.go b/pkg/util/syscall_credentials.go index f0a2a34060..8aa9b11aa6 100644 --- a/pkg/util/syscall_credentials.go +++ b/pkg/util/syscall_credentials.go @@ -34,7 +34,7 @@ func SyscallCredentials(userStr string) (*syscall.Credential, error) { if err != nil { return nil, errors.Wrap(err, "lookup") } - logrus.Infof("util.Lookup returned: %+v", u) + logrus.Infof("Util.Lookup returned: %+v", u) var groups []uint32 diff --git a/pkg/util/tar_util.go b/pkg/util/tar_util.go index ac83311111..963babc9ff 100644 --- a/pkg/util/tar_util.go +++ b/pkg/util/tar_util.go @@ -70,7 +70,7 @@ func (t *Tar) AddFileToTar(p string) error { } } if i.Mode()&os.ModeSocket != 0 { - logrus.Infof("ignoring socket %s, not adding to tar", i.Name()) + logrus.Infof("Ignoring socket %s, not adding to tar", i.Name()) return nil } hdr, err := tar.FileInfoHeader(i, linkDst) diff --git a/vendor/github.com/containerd/cgroups/v2/utils.go b/vendor/github.com/containerd/cgroups/v2/utils.go index 902466f51b..2647aecaed 100644 --- a/vendor/github.com/containerd/cgroups/v2/utils.go +++ b/vendor/github.com/containerd/cgroups/v2/utils.go @@ -253,7 +253,7 @@ func getStatFileContentUint64(filePath string) uint64 { res, err := parseUint(trimmed, 10, 64) if err != nil { - logrus.Errorf("unable to parse %q as a uint from Cgroup file %q", string(contents), filePath) + logrus.Errorf("Unable to parse %q as a uint from Cgroup file %q", string(contents), filePath) return res } diff --git a/vendor/github.com/docker/docker/builder/remotecontext/detect.go b/vendor/github.com/docker/docker/builder/remotecontext/detect.go index 9b126ef775..be885e0305 100644 --- a/vendor/github.com/docker/docker/builder/remotecontext/detect.go +++ b/vendor/github.com/docker/docker/builder/remotecontext/detect.go @@ -132,7 +132,7 @@ func removeDockerfile(c modifiableContext, filesToRemove ...string) error { for _, fileToRemove := range filesToRemove { if rm, _ := fileutils.Matches(fileToRemove, excludes); rm { if err := c.Remove(fileToRemove); err != nil { - logrus.Errorf("failed to remove %s: %v", fileToRemove, err) + logrus.Errorf("Failed to remove %s: %v", fileToRemove, err) } } } diff --git a/vendor/github.com/docker/docker/container/container.go b/vendor/github.com/docker/docker/container/container.go index e96f087c55..4a0db0ecf6 100644 --- a/vendor/github.com/docker/docker/container/container.go +++ b/vendor/github.com/docker/docker/container/container.go @@ -681,7 +681,7 @@ func (container *Container) InitializeStdio(iop *cio.DirectIO) (cio.IO, error) { if container.StreamConfig.Stdin() == nil && !container.Config.Tty { if iop.Stdin != nil { if err := iop.Stdin.Close(); err != nil { - logrus.Warnf("error closing stdin: %+v", err) + logrus.Warnf("Error closing stdin: %+v", err) } } } diff --git a/vendor/github.com/docker/docker/container/container_unix.go b/vendor/github.com/docker/docker/container/container_unix.go index 7a49ff55aa..b253234bb6 100644 --- a/vendor/github.com/docker/docker/container/container_unix.go +++ b/vendor/github.com/docker/docker/container/container_unix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package container // import "github.com/docker/docker/container" @@ -144,7 +145,7 @@ func (container *Container) CopyImagePathContent(v volume.Volume, destination st defer func() { if err := v.Unmount(id); err != nil { - logrus.Warnf("error while unmounting volume %s: %v", v.Name(), err) + logrus.Warnf("Error while unmounting volume %s: %v", v.Name(), err) } }() if err := label.Relabel(path, container.MountLabel, true); err != nil && !errors.Is(err, syscall.ENOTSUP) { diff --git a/vendor/github.com/docker/docker/container/stream/attach.go b/vendor/github.com/docker/docker/container/stream/attach.go index 0269a226b1..970e287dd2 100644 --- a/vendor/github.com/docker/docker/container/stream/attach.go +++ b/vendor/github.com/docker/docker/container/stream/attach.go @@ -63,8 +63,8 @@ func (c *Config) CopyStreams(ctx context.Context, cfg *AttachConfig) <-chan erro // Connect stdin of container to the attach stdin stream. if cfg.Stdin != nil { group.Go(func() error { - logrus.Debug("attach: stdin: begin") - defer logrus.Debug("attach: stdin: end") + logrus.Debug("Attach: stdin: begin") + defer logrus.Debug("Attach: stdin: end") defer func() { if cfg.CloseStdin && !cfg.TTY { @@ -98,8 +98,8 @@ func (c *Config) CopyStreams(ctx context.Context, cfg *AttachConfig) <-chan erro } attachStream := func(name string, stream io.Writer, streamPipe io.ReadCloser) error { - logrus.Debugf("attach: %s: begin", name) - defer logrus.Debugf("attach: %s: end", name) + logrus.Debugf("Attach: %s: begin", name) + defer logrus.Debugf("Attach: %s: end", name) defer func() { // Make sure stdin gets closed if cfg.Stdin != nil { @@ -132,7 +132,7 @@ func (c *Config) CopyStreams(ctx context.Context, cfg *AttachConfig) <-chan erro errs := make(chan error, 1) go func() { - defer logrus.Debug("attach done") + defer logrus.Debug("Attach done") groupErr := make(chan error, 1) go func() { groupErr <- group.Wait() diff --git a/vendor/github.com/docker/docker/container/stream/streams.go b/vendor/github.com/docker/docker/container/stream/streams.go index 585f9e8e3a..52422331a6 100644 --- a/vendor/github.com/docker/docker/container/stream/streams.go +++ b/vendor/github.com/docker/docker/container/stream/streams.go @@ -122,7 +122,7 @@ func (c *Config) CopyToPipe(iop *cio.DirectIO) { c.wg.Add(1) go func() { if _, err := pools.Copy(w, r); err != nil { - logrus.Errorf("stream copy error: %v", err) + logrus.Errorf("Stream copy error: %v", err) } r.Close() c.wg.Done() @@ -141,7 +141,7 @@ func (c *Config) CopyToPipe(iop *cio.DirectIO) { go func() { pools.Copy(iop.Stdin, stdin) if err := iop.Stdin.Close(); err != nil { - logrus.Warnf("failed to close stdin: %v", err) + logrus.Warnf("Failed to close stdin: %v", err) } }() } diff --git a/vendor/github.com/docker/docker/container/view.go b/vendor/github.com/docker/docker/container/view.go index 962a20b9e1..3cb8f93ae9 100644 --- a/vendor/github.com/docker/docker/container/view.go +++ b/vendor/github.com/docker/docker/container/view.go @@ -382,7 +382,7 @@ func (v *memdbView) transform(container *Container) *Snapshot { for port, bindings := range container.NetworkSettings.Ports { p, err := nat.ParsePort(port.Port()) if err != nil { - logrus.Warnf("invalid port map %+v", err) + logrus.Warnf("Invalid port map %+v", err) continue } if len(bindings) == 0 { @@ -395,7 +395,7 @@ func (v *memdbView) transform(container *Container) *Snapshot { for _, binding := range bindings { h, err := nat.ParsePort(binding.HostPort) if err != nil { - logrus.Warnf("invalid host port map %+v", err) + logrus.Warnf("Invalid host port map %+v", err) continue } snapshot.Ports = append(snapshot.Ports, types.Port{ diff --git a/vendor/github.com/docker/docker/daemon/exec/exec.go b/vendor/github.com/docker/docker/daemon/exec/exec.go index f1b5259854..3f2b742cd3 100644 --- a/vendor/github.com/docker/docker/daemon/exec/exec.go +++ b/vendor/github.com/docker/docker/daemon/exec/exec.go @@ -71,7 +71,7 @@ func (c *Config) InitializeStdio(iop *cio.DirectIO) (cio.IO, error) { if c.StreamConfig.Stdin() == nil && !c.Tty && runtime.GOOS == "windows" { if iop.Stdin != nil { if err := iop.Stdin.Close(); err != nil { - logrus.Errorf("error closing exec stdin: %+v", err) + logrus.Errorf("Error closing exec stdin: %+v", err) } } } diff --git a/vendor/github.com/docker/docker/daemon/graphdriver/driver.go b/vendor/github.com/docker/docker/daemon/graphdriver/driver.go index a9e8ce4c91..ca38c04426 100644 --- a/vendor/github.com/docker/docker/daemon/graphdriver/driver.go +++ b/vendor/github.com/docker/docker/daemon/graphdriver/driver.go @@ -293,7 +293,7 @@ func IsInitialized(driverHome string) bool { return false } if err != nil { - logrus.Warnf("graphdriver.IsInitialized: stat failed: %v", err) + logrus.Warnf("Graphdriver.IsInitialized: stat failed: %v", err) } return !isEmptyDir(driverHome) } diff --git a/vendor/github.com/docker/docker/daemon/logger/loggerutils/follow.go b/vendor/github.com/docker/docker/daemon/logger/loggerutils/follow.go index 755a483d7a..cb1dad0ab6 100644 --- a/vendor/github.com/docker/docker/daemon/logger/loggerutils/follow.go +++ b/vendor/github.com/docker/docker/daemon/logger/loggerutils/follow.go @@ -85,7 +85,7 @@ func (fl *follow) waitRead() error { } return errRetry case err := <-fl.fileWatcher.Errors(): - logrus.Debugf("logger got error watching file: %v", err) + logrus.Debugf("Logger got error watching file: %v", err) // Something happened, let's try and stay alive and create a new watcher if fl.retries <= 5 { fl.fileWatcher.Close() diff --git a/vendor/github.com/docker/docker/image/fs.go b/vendor/github.com/docker/docker/image/fs.go index 8300c41884..a95be86cfd 100644 --- a/vendor/github.com/docker/docker/image/fs.go +++ b/vendor/github.com/docker/docker/image/fs.go @@ -76,7 +76,7 @@ func (s *fs) Walk(f DigestWalkFunc) error { for _, v := range dir { dgst := digest.NewDigestFromHex(string(digest.Canonical), v.Name()) if err := dgst.Validate(); err != nil { - logrus.Debugf("skipping invalid digest %s: %s", dgst, err) + logrus.Debugf("Skipping invalid digest %s: %s", dgst, err) continue } if err := f(dgst); err != nil { diff --git a/vendor/github.com/docker/docker/image/store.go b/vendor/github.com/docker/docker/image/store.go index 20a6d49257..17c452d35e 100644 --- a/vendor/github.com/docker/docker/image/store.go +++ b/vendor/github.com/docker/docker/image/store.go @@ -69,19 +69,19 @@ func (is *store) restore() error { err := is.fs.Walk(func(dgst digest.Digest) error { img, err := is.Get(IDFromDigest(dgst)) if err != nil { - logrus.Errorf("invalid image %v, %v", dgst, err) + logrus.Errorf("Invalid image %v, %v", dgst, err) return nil } var l layer.Layer if chainID := img.RootFS.ChainID(); chainID != "" { if !system.IsOSSupported(img.OperatingSystem()) { - logrus.Errorf("not restoring image with unsupported operating system %v, %v, %s", dgst, chainID, img.OperatingSystem()) + logrus.Errorf("Not restoring image with unsupported operating system %v, %v, %s", dgst, chainID, img.OperatingSystem()) return nil } l, err = is.lss[img.OperatingSystem()].Get(chainID) if err != nil { if err == layer.ErrLayerDoesNotExist { - logrus.Errorf("layer does not exist, not restoring image %v, %v, %s", dgst, chainID, img.OperatingSystem()) + logrus.Errorf("Layer does not exist, not restoring image %v, %v, %s", dgst, chainID, img.OperatingSystem()) return nil } return err @@ -244,7 +244,7 @@ func (is *store) Delete(id ID) ([]layer.Metadata, error) { } if err := is.digestSet.Remove(id.Digest()); err != nil { - logrus.Errorf("error removing %s from digest set: %q", id, err) + logrus.Errorf("Error removing %s from digest set: %q", id, err) } delete(is.images, id) is.fs.Delete(id.Digest()) @@ -330,7 +330,7 @@ func (is *store) imagesMap(all bool) map[ID]*Image { } img, err := is.Get(id) if err != nil { - logrus.Errorf("invalid image access: %q, error: %q", id, err) + logrus.Errorf("Invalid image access: %q, error: %q", id, err) continue } images[id] = img diff --git a/vendor/github.com/docker/docker/layer/filestore.go b/vendor/github.com/docker/docker/layer/filestore.go index 0c15cc9b96..6f45580c53 100644 --- a/vendor/github.com/docker/docker/layer/filestore.go +++ b/vendor/github.com/docker/docker/layer/filestore.go @@ -339,7 +339,7 @@ func (fms *fileMetadataStore) getOrphan() ([]roLayer, error) { } cacheID := strings.TrimSpace(string(contentBytes)) if cacheID == "" { - logrus.Error("invalid cache ID") + logrus.Error("Invalid cache ID") continue } diff --git a/vendor/github.com/docker/docker/layer/layer_store.go b/vendor/github.com/docker/docker/layer/layer_store.go index c58f501982..0fd84140dd 100644 --- a/vendor/github.com/docker/docker/layer/layer_store.go +++ b/vendor/github.com/docker/docker/layer/layer_store.go @@ -764,9 +764,9 @@ func (ls *layerStore) Cleanup() error { if err != nil { logrus.Errorf("Cannot get orphan layers: %v", err) } - logrus.Debugf("found %v orphan layers", len(orphanLayers)) + logrus.Debugf("Found %v orphan layers", len(orphanLayers)) for _, orphan := range orphanLayers { - logrus.Debugf("removing orphan layer, chain ID: %v , cache ID: %v", orphan.chainID, orphan.cacheID) + logrus.Debugf("Removing orphan layer, chain ID: %v , cache ID: %v", orphan.chainID, orphan.cacheID) err = ls.driver.Remove(orphan.cacheID) if err != nil && !os.IsNotExist(err) { logrus.WithError(err).WithField("cache-id", orphan.cacheID).Error("cannot remove orphan layer") diff --git a/vendor/github.com/docker/docker/pkg/archive/archive.go b/vendor/github.com/docker/docker/pkg/archive/archive.go index 50b83c62c6..91ae48d4bc 100644 --- a/vendor/github.com/docker/docker/pkg/archive/archive.go +++ b/vendor/github.com/docker/docker/pkg/archive/archive.go @@ -165,7 +165,7 @@ func gzDecompress(ctx context.Context, buf io.Reader) (io.ReadCloser, error) { unpigzPath, err := exec.LookPath("unpigz") if err != nil { - logrus.Debugf("unpigz binary not found, falling back to go gzip library") + logrus.Debugf("Unpigz binary not found, falling back to go gzip library") return gzip.NewReader(buf) } diff --git a/vendor/github.com/docker/docker/pkg/archive/changes.go b/vendor/github.com/docker/docker/pkg/archive/changes.go index aedb91b035..3e287243af 100644 --- a/vendor/github.com/docker/docker/pkg/archive/changes.go +++ b/vendor/github.com/docker/docker/pkg/archive/changes.go @@ -438,7 +438,7 @@ func ExportChanges(dir string, changes []Change, uidMaps, gidMaps []idtools.IDMa logrus.Debugf("Can't close layer: %s", err) } if err := writer.Close(); err != nil { - logrus.Debugf("failed close Changes writer: %s", err) + logrus.Debugf("Failed close Changes writer: %s", err) } }() return reader, nil diff --git a/vendor/github.com/docker/docker/pkg/archive/copy.go b/vendor/github.com/docker/docker/pkg/archive/copy.go index 57fddac078..4e15d1dce3 100644 --- a/vendor/github.com/docker/docker/pkg/archive/copy.go +++ b/vendor/github.com/docker/docker/pkg/archive/copy.go @@ -108,7 +108,7 @@ func TarResourceRebase(sourcePath, rebaseName string) (content io.ReadCloser, er sourceDir, sourceBase := SplitPathDirEntry(sourcePath) opts := TarResourceRebaseOpts(sourceBase, rebaseName) - logrus.Debugf("copying %q from %q", sourceBase, sourceDir) + logrus.Debugf("Copying %q from %q", sourceBase, sourceDir) return TarWithOptions(sourceDir, opts) } diff --git a/vendor/github.com/docker/docker/pkg/filenotify/poller.go b/vendor/github.com/docker/docker/pkg/filenotify/poller.go index 01ef057981..28fa45ae17 100644 --- a/vendor/github.com/docker/docker/pkg/filenotify/poller.go +++ b/vendor/github.com/docker/docker/pkg/filenotify/poller.go @@ -159,7 +159,7 @@ func (w *filePoller) watch(f *os.File, lastFi os.FileInfo, chClose chan struct{} select { case <-timer.C: case <-chClose: - logrus.Debugf("watch for %s closed", f.Name()) + logrus.Debugf("Watch for %s closed", f.Name()) return } diff --git a/vendor/github.com/docker/docker/pkg/parsers/kernel/kernel_unix.go b/vendor/github.com/docker/docker/pkg/parsers/kernel/kernel_unix.go index 8a9aa31225..e761eb05fb 100644 --- a/vendor/github.com/docker/docker/pkg/parsers/kernel/kernel_unix.go +++ b/vendor/github.com/docker/docker/pkg/parsers/kernel/kernel_unix.go @@ -1,3 +1,4 @@ +//go:build linux || freebsd || openbsd // +build linux freebsd openbsd // Package kernel provides helper function to get, parse and compare kernel @@ -25,7 +26,7 @@ func GetKernelVersion() (*VersionInfo, error) { // the given version. func CheckKernelVersion(k, major, minor int) bool { if v, err := GetKernelVersion(); err != nil { - logrus.Warnf("error getting kernel version: %s", err) + logrus.Warnf("Error getting kernel version: %s", err) } else { if CompareKernelVersion(*v, VersionInfo{Kernel: k, Major: major, Minor: minor}) < 0 { return false diff --git a/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go b/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go index 540f0f85d2..91a4bf018c 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go @@ -291,7 +291,7 @@ func (hooks *Hooks) MarshalJSON() ([]byte, error) { case CommandHook: serializableHooks = append(serializableHooks, chook) default: - logrus.Warnf("cannot serialize hook of type %T, skipping", hook) + logrus.Warnf("Cannot serialize hook of type %T, skipping", hook) } } From fc6a54c556140499fec09889f23c096a738fccac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20N=C3=BCtzi?= Date: Wed, 18 May 2022 00:54:43 +0200 Subject: [PATCH 09/10] fix: Key function - Merge only last layer onto `currentImage`. --- pkg/snapshot/layered_map.go | 76 +++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 29 deletions(-) diff --git a/pkg/snapshot/layered_map.go b/pkg/snapshot/layered_map.go index 7e973793a0..9a5c9e70e5 100644 --- a/pkg/snapshot/layered_map.go +++ b/pkg/snapshot/layered_map.go @@ -38,6 +38,7 @@ type LayeredMap struct { cacheHasher func(string) (string, error) } +// NewLayeredMap creates a new layered map which keeps track of adds and deletes. func NewLayeredMap(h func(string) (string, error), c func(string) (string, error)) *LayeredMap { l := LayeredMap{ hasher: h, @@ -49,69 +50,86 @@ func NewLayeredMap(h func(string) (string, error), c func(string) (string, error return &l } +// Snapshot creates a new layer. func (l *LayeredMap) Snapshot() { // Save current state of image - l.UpdateCurrentImage() + l.updateCurrentImage() l.whiteouts = append(l.whiteouts, map[string]struct{}{}) l.layers = append(l.layers, map[string]string{}) l.layerHashCache = map[string]string{} // Erase the hash cache for this new layer. } -// Key returns a hash for added files +// Key returns a hash for added and delted files. func (l *LayeredMap) Key() (string, error) { c := bytes.NewBuffer([]byte{}) enc := json.NewEncoder(c) - err := enc.Encode(l.layers) + err := enc.Encode(l.layers[len(l.layers)-1]) + if err != nil { + return "", err + } + err = enc.Encode(l.whiteouts[len(l.whiteouts)-1]) if err != nil { return "", err } return util.SHA256(c) } -// UpdateCurrentImage computes the current image by -// flattening all layers and stores the result in currentImage. -func (l *LayeredMap) UpdateCurrentImage() { - if l.isCurrentImageValid { - return +// getCurrentImage returns the current image by merging the latest +// adds and deletes on to the current image (if its not yet valid.) +func (l *LayeredMap) getCurrentImage() map[string]string { + if l.isCurrentImageValid || len(l.layers) == 0 { + // No layers yet or current image is valid. + return l.currentImage } - l.currentImage = map[string]string{} + current := map[string]string{} - // Adding and deleting files over all layers. - for i := 0; i < len(l.layers); i++ { - addedFiles := l.layers[i] - deletedFiles := l.whiteouts[i] + // Copy current image paths/hashes. + for p, h := range l.currentImage { + current[p] = h + } - for add, hash := range addedFiles { - l.currentImage[add] = hash - } + // Add the last layer on top. + addedFiles := l.layers[len(l.layers)-1] + deletedFiles := l.whiteouts[len(l.whiteouts)-1] - for del := range deletedFiles { - delete(l.currentImage, del) - } + for add, hash := range addedFiles { + current[add] = hash } - l.isCurrentImageValid = true + for del := range deletedFiles { + delete(current, del) + } + + return current } -func (l *LayeredMap) Get(s string) (string, bool) { - for i := len(l.layers) - 1; i >= 0; i-- { - if v, ok := l.layers[i][s]; ok { - return v, ok - } +// updateCurrentImage update the internal current image by merging the +// top adds and deletes onto the current image. +func (l *LayeredMap) updateCurrentImage() { + if l.isCurrentImageValid { + return } - return "", false + + l.currentImage = l.getCurrentImage() + l.isCurrentImageValid = true +} + +// get returns the current hash in the current image `l.currentImage`. +func (l *LayeredMap) get(s string) (string, bool) { + h, ok := l.currentImage[s] + return h, ok } // GetCurrentPaths returns all existing paths in the actual current image // cached by FlattenLayers. func (l *LayeredMap) GetCurrentPaths() map[string]struct{} { - l.UpdateCurrentImage() + current := l.getCurrentImage() paths := map[string]struct{}{} - for f := range l.currentImage { + for f := range current { paths[f] = struct{}{} } return paths @@ -162,7 +180,7 @@ func (l *LayeredMap) CheckFileChange(s string) (bool, error) { // adding the file. l.layerHashCache[s] = newV - oldV, ok := l.currentImage[s] + oldV, ok := l.get(s) if ok && newV == oldV { // File hash did not change => Unchanged. return false, nil From d37d622963e303e88f0bf8b927025ee2dba77a60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20N=C3=BCtzi?= Date: Wed, 18 May 2022 01:21:38 +0200 Subject: [PATCH 10/10] fix: Remove old obsolete `cacheHasher` --- pkg/executor/build.go | 2 +- pkg/snapshot/layered_map.go | 42 ++++++++++++++++++-------------- pkg/snapshot/layered_map_test.go | 16 ++++++------ pkg/snapshot/snapshot.go | 4 +-- pkg/snapshot/snapshot_test.go | 2 +- 5 files changed, 36 insertions(+), 30 deletions(-) diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 1dbcd65a01..8b3db6a3f3 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -108,7 +108,7 @@ func newStageBuilder(args *dockerfile.BuildArgs, opts *config.KanikoOptions, sta if err != nil { return nil, err } - l := snapshot.NewLayeredMap(hasher, util.CacheHasher()) + l := snapshot.NewLayeredMap(hasher) snapshotter := snapshot.NewSnapshotter(l, config.RootDir) digest, err := sourceImage.Digest() diff --git a/pkg/snapshot/layered_map.go b/pkg/snapshot/layered_map.go index 9a5c9e70e5..9fa8e467de 100644 --- a/pkg/snapshot/layered_map.go +++ b/pkg/snapshot/layered_map.go @@ -26,23 +26,20 @@ import ( ) type LayeredMap struct { - layers []map[string]string // All layers with added files. - whiteouts []map[string]struct{} // All layers with deleted files. + adds []map[string]string // All layers with added files. + deletes []map[string]struct{} // All layers with deleted files. currentImage map[string]string // All files and hashes in the current image (up to the last layer). isCurrentImageValid bool // If the currentImage is not out-of-date. layerHashCache map[string]string hasher func(string) (string, error) - // cacheHasher doesn't include mtime in it's hash so that filesystem cache keys are stable - cacheHasher func(string) (string, error) } // NewLayeredMap creates a new layered map which keeps track of adds and deletes. -func NewLayeredMap(h func(string) (string, error), c func(string) (string, error)) *LayeredMap { +func NewLayeredMap(h func(string) (string, error)) *LayeredMap { l := LayeredMap{ - hasher: h, - cacheHasher: c, + hasher: h, } l.currentImage = map[string]string{} @@ -56,20 +53,29 @@ func (l *LayeredMap) Snapshot() { // Save current state of image l.updateCurrentImage() - l.whiteouts = append(l.whiteouts, map[string]struct{}{}) - l.layers = append(l.layers, map[string]string{}) + l.adds = append(l.adds, map[string]string{}) + l.deletes = append(l.deletes, map[string]struct{}{}) l.layerHashCache = map[string]string{} // Erase the hash cache for this new layer. } // Key returns a hash for added and delted files. func (l *LayeredMap) Key() (string, error) { + + var adds map[string]string + var deletes map[string]struct{} + + if len(l.adds) != 0 { + adds = l.adds[len(l.adds)-1] + deletes = l.deletes[len(l.deletes)-1] + } + c := bytes.NewBuffer([]byte{}) enc := json.NewEncoder(c) - err := enc.Encode(l.layers[len(l.layers)-1]) + err := enc.Encode(adds) if err != nil { return "", err } - err = enc.Encode(l.whiteouts[len(l.whiteouts)-1]) + err = enc.Encode(deletes) if err != nil { return "", err } @@ -79,7 +85,7 @@ func (l *LayeredMap) Key() (string, error) { // getCurrentImage returns the current image by merging the latest // adds and deletes on to the current image (if its not yet valid.) func (l *LayeredMap) getCurrentImage() map[string]string { - if l.isCurrentImageValid || len(l.layers) == 0 { + if l.isCurrentImageValid || len(l.adds) == 0 { // No layers yet or current image is valid. return l.currentImage } @@ -92,8 +98,8 @@ func (l *LayeredMap) getCurrentImage() map[string]string { } // Add the last layer on top. - addedFiles := l.layers[len(l.layers)-1] - deletedFiles := l.whiteouts[len(l.whiteouts)-1] + addedFiles := l.adds[len(l.adds)-1] + deletedFiles := l.deletes[len(l.deletes)-1] for add, hash := range addedFiles { current[add] = hash @@ -135,11 +141,11 @@ func (l *LayeredMap) GetCurrentPaths() map[string]struct{} { return paths } -// AddWhiteout will delete the specific files in the current layer. -func (l *LayeredMap) AddWhiteout(s string) error { +// AddDelete will delete the specific files in the current layer. +func (l *LayeredMap) AddDelete(s string) error { l.isCurrentImageValid = false - l.whiteouts[len(l.whiteouts)-1][s] = struct{}{} + l.deletes[len(l.deletes)-1][s] = struct{}{} return nil } @@ -159,7 +165,7 @@ func (l *LayeredMap) Add(s string) error { return fmt.Errorf("Error creating hash for %s: %w", s, err) } - l.layers[len(l.layers)-1][s] = newV + l.adds[len(l.adds)-1][s] = newV return nil } diff --git a/pkg/snapshot/layered_map_test.go b/pkg/snapshot/layered_map_test.go index d42003d0bd..fdcf44ca34 100644 --- a/pkg/snapshot/layered_map_test.go +++ b/pkg/snapshot/layered_map_test.go @@ -61,8 +61,8 @@ func Test_CacheKey(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - lm1 := LayeredMap{layers: []map[string]string{test.map1}} - lm2 := LayeredMap{layers: []map[string]string{test.map2}} + lm1 := LayeredMap{adds: []map[string]string{test.map1}, deletes: []map[string]struct{}{nil, nil}} + lm2 := LayeredMap{adds: []map[string]string{test.map2}, deletes: []map[string]struct{}{nil, nil}} k1, err := lm1.Key() if err != nil { t.Fatalf("error getting key for map 1: %v", err) @@ -106,8 +106,8 @@ func Test_FlattenPaths(t *testing.T) { } lm := LayeredMap{ - layers: []map[string]string{layers[0]}, - whiteouts: []map[string]struct{}{whiteouts[0]}} + adds: []map[string]string{layers[0]}, + deletes: []map[string]struct{}{whiteouts[0]}} paths := lm.GetCurrentPaths() @@ -124,8 +124,8 @@ func Test_FlattenPaths(t *testing.T) { assertPath("b", true) lm = LayeredMap{ - layers: []map[string]string{layers[0], layers[1]}, - whiteouts: []map[string]struct{}{whiteouts[0], whiteouts[1]}} + adds: []map[string]string{layers[0], layers[1]}, + deletes: []map[string]struct{}{whiteouts[0], whiteouts[1]}} paths = lm.GetCurrentPaths() assertPath("a", false) @@ -133,8 +133,8 @@ func Test_FlattenPaths(t *testing.T) { assertPath("c", true) lm = LayeredMap{ - layers: []map[string]string{layers[0], layers[1], layers[2]}, - whiteouts: []map[string]struct{}{whiteouts[0], whiteouts[1], whiteouts[2]}} + adds: []map[string]string{layers[0], layers[1], layers[2]}, + deletes: []map[string]struct{}{whiteouts[0], whiteouts[1], whiteouts[2]}} paths = lm.GetCurrentPaths() assertPath("a", true) diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 4d82f48ba0..815542aebf 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -101,7 +101,7 @@ func (s *Snapshotter) TakeSnapshot(files []string, shdCheckDelete bool, forceBui logrus.Debugf("Deleting in layer: %v", deletedFiles) // Whiteout files in current layer. for file := range deletedFiles { - if err := s.l.AddWhiteout(file); err != nil { + if err := s.l.AddDelete(file); err != nil { return "", fmt.Errorf("Unable to whiteout file %s in layered map: %w", file, err) } } @@ -186,7 +186,7 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { } } for file := range deletedPaths { - if err := s.l.AddWhiteout(file); err != nil { + if err := s.l.AddDelete(file); err != nil { return nil, nil, fmt.Errorf("Unable to whiteout file %s in layered map: %w", file, err) } } diff --git a/pkg/snapshot/snapshot_test.go b/pkg/snapshot/snapshot_test.go index fe80bd969b..e56dd43f90 100644 --- a/pkg/snapshot/snapshot_test.go +++ b/pkg/snapshot/snapshot_test.go @@ -585,7 +585,7 @@ func setUpTest(t *testing.T) (string, *Snapshotter, func(), error) { snapshotPathPrefix = snapshotPath // Take the initial snapshot - l := NewLayeredMap(util.Hasher(), util.CacheHasher()) + l := NewLayeredMap(util.Hasher()) snapshotter := NewSnapshotter(l, testDir) if err := snapshotter.Init(); err != nil { return "", nil, nil, errors.Wrap(err, "initializing snapshotter")