Merge pull request GoogleContainerTools#973 from cvgw/u/cgwippern/fix…

…-issue-704

Fix issue GoogleContainerTools#704

go.mod

@@ -36,6 +36,7 @@ require (
 	github.com/gliderlabs/ssh v0.2.2 // indirect
 	github.com/gogo/protobuf v1.1.1 // indirect
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
+	github.com/golang/mock v1.3.1
 	github.com/golang/protobuf v1.1.0 // indirect
 	github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a // indirect
 	github.com/google/go-cmp v0.2.0
@@ -94,7 +95,7 @@ require (
 	go.opencensus.io v0.14.0 // indirect
 	golang.org/x/net v0.0.0-20190311183353-d8887717615a
 	golang.org/x/oauth2 v0.0.0-20180724155351-3d292e4d0cdc
-	golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f
+	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect
 	google.golang.org/api v0.0.0-20180730000901-31ca0e01cd79 // indirect
 	google.golang.org/appengine v1.1.0 // indirect

go.sum

@@ -78,6 +78,8 @@ github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.1.0 h1:0iH4Ffd/meGoXqF2lSAhZHt8X+cPgkfn/cb6Cce5Vpc=
 github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -239,6 +241,8 @@ golang.org/x/oauth2 v0.0.0-20180724155351-3d292e4d0cdc h1:3ElrZeO6IBP+M8kgu5YFwR
 golang.org/x/oauth2 v0.0.0-20180724155351-3d292e4d0cdc/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190130150945-aca44879d564 h1:o6ENHFwwr1TZ9CUPQcfo1HGvLP1OPsPOTB7xCIOPNmU=
 golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -252,6 +256,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262 h1:qsl9y/CJx34tuA7QCPNp86JNJe4spst6Ff8MjvPUdPg=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 google.golang.org/api v0.0.0-20180730000901-31ca0e01cd79 h1:wCy2/9bhO1JeP2zZUALrj7ZdZuZoR4mRV57kTxjqRpo=
 google.golang.org/api v0.0.0-20180730000901-31ca0e01cd79/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=

integration/dockerfiles/Dockerfile_test_deleted_file_cached

@@ -0,0 +1,5 @@
+FROM alpine
+
+RUN mkdir -p /some/dir/ && echo 'first' > /some/dir/first.txt
+
+RUN rm /some/dir/first.txt

pkg/commands/copy.go

@@ -167,7 +167,13 @@ func (cr *CachingCopyCommand) ExecuteCommand(config *v1.Config, buildArgs *docke
 	if cr.img == nil {
 		return errors.New(fmt.Sprintf("cached command image is nil %v", cr.String()))
 	}
-	cr.extractedFiles, err = util.GetFSFromImage(RootDir, cr.img, cr.extractFn)
+
+	layers, err := cr.img.Layers()
+	if err != nil {
+		return errors.Wrapf(err, "retrieve image layers")
+	}
+
+	cr.extractedFiles, err = util.GetFSFromLayers(RootDir, layers, util.ExtractFunc(cr.extractFn), util.IncludeWhiteout())
 
 	logrus.Infof("extractedFiles: %s", cr.extractedFiles)
 	if err != nil {
@@ -182,7 +188,10 @@ func (cr *CachingCopyCommand) FilesUsedFromContext(config *v1.Config, buildArgs
 }
 
 func (cr *CachingCopyCommand) FilesToSnapshot() []string {
-	return cr.extractedFiles
+	f := cr.extractedFiles
+	logrus.Debugf("files extracted by caching copy command %s", f)
+
+	return f
 }
 
 func (cr *CachingCopyCommand) String() string {

pkg/commands/run.go

@@ -197,7 +197,18 @@ func (cr *CachingRunCommand) ExecuteCommand(config *v1.Config, buildArgs *docker
 	if cr.img == nil {
 		return errors.New(fmt.Sprintf("command image is nil %v", cr.String()))
 	}
-	cr.extractedFiles, err = util.GetFSFromImage(constants.RootDir, cr.img, cr.extractFn)
+
+	layers, err := cr.img.Layers()
+	if err != nil {
+		return errors.Wrap(err, "retrieving image layers")
+	}
+
+	cr.extractedFiles, err = util.GetFSFromLayers(
+		constants.RootDir,
+		layers,
+		util.ExtractFunc(cr.extractFn),
+		util.IncludeWhiteout(),
+	)
 	if err != nil {
 		return errors.Wrap(err, "extracting fs from image")
 	}
@@ -206,7 +217,10 @@ func (cr *CachingRunCommand) ExecuteCommand(config *v1.Config, buildArgs *docker
 }
 
 func (cr *CachingRunCommand) FilesToSnapshot() []string {
-	return cr.extractedFiles
+	f := cr.extractedFiles
+	logrus.Debugf("files extracted from caching run command %s", f)
+
+	return f
 }
 
 func (cr *CachingRunCommand) String() string {

pkg/util/fs_util.go

@@ -71,25 +71,58 @@ var excluded []string
 
 type ExtractFunction func(string, *tar.Header, io.Reader) error
 
+type FSConfig struct {
+	includeWhiteout bool
+	extractFunc     ExtractFunction
+}
+
+type FSOpt func(*FSConfig)
+
+func IncludeWhiteout() FSOpt {
+	return func(opts *FSConfig) {
+		opts.includeWhiteout = true
+	}
+}
+
+func ExtractFunc(extractFunc ExtractFunction) FSOpt {
+	return func(opts *FSConfig) {
+		opts.extractFunc = extractFunc
+	}
+}
+
 // GetFSFromImage extracts the layers of img to root
 // It returns a list of all files extracted
 func GetFSFromImage(root string, img v1.Image, extract ExtractFunction) ([]string, error) {
-	if extract == nil {
-		return nil, errors.New("must supply an extract function")
-	}
 	if img == nil {
 		return nil, errors.New("image cannot be nil")
 	}
-	if err := DetectFilesystemWhitelist(constants.WhitelistPath); err != nil {
-		return nil, err
-	}
-	logrus.Debugf("Mounted directories: %v", whitelist)
+
 	layers, err := img.Layers()
 	if err != nil {
 		return nil, err
 	}
-	extractedFiles := []string{}
 
+	return GetFSFromLayers(root, layers, ExtractFunc(extract))
+}
+
+func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, error) {
+	cfg := new(FSConfig)
+
+	for _, opt := range opts {
+		opt(cfg)
+	}
+
+	if cfg.extractFunc == nil {
+		return nil, errors.New("must supply an extract function")
+	}
+
+	if err := DetectFilesystemWhitelist(constants.WhitelistPath); err != nil {
+		return nil, err
+	}
+
+	logrus.Debugf("Mounted directories: %v", whitelist)
+
+	extractedFiles := []string{}
 	for i, l := range layers {
 		if mediaType, err := l.MediaType(); err == nil {
 			logrus.Tracef("Extracting layer %d of media type %s", i, mediaType)
@@ -102,29 +135,41 @@ func GetFSFromImage(root string, img v1.Image, extract ExtractFunction) ([]strin
 			return nil, err
 		}
 		defer r.Close()
+
 		tr := tar.NewReader(r)
 		for {
 			hdr, err := tr.Next()
 			if err == io.EOF {
 				break
 			}
+
 			if err != nil {
 				return nil, errors.Wrap(err, fmt.Sprintf("error reading tar %d", i))
 			}
+
 			path := filepath.Join(root, filepath.Clean(hdr.Name))
 			base := filepath.Base(path)
 			dir := filepath.Dir(path)
+
 			if strings.HasPrefix(base, ".wh.") {
 				logrus.Debugf("Whiting out %s", path)
+
 				name := strings.TrimPrefix(base, ".wh.")
 				if err := os.RemoveAll(filepath.Join(dir, name)); err != nil {
 					return nil, errors.Wrapf(err, "removing whiteout %s", hdr.Name)
 				}
-				continue
+
+				if !cfg.includeWhiteout {
+					logrus.Debug("not including whiteout files")
+					continue
+				}
+
 			}
-			if err := extract(root, hdr, tr); err != nil {
+
+			if err := cfg.extractFunc(root, hdr, tr); err != nil {
 				return nil, err
 			}
+
 			extractedFiles = append(extractedFiles, filepath.Join(root, filepath.Clean(hdr.Name)))
 		}
 	}