diff --git a/cache/util/fsutil.go b/cache/util/fsutil.go index 162c164df288..6ed594c7b058 100644 --- a/cache/util/fsutil.go +++ b/cache/util/fsutil.go @@ -90,17 +90,17 @@ type ReadDirRequest struct { func ReadDir(ctx context.Context, mount snapshot.Mountable, req ReadDirRequest) ([]*fstypes.Stat, error) { var ( rd []*fstypes.Stat - wo fsutil.WalkOpt + fo fsutil.FilterOpt ) if req.IncludePattern != "" { - wo.IncludePatterns = append(wo.IncludePatterns, req.IncludePattern) + fo.IncludePatterns = append(fo.IncludePatterns, req.IncludePattern) } err := withMount(ctx, mount, func(root string) error { fp, err := fs.RootPath(root, req.Path) if err != nil { return errors.WithStack(err) } - return fsutil.Walk(ctx, fp, &wo, func(path string, info os.FileInfo, err error) error { + return fsutil.Walk(ctx, fp, &fo, func(path string, info os.FileInfo, err error) error { if err != nil { return errors.Wrapf(err, "walking %q", root) } diff --git a/client/solve.go b/client/solve.go index 22ff2031d442..04090ad1400f 100644 --- a/client/solve.go +++ b/client/solve.go @@ -35,7 +35,8 @@ import ( type SolveOpt struct { Exports []ExportEntry - LocalDirs map[string]string + LocalDirs map[string]string // Deprecated: use LocalMounts + LocalMounts map[string]fsutil.FS OCIStores map[string]content.Store SharedKey string Frontend string @@ -90,7 +91,11 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG return nil, errors.New("invalid with def and cb") } - syncedDirs, err := prepareSyncedDirs(def, opt.LocalDirs) + mounts, err := prepareMounts(&opt) + if err != nil { + return nil, err + } + syncedDirs, err := prepareSyncedFiles(def, mounts) if err != nil { return nil, err } @@ -361,26 +366,23 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG return res, nil } -func prepareSyncedDirs(def *llb.Definition, localDirs map[string]string) (filesync.StaticDirSource, error) { - for _, d := range localDirs { - fi, err := os.Stat(d) - if err != nil { - return nil, errors.Wrapf(err, "could not find %s", d) - } - if !fi.IsDir() { - return nil, errors.Errorf("%s not a directory", d) - } - } +func prepareSyncedFiles(def *llb.Definition, localMounts map[string]fsutil.FS) (filesync.StaticDirSource, error) { resetUIDAndGID := func(p string, st *fstypes.Stat) fsutil.MapResult { st.Uid = 0 st.Gid = 0 return fsutil.MapResultKeep } - dirs := make(filesync.StaticDirSource, len(localDirs)) + result := make(filesync.StaticDirSource, len(localMounts)) if def == nil { - for name, d := range localDirs { - dirs[name] = filesync.SyncedDir{Dir: d, Map: resetUIDAndGID} + for name, mount := range localMounts { + mount, err := fsutil.NewFilterFS(mount, &fsutil.FilterOpt{ + Map: resetUIDAndGID, + }) + if err != nil { + return nil, err + } + result[name] = mount } } else { for _, dt := range def.Def { @@ -391,16 +393,22 @@ func prepareSyncedDirs(def *llb.Definition, localDirs map[string]string) (filesy if src := op.GetSource(); src != nil { if strings.HasPrefix(src.Identifier, "local://") { name := strings.TrimPrefix(src.Identifier, "local://") - d, ok := localDirs[name] + mount, ok := localMounts[name] if !ok { return nil, errors.Errorf("local directory %s not enabled", name) } - dirs[name] = filesync.SyncedDir{Dir: d, Map: resetUIDAndGID} + mount, err := fsutil.NewFilterFS(mount, &fsutil.FilterOpt{ + Map: resetUIDAndGID, + }) + if err != nil { + return nil, err + } + result[name] = mount } } } } - return dirs, nil + return result, nil } func defaultSessionName() string { @@ -523,3 +531,22 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach } return &res, nil } + +func prepareMounts(opt *SolveOpt) (map[string]fsutil.FS, error) { + // merge local mounts and fallback local directories together + mounts := make(map[string]fsutil.FS) + for k, mount := range opt.LocalMounts { + mounts[k] = mount + } + for k, dir := range opt.LocalDirs { + mount, err := fsutil.NewFS(dir) + if err != nil { + return nil, err + } + if _, ok := mounts[k]; ok { + return nil, errors.Errorf("local mount %s already exists", k) + } + mounts[k] = mount + } + return mounts, nil +} diff --git a/exporter/local/export.go b/exporter/local/export.go index 771b7aaf2252..47186c018ddf 100644 --- a/exporter/local/export.go +++ b/exporter/local/export.go @@ -108,8 +108,8 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source if !e.opts.PlatformSplit { // check for duplicate paths - err = outputFS.Walk(ctx, func(p string, fi os.FileInfo, err error) error { - if fi.IsDir() { + err = outputFS.Walk(ctx, "", func(p string, entry os.DirEntry, err error) error { + if entry.IsDir() { return nil } if err != nil && !errors.Is(err, os.ErrNotExist) { diff --git a/exporter/local/fs.go b/exporter/local/fs.go index d8e4703ac146..b24d6aacda7a 100644 --- a/exporter/local/fs.go +++ b/exporter/local/fs.go @@ -98,9 +98,14 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab cleanup = lm.Unmount } - walkOpt := &fsutil.WalkOpt{} - var idMapFunc func(p string, st *fstypes.Stat) fsutil.MapResult + outputFS, err := fsutil.NewFS(src) + if err != nil { + return nil, nil, err + } + // wrap the output filesystem, applying appropriate filters + filterOpt := &fsutil.FilterOpt{} + var idMapFunc func(p string, st *fstypes.Stat) fsutil.MapResult if idmap != nil { idMapFunc = func(p string, st *fstypes.Stat) fsutil.MapResult { uid, gid, err := idmap.ToContainer(idtools.Identity{ @@ -115,19 +120,23 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab return fsutil.MapResultKeep } } - - walkOpt.Map = func(p string, st *fstypes.Stat) fsutil.MapResult { + filterOpt.Map = func(p string, st *fstypes.Stat) fsutil.MapResult { res := fsutil.MapResultKeep if idMapFunc != nil { + // apply host uid/gid res = idMapFunc(p, st) } if opt.Epoch != nil { + // apply used-specified epoch time st.ModTime = opt.Epoch.UnixNano() } return res } + outputFS, err = fsutil.NewFilterFS(outputFS, filterOpt) + if err != nil { + return nil, nil, err + } - outputFS := fsutil.NewFS(src, walkOpt) attestations = attestation.Filter(attestations, nil, map[string][]byte{ result.AttestationInlineOnlyKey: []byte(strconv.FormatBool(true)), }) @@ -137,11 +146,11 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab } if len(attestations) > 0 { subjects := []intoto.Subject{} - err = outputFS.Walk(ctx, func(path string, info fs.FileInfo, err error) error { + err = outputFS.Walk(ctx, "", func(path string, entry fs.DirEntry, err error) error { if err != nil { return err } - if !info.Mode().IsRegular() { + if !entry.Type().IsRegular() { return nil } f, err := outputFS.Open(path) diff --git a/go.mod b/go.mod index b784581903af..bc787fdb8f2a 100644 --- a/go.mod +++ b/go.mod @@ -64,7 +64,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spdx/tools-golang v0.5.1 github.com/stretchr/testify v1.8.4 - github.com/tonistiigi/fsutil v0.0.0-20230629203738-36ef4d8c0dbb + github.com/tonistiigi/fsutil v0.0.0-20230825212630-f09800878302 github.com/tonistiigi/go-actions-cache v0.0.0-20220404170428-0bdeb6e1eac7 github.com/tonistiigi/go-archvariant v1.0.0 github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea diff --git a/go.sum b/go.sum index 0077a348ff07..655d76a7c182 100644 --- a/go.sum +++ b/go.sum @@ -1206,8 +1206,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/tommy-muehle/go-mnd v1.1.1/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig= github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig= github.com/tonistiigi/fsutil v0.0.0-20201103201449-0834f99b7b85/go.mod h1:a7cilN64dG941IOXfhJhlH0qB92hxJ9A1ewrdUmJ6xo= -github.com/tonistiigi/fsutil v0.0.0-20230629203738-36ef4d8c0dbb h1:uUe8rNyVXM8moActoBol6Xf6xX2GMr7SosR2EywMvGg= -github.com/tonistiigi/fsutil v0.0.0-20230629203738-36ef4d8c0dbb/go.mod h1:SxX/oNQ/ag6Vaoli547ipFK9J7BZn5JqJG0JE8lf8bA= +github.com/tonistiigi/fsutil v0.0.0-20230825212630-f09800878302 h1:ZT8ibgassurSISJ1Pj26NsM3vY2jxFZn63Nd/TpHmRw= +github.com/tonistiigi/fsutil v0.0.0-20230825212630-f09800878302/go.mod h1:9kMVqMyQ/Sx2df5LtnGG+nbrmiZzCS7V6gjW3oGHsvI= github.com/tonistiigi/go-actions-cache v0.0.0-20220404170428-0bdeb6e1eac7 h1:8eY6m1mjgyB8XySUR7WvebTM8D/Vs86jLJzD/Tw7zkc= github.com/tonistiigi/go-actions-cache v0.0.0-20220404170428-0bdeb6e1eac7/go.mod h1:qqvyZqkfwkoJuPU/bw61bItaoO0SJ8YSW0vSVRRvsRg= github.com/tonistiigi/go-archvariant v1.0.0 h1:5LC1eDWiBNflnTF1prCiX09yfNHIxDC/aukdhCdTyb0= diff --git a/session/filesync/filesync.go b/session/filesync/filesync.go index f8d8385d462a..7254ddc08a39 100644 --- a/session/filesync/filesync.go +++ b/session/filesync/filesync.go @@ -35,20 +35,15 @@ type fsSyncProvider struct { doneCh chan error } -type SyncedDir struct { - Dir string - Map func(string, *fstypes.Stat) fsutil.MapResult -} - type DirSource interface { - LookupDir(string) (SyncedDir, bool) + LookupDir(string) (fsutil.FS, bool) } -type StaticDirSource map[string]SyncedDir +type StaticDirSource map[string]fsutil.FS var _ DirSource = StaticDirSource{} -func (dirs StaticDirSource) LookupDir(name string) (SyncedDir, bool) { +func (dirs StaticDirSource) LookupDir(name string) (fsutil.FS, bool) { dir, found := dirs[name] return dir, found } @@ -92,15 +87,22 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr dirName = name[0] } + excludes := opts[keyExcludePatterns] + includes := opts[keyIncludePatterns] + followPaths := opts[keyFollowPaths] + dir, ok := sp.dirs.LookupDir(dirName) if !ok { return InvalidSessionError{status.Errorf(codes.NotFound, "no access allowed to dir %q", dirName)} } - - excludes := opts[keyExcludePatterns] - includes := opts[keyIncludePatterns] - - followPaths := opts[keyFollowPaths] + dir, err := fsutil.NewFilterFS(dir, &fsutil.FilterOpt{ + ExcludePatterns: excludes, + IncludePatterns: includes, + FollowPaths: followPaths, + }) + if err != nil { + return err + } var progress progressCb if sp.p != nil { @@ -113,12 +115,7 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr doneCh = sp.doneCh sp.doneCh = nil } - err := pr.sendFn(stream, fsutil.NewFS(dir.Dir, &fsutil.WalkOpt{ - ExcludePatterns: excludes, - IncludePatterns: includes, - FollowPaths: followPaths, - Map: dir.Map, - }), progress) + err = pr.sendFn(stream, dir, progress) if doneCh != nil { if err != nil { doneCh <- err diff --git a/session/filesync/filesync_test.go b/session/filesync/filesync_test.go index 424ffe31f25b..70fcb5771bbf 100644 --- a/session/filesync/filesync_test.go +++ b/session/filesync/filesync_test.go @@ -10,6 +10,7 @@ import ( "github.com/moby/buildkit/session/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/tonistiigi/fsutil" "golang.org/x/sync/errgroup" ) @@ -18,9 +19,11 @@ func TestFileSyncIncludePatterns(t *testing.T) { t.Parallel() tmpDir := t.TempDir() + tmpFS, err := fsutil.NewFS(tmpDir) + require.NoError(t, err) destDir := t.TempDir() - err := os.WriteFile(filepath.Join(tmpDir, "foo"), []byte("content1"), 0600) + err = os.WriteFile(filepath.Join(tmpDir, "foo"), []byte("content1"), 0600) require.NoError(t, err) err = os.WriteFile(filepath.Join(tmpDir, "bar"), []byte("content2"), 0600) @@ -32,7 +35,7 @@ func TestFileSyncIncludePatterns(t *testing.T) { m, err := session.NewManager() require.NoError(t, err) - fs := NewFSSyncProvider(StaticDirSource{"test0": {Dir: tmpDir}}) + fs := NewFSSyncProvider(StaticDirSource{"test0": tmpFS}) s.Allow(fs) dialer := session.Dialer(testutil.TestStream(testutil.Handler(m.HandleConn))) diff --git a/util/staticfs/merge.go b/util/staticfs/merge.go index d680b80cfcce..0ff03f504861 100644 --- a/util/staticfs/merge.go +++ b/util/staticfs/merge.go @@ -5,7 +5,6 @@ import ( "io" "io/fs" "os" - "path/filepath" "github.com/tonistiigi/fsutil" "golang.org/x/sync/errgroup" @@ -26,9 +25,9 @@ func NewMergeFS(lower, upper fsutil.FS) *MergeFS { } type record struct { - path string - fi fs.FileInfo - err error + path string + entry fs.DirEntry + err error } func (r *record) key() string { @@ -38,16 +37,16 @@ func (r *record) key() string { return convertPathToKey(r.path) } -func (mfs *MergeFS) Walk(ctx context.Context, fn filepath.WalkFunc) error { +func (mfs *MergeFS) Walk(ctx context.Context, target string, fn fs.WalkDirFunc) error { ch1 := make(chan *record, 10) ch2 := make(chan *record, 10) eg, ctx := errgroup.WithContext(ctx) eg.Go(func() error { defer close(ch1) - return mfs.Lower.Walk(ctx, func(path string, info fs.FileInfo, err error) error { + return mfs.Lower.Walk(ctx, target, func(path string, entry fs.DirEntry, err error) error { select { - case ch1 <- &record{path: path, fi: info, err: err}: + case ch1 <- &record{path: path, entry: entry, err: err}: case <-ctx.Done(): } return ctx.Err() @@ -55,9 +54,9 @@ func (mfs *MergeFS) Walk(ctx context.Context, fn filepath.WalkFunc) error { }) eg.Go(func() error { defer close(ch2) - return mfs.Upper.Walk(ctx, func(path string, info fs.FileInfo, err error) error { + return mfs.Upper.Walk(ctx, target, func(path string, entry fs.DirEntry, err error) error { select { - case ch2 <- &record{path: path, fi: info, err: err}: + case ch2 <- &record{path: path, entry: entry, err: err}: case <-ctx.Done(): } return ctx.Err() @@ -75,13 +74,13 @@ func (mfs *MergeFS) Walk(ctx context.Context, fn filepath.WalkFunc) error { break } if !ok2 || ok1 && key1 < key2 { - if err := fn(next1.path, next1.fi, next1.err); err != nil { + if err := fn(next1.path, next1.entry, next1.err); err != nil { return err } next1, ok1 = <-ch1 key1 = next1.key() } else if !ok1 || ok2 && key1 >= key2 { - if err := fn(next2.path, next2.fi, next2.err); err != nil { + if err := fn(next2.path, next2.entry, next2.err); err != nil { return err } if ok1 && key1 == key2 { diff --git a/util/staticfs/merge_test.go b/util/staticfs/merge_test.go index 76dc165b0af1..26f665b59362 100644 --- a/util/staticfs/merge_test.go +++ b/util/staticfs/merge_test.go @@ -38,7 +38,9 @@ func TestMerge(t *testing.T) { require.Equal(t, []byte("barbarbar"), data) var files []string - err = fs.Walk(context.TODO(), func(path string, info iofs.FileInfo, err error) error { + err = fs.Walk(context.TODO(), "", func(path string, entry iofs.DirEntry, err error) error { + require.NoError(t, err) + info, err := entry.Info() require.NoError(t, err) switch path { case "foo": @@ -87,7 +89,7 @@ func TestMerge(t *testing.T) { require.True(t, os.IsNotExist(err)) files = nil - err = fs.Walk(context.TODO(), func(path string, info iofs.FileInfo, err error) error { + err = fs.Walk(context.TODO(), "", func(path string, entry iofs.DirEntry, err error) error { require.NoError(t, err) files = append(files, path) return nil diff --git a/util/staticfs/static.go b/util/staticfs/static.go index 3b00060688f0..3cfc50169218 100644 --- a/util/staticfs/static.go +++ b/util/staticfs/static.go @@ -4,8 +4,8 @@ import ( "bytes" "context" "io" + "io/fs" "os" - "path/filepath" "sort" "strings" @@ -31,6 +31,7 @@ func NewFS() *FS { } func (fs *FS) Add(p string, stat types.Stat, data []byte) { + p = strings.TrimPrefix(p, "/") stat.Size_ = int64(len(data)) if stat.Mode == 0 { stat.Mode = 0644 @@ -42,16 +43,20 @@ func (fs *FS) Add(p string, stat types.Stat, data []byte) { } } -func (fs *FS) Walk(ctx context.Context, fn filepath.WalkFunc) error { +func (fs *FS) Walk(ctx context.Context, target string, fn fs.WalkDirFunc) error { + target = strings.TrimPrefix(target, "/") keys := make([]string, 0, len(fs.files)) for k := range fs.files { + if !strings.HasPrefix(k, target) { + continue + } keys = append(keys, convertPathToKey(k)) } sort.Strings(keys) for _, k := range keys { p := convertKeyToPath(k) st := fs.files[p].Stat - if err := fn(p, &fsutil.StatInfo{Stat: &st}, nil); err != nil { + if err := fn(p, &fsutil.DirEntryInfo{Stat: &st}, nil); err != nil { return err } } diff --git a/util/staticfs/static_test.go b/util/staticfs/static_test.go index d8771af092d4..808c862eff7d 100644 --- a/util/staticfs/static_test.go +++ b/util/staticfs/static_test.go @@ -29,7 +29,9 @@ func TestStatic(t *testing.T) { require.True(t, os.IsNotExist(err)) var files []string - err = fs.Walk(context.TODO(), func(path string, info iofs.FileInfo, err error) error { + err = fs.Walk(context.TODO(), "", func(path string, entry iofs.DirEntry, err error) error { + require.NoError(t, err) + info, err := entry.Info() require.NoError(t, err) switch path { case "foo": diff --git a/vendor/github.com/tonistiigi/fsutil/walker.go b/vendor/github.com/tonistiigi/fsutil/filter.go similarity index 54% rename from vendor/github.com/tonistiigi/fsutil/walker.go rename to vendor/github.com/tonistiigi/fsutil/filter.go index 545f5e905f21..05f4a466f9ff 100644 --- a/vendor/github.com/tonistiigi/fsutil/walker.go +++ b/vendor/github.com/tonistiigi/fsutil/filter.go @@ -2,25 +2,35 @@ package fsutil import ( "context" + "io" gofs "io/fs" "os" "path/filepath" "strings" "syscall" - "time" "github.com/moby/patternmatcher" "github.com/pkg/errors" "github.com/tonistiigi/fsutil/types" ) -type WalkOpt struct { +type FilterOpt struct { + // IncludePatterns requires that the path matches at least one of the + // specified patterns. IncludePatterns []string + + // ExcludePatterns requires that the path does not match any of the + // specified patterns. ExcludePatterns []string - // FollowPaths contains symlinks that are resolved into include patterns - // before performing the fs walk + + // FollowPaths contains symlinks that are resolved into IncludePatterns + // at the time of the call to NewFilterFS. FollowPaths []string - Map MapFunc + + // Map is called for each path that is included in the result. + // The function can modify the stat info for each element, while the result + // of the function controls both how Walk continues. + Map MapFunc } type MapFunc func(string, *types.Stat) MapResult @@ -43,33 +53,41 @@ const ( MapResultSkipDir ) -func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) error { - root, err := filepath.EvalSymlinks(p) - if err != nil { - return errors.WithStack(&os.PathError{Op: "resolve", Path: root, Err: err}) - } - rootFI, err := os.Stat(root) - if err != nil { - return errors.WithStack(err) - } - if !rootFI.IsDir() { - return errors.WithStack(&os.PathError{Op: "walk", Path: root, Err: syscall.ENOTDIR}) - } +type filterFS struct { + fs FS - var ( - includePatterns []string - includeMatcher *patternmatcher.PatternMatcher - excludeMatcher *patternmatcher.PatternMatcher - ) + includeMatcher *patternmatcher.PatternMatcher + excludeMatcher *patternmatcher.PatternMatcher + onlyPrefixIncludes bool + onlyPrefixExcludeExceptions bool + + mapFn MapFunc +} + +// NewFilterFS creates a new FS that filters the given FS using the given +// FilterOpt. + +// The returned FS will not contain any paths that do not match the provided +// include and exclude patterns, or that are are exlcluded using the mapping +// function. +// +// The FS is assumed to be a snapshot of the filesystem at the time of the +// call to NewFilterFS. If the underlying filesystem changes, calls to the +// underlying FS may be inconsistent. +func NewFilterFS(fs FS, opt *FilterOpt) (FS, error) { + if opt == nil { + return fs, nil + } - if opt != nil && opt.IncludePatterns != nil { + var includePatterns []string + if opt.IncludePatterns != nil { includePatterns = make([]string, len(opt.IncludePatterns)) copy(includePatterns, opt.IncludePatterns) } - if opt != nil && opt.FollowPaths != nil { - targets, err := FollowLinks(p, opt.FollowPaths) + if opt.FollowPaths != nil { + targets, err := FollowLinks(fs, opt.FollowPaths) if err != nil { - return err + return nil, err } if targets != nil { includePatterns = append(includePatterns, targets...) @@ -82,11 +100,18 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err patternChars += `\` } - onlyPrefixIncludes := true - if len(includePatterns) != 0 { + var ( + includeMatcher *patternmatcher.PatternMatcher + excludeMatcher *patternmatcher.PatternMatcher + err error + onlyPrefixIncludes = true + onlyPrefixExcludeExceptions = true + ) + + if len(includePatterns) > 0 { includeMatcher, err = patternmatcher.New(includePatterns) if err != nil { - return errors.Wrapf(err, "invalid includepatterns: %s", opt.IncludePatterns) + return nil, errors.Wrapf(err, "invalid includepatterns: %s", opt.IncludePatterns) } for _, p := range includeMatcher.Patterns() { @@ -98,11 +123,10 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err } - onlyPrefixExcludeExceptions := true - if opt != nil && opt.ExcludePatterns != nil { + if len(opt.ExcludePatterns) > 0 { excludeMatcher, err = patternmatcher.New(opt.ExcludePatterns) if err != nil { - return errors.Wrapf(err, "invalid excludepatterns: %s", opt.ExcludePatterns) + return nil, errors.Wrapf(err, "invalid excludepatterns: %s", opt.ExcludePatterns) } for _, p := range excludeMatcher.Patterns() { @@ -113,10 +137,41 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err } } + return &filterFS{ + fs: fs, + includeMatcher: includeMatcher, + excludeMatcher: excludeMatcher, + onlyPrefixIncludes: onlyPrefixIncludes, + onlyPrefixExcludeExceptions: onlyPrefixExcludeExceptions, + mapFn: opt.Map, + }, nil +} + +func (fs *filterFS) Open(p string) (io.ReadCloser, error) { + if fs.includeMatcher != nil { + m, err := fs.includeMatcher.MatchesOrParentMatches(p) + if err != nil { + return nil, err + } + if !m { + return nil, errors.Wrapf(os.ErrNotExist, "open %s", p) + } + } + if fs.excludeMatcher != nil { + m, err := fs.excludeMatcher.MatchesOrParentMatches(p) + if err != nil { + return nil, err + } + if m { + return nil, errors.Wrapf(os.ErrNotExist, "open %s", p) + } + } + return fs.fs.Open(p) +} + +func (fs *filterFS) Walk(ctx context.Context, target string, fn gofs.WalkDirFunc) error { type visitedDir struct { - fi os.FileInfo - path string - origpath string + entry gofs.DirEntry pathWithSep string includeMatchInfo patternmatcher.MatchInfo excludeMatchInfo patternmatcher.MatchInfo @@ -126,34 +181,22 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err // used only for include/exclude handling var parentDirs []visitedDir - seenFiles := make(map[uint64]string) - return filepath.WalkDir(root, func(path string, dirEntry gofs.DirEntry, walkErr error) (retErr error) { + return fs.fs.Walk(ctx, target, func(path string, dirEntry gofs.DirEntry, walkErr error) (retErr error) { defer func() { if retErr != nil && isNotExist(retErr) { retErr = filepath.SkipDir } }() - origpath := path - path, err = filepath.Rel(root, path) - if err != nil { - return err - } - // Skip root - if path == "." { - return nil - } - var ( dir visitedDir isDir bool - fi gofs.FileInfo ) if dirEntry != nil { isDir = dirEntry.IsDir() } - if includeMatcher != nil || excludeMatcher != nil { + if fs.includeMatcher != nil || fs.excludeMatcher != nil { for len(parentDirs) != 0 { lastParentDir := parentDirs[len(parentDirs)-1].pathWithSep if strings.HasPrefix(path, lastParentDir) { @@ -163,15 +206,8 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err } if isDir { - fi, err = dirEntry.Info() - if err != nil { - return err - } - dir = visitedDir{ - fi: fi, - path: path, - origpath: origpath, + entry: dirEntry, pathWithSep: path + string(filepath.Separator), } } @@ -179,12 +215,12 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err skip := false - if includeMatcher != nil { + if fs.includeMatcher != nil { var parentIncludeMatchInfo patternmatcher.MatchInfo if len(parentDirs) != 0 { parentIncludeMatchInfo = parentDirs[len(parentDirs)-1].includeMatchInfo } - m, matchInfo, err := includeMatcher.MatchesUsingParentResults(path, parentIncludeMatchInfo) + m, matchInfo, err := fs.includeMatcher.MatchesUsingParentResults(path, parentIncludeMatchInfo) if err != nil { return errors.Wrap(err, "failed to match includepatterns") } @@ -194,11 +230,11 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err } if !m { - if isDir && onlyPrefixIncludes { + if isDir && fs.onlyPrefixIncludes { // Optimization: we can skip walking this dir if no include // patterns could match anything inside it. dirSlash := path + string(filepath.Separator) - for _, pat := range includeMatcher.Patterns() { + for _, pat := range fs.includeMatcher.Patterns() { if pat.Exclusion() { continue } @@ -214,12 +250,12 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err } } - if excludeMatcher != nil { + if fs.excludeMatcher != nil { var parentExcludeMatchInfo patternmatcher.MatchInfo if len(parentDirs) != 0 { parentExcludeMatchInfo = parentDirs[len(parentDirs)-1].excludeMatchInfo } - m, matchInfo, err := excludeMatcher.MatchesUsingParentResults(path, parentExcludeMatchInfo) + m, matchInfo, err := fs.excludeMatcher.MatchesUsingParentResults(path, parentExcludeMatchInfo) if err != nil { return errors.Wrap(err, "failed to match excludepatterns") } @@ -229,16 +265,16 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err } if m { - if isDir && onlyPrefixExcludeExceptions { + if isDir && fs.onlyPrefixExcludeExceptions { // Optimization: we can skip walking this dir if no // exceptions to exclude patterns could match anything // inside it. - if !excludeMatcher.Exclusions() { + if !fs.excludeMatcher.Exclusions() { return filepath.SkipDir } dirSlash := path + string(filepath.Separator) - for _, pat := range excludeMatcher.Patterns() { + for _, pat := range fs.excludeMatcher.Patterns() { if !pat.Exclusion() { continue } @@ -261,7 +297,7 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err return walkErr } - if includeMatcher != nil || excludeMatcher != nil { + if fs.includeMatcher != nil || fs.excludeMatcher != nil { defer func() { if isDir { parentDirs = append(parentDirs, dir) @@ -275,25 +311,21 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err dir.calledFn = true - // The FileInfo might have already been read further up. - if fi == nil { - fi, err = dirEntry.Info() - if err != nil { - return err - } - } - - stat, err := mkstat(origpath, path, fi, seenFiles) + fi, err := dirEntry.Info() if err != nil { return err } + stat, ok := fi.Sys().(*types.Stat) + if !ok { + return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"}) + } select { case <-ctx.Done(): return ctx.Err() default: - if opt != nil && opt.Map != nil { - result := opt.Map(stat.Path, stat) + if fs.mapFn != nil { + result := fs.mapFn(stat.Path, stat) if result == MapResultSkipDir { return filepath.SkipDir } else if result == MapResultExclude { @@ -304,29 +336,33 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err if parentDir.calledFn { continue } - parentStat, err := mkstat(parentDir.origpath, parentDir.path, parentDir.fi, seenFiles) + parentFi, err := parentDir.entry.Info() if err != nil { return err } + parentStat, ok := parentFi.Sys().(*types.Stat) + if !ok { + return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"}) + } select { case <-ctx.Done(): return ctx.Err() default: } - if opt != nil && opt.Map != nil { - result := opt.Map(parentStat.Path, parentStat) + if fs.mapFn != nil { + result := fs.mapFn(parentStat.Path, parentStat) if result == MapResultSkipDir || result == MapResultExclude { continue } } - if err := fn(parentStat.Path, &StatInfo{parentStat}, nil); err != nil { + if err := fn(parentStat.Path, &DirEntryInfo{Stat: parentStat}, nil); err != nil { return err } parentDirs[i].calledFn = true } - if err := fn(stat.Path, &StatInfo{stat}, nil); err != nil { + if err := fn(stat.Path, &DirEntryInfo{Stat: stat}, nil); err != nil { return err } } @@ -334,6 +370,40 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err }) } +func Walk(ctx context.Context, p string, opt *FilterOpt, fn filepath.WalkFunc) error { + f, err := NewFS(p) + if err != nil { + return err + } + f, err = NewFilterFS(f, opt) + if err != nil { + return err + } + return f.Walk(ctx, "/", func(path string, d gofs.DirEntry, err error) error { + var info gofs.FileInfo + if d != nil { + var err2 error + info, err2 = d.Info() + if err == nil { + err = err2 + } + } + return fn(path, info, err) + }) +} + +func WalkDir(ctx context.Context, p string, opt *FilterOpt, fn gofs.WalkDirFunc) error { + f, err := NewFS(p) + if err != nil { + return err + } + f, err = NewFilterFS(f, opt) + if err != nil { + return err + } + return f.Walk(ctx, "/", fn) +} + func patternWithoutTrailingGlob(p *patternmatcher.Pattern) string { patStr := p.String() // We use filepath.Separator here because patternmatcher.Pattern patterns @@ -344,29 +414,6 @@ func patternWithoutTrailingGlob(p *patternmatcher.Pattern) string { return patStr } -type StatInfo struct { - *types.Stat -} - -func (s *StatInfo) Name() string { - return filepath.Base(s.Stat.Path) -} -func (s *StatInfo) Size() int64 { - return s.Stat.Size_ -} -func (s *StatInfo) Mode() os.FileMode { - return os.FileMode(s.Stat.Mode) -} -func (s *StatInfo) ModTime() time.Time { - return time.Unix(s.Stat.ModTime/1e9, s.Stat.ModTime%1e9) -} -func (s *StatInfo) IsDir() bool { - return s.Mode().IsDir() -} -func (s *StatInfo) Sys() interface{} { - return s.Stat -} - func isNotExist(err error) bool { return errors.Is(err, os.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) } diff --git a/vendor/github.com/tonistiigi/fsutil/followlinks.go b/vendor/github.com/tonistiigi/fsutil/followlinks.go index f03a9cf3500b..3a0bd2bb790a 100644 --- a/vendor/github.com/tonistiigi/fsutil/followlinks.go +++ b/vendor/github.com/tonistiigi/fsutil/followlinks.go @@ -1,17 +1,21 @@ package fsutil import ( + "context" + gofs "io/fs" "os" "path/filepath" "runtime" "sort" strings "strings" + "syscall" "github.com/pkg/errors" + "github.com/tonistiigi/fsutil/types" ) -func FollowLinks(root string, paths []string) ([]string, error) { - r := &symlinkResolver{root: root, resolved: map[string]struct{}{}} +func FollowLinks(fs FS, paths []string) ([]string, error) { + r := &symlinkResolver{fs: fs, resolved: map[string]struct{}{}} for _, p := range paths { if err := r.append(p); err != nil { return nil, err @@ -26,7 +30,7 @@ func FollowLinks(root string, paths []string) ([]string, error) { } type symlinkResolver struct { - root string + fs FS resolved map[string]struct{} } @@ -76,10 +80,9 @@ func (r *symlinkResolver) append(p string) error { } func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, error) { - realPath := filepath.Join(r.root, p) base := filepath.Base(p) if allowWildcard && containsWildcards(base) { - fis, err := os.ReadDir(filepath.Dir(realPath)) + fis, err := readDir(r.fs, filepath.Dir(p)) if err != nil { if isNotFound(err) { return nil, nil @@ -99,21 +102,30 @@ func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, e return out, nil } - fi, err := os.Lstat(realPath) + entry, err := statFile(r.fs, p) if err != nil { if isNotFound(err) { return nil, nil } return nil, errors.WithStack(err) } - if fi.Mode()&os.ModeSymlink == 0 { + if entry == nil { return nil, nil } - link, err := os.Readlink(realPath) + if entry.Type()&os.ModeSymlink == 0 { + return nil, nil + } + + fi, err := entry.Info() if err != nil { return nil, errors.WithStack(err) } - link = filepath.Clean(link) + stat, ok := fi.Sys().(*types.Stat) + if !ok { + return nil, errors.WithStack(&os.PathError{Path: p, Err: syscall.EBADMSG, Op: "fileinfo without stat info"}) + } + + link := filepath.Clean(stat.Linkname) if filepath.IsAbs(link) { return []string{link}, nil } @@ -122,6 +134,76 @@ func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, e }, nil } +func statFile(fs FS, root string) (os.DirEntry, error) { + var out os.DirEntry + + root = filepath.Clean(root) + if root == "/" || root == "." { + return nil, nil + } + + err := fs.Walk(context.TODO(), root, func(p string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if p != root { + return errors.Errorf("expected single entry %q but got %q", root, p) + } + out = entry + if entry.IsDir() { + return filepath.SkipDir + } + return nil + }) + if err != nil { + return nil, err + } + + if out == nil { + return nil, errors.Wrapf(os.ErrNotExist, "readFile %s", root) + } + return out, nil +} + +func readDir(fs FS, root string) ([]os.DirEntry, error) { + var out []os.DirEntry + + root = filepath.Clean(root) + if root == "/" || root == "." { + root = "." + out = make([]gofs.DirEntry, 0) + } + + err := fs.Walk(context.TODO(), root, func(p string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if p == root { + if !entry.IsDir() { + return errors.WithStack(&os.PathError{Op: "walk", Path: root, Err: syscall.ENOTDIR}) + } + out = make([]gofs.DirEntry, 0) + return nil + } + if out == nil { + return errors.Errorf("expected to read parent entry %q before child %q", root, p) + } + out = append(out, entry) + if entry.IsDir() { + return filepath.SkipDir + } + return nil + }) + if err != nil { + return nil, err + } + + if out == nil && root != "." { + return nil, errors.Wrapf(os.ErrNotExist, "readDir %s", root) + } + return out, nil +} + func containsWildcards(name string) bool { isWindows := runtime.GOOS == "windows" for i := 0; i < len(name); i++ { diff --git a/vendor/github.com/tonistiigi/fsutil/fs.go b/vendor/github.com/tonistiigi/fsutil/fs.go index db587b77cd80..fe370194bed3 100644 --- a/vendor/github.com/tonistiigi/fsutil/fs.go +++ b/vendor/github.com/tonistiigi/fsutil/fs.go @@ -3,36 +3,86 @@ package fsutil import ( "context" "io" + gofs "io/fs" "os" "path" "path/filepath" "sort" "strings" "syscall" + "time" "github.com/pkg/errors" "github.com/tonistiigi/fsutil/types" ) type FS interface { - Walk(context.Context, filepath.WalkFunc) error + Walk(context.Context, string, gofs.WalkDirFunc) error Open(string) (io.ReadCloser, error) } -func NewFS(root string, opt *WalkOpt) FS { +// NewFS creates a new FS from a root directory on the host filesystem. +func NewFS(root string) (FS, error) { + root, err := filepath.EvalSymlinks(root) + if err != nil { + return nil, errors.WithStack(&os.PathError{Op: "resolve", Path: root, Err: err}) + } + fi, err := os.Stat(root) + if err != nil { + return nil, err + } + if !fi.IsDir() { + return nil, errors.WithStack(&os.PathError{Op: "stat", Path: root, Err: syscall.ENOTDIR}) + } + return &fs{ root: root, - opt: opt, - } + }, nil } type fs struct { root string - opt *WalkOpt } -func (fs *fs) Walk(ctx context.Context, fn filepath.WalkFunc) error { - return Walk(ctx, fs.root, fs.opt, fn) +func (fs *fs) Walk(ctx context.Context, target string, fn gofs.WalkDirFunc) error { + seenFiles := make(map[uint64]string) + return filepath.WalkDir(filepath.Join(fs.root, target), func(path string, dirEntry gofs.DirEntry, walkErr error) (retErr error) { + defer func() { + if retErr != nil && isNotExist(retErr) { + retErr = filepath.SkipDir + } + }() + + origpath := path + path, err := filepath.Rel(fs.root, path) + if err != nil { + return err + } + // Skip root + if path == "." { + return nil + } + + var entry gofs.DirEntry + if dirEntry != nil { + entry = &DirEntryInfo{ + path: path, + origpath: origpath, + entry: dirEntry, + seenFiles: seenFiles, + } + } + + select { + case <-ctx.Done(): + return ctx.Err() + default: + if err := fn(path, entry, walkErr); err != nil { + return err + } + } + return nil + }) } func (fs *fs) Open(p string) (io.ReadCloser, error) { @@ -67,16 +117,31 @@ type subDirFS struct { dirs []Dir } -func (fs *subDirFS) Walk(ctx context.Context, fn filepath.WalkFunc) error { +func (fs *subDirFS) Walk(ctx context.Context, target string, fn gofs.WalkDirFunc) error { + first, rest, _ := strings.Cut(target, string(filepath.Separator)) + for _, d := range fs.dirs { - fi := &StatInfo{Stat: &d.Stat} + if first != "" && first != d.Stat.Path { + continue + } + + fi := &StatInfo{&d.Stat} if !fi.IsDir() { return errors.WithStack(&os.PathError{Path: d.Stat.Path, Err: syscall.ENOTDIR, Op: "walk subdir"}) } - if err := fn(d.Stat.Path, fi, nil); err != nil { + dStat := d.Stat + if err := fn(d.Stat.Path, &DirEntryInfo{Stat: &dStat}, nil); err != nil { return err } - if err := d.FS.Walk(ctx, func(p string, fi os.FileInfo, err error) error { + if err := d.FS.Walk(ctx, rest, func(p string, entry gofs.DirEntry, err error) error { + if err != nil { + return err + } + + fi, err := entry.Info() + if err != nil { + return err + } stat, ok := fi.Sys().(*types.Stat) if !ok { return errors.WithStack(&os.PathError{Path: d.Stat.Path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"}) @@ -91,7 +156,7 @@ func (fs *subDirFS) Walk(ctx context.Context, fn filepath.WalkFunc) error { stat.Linkname = path.Join(d.Stat.Path, stat.Linkname) } } - return fn(filepath.Join(d.Stat.Path, p), &StatInfo{stat}, nil) + return fn(filepath.Join(d.Stat.Path, p), &DirEntryInfo{Stat: stat}, nil) }); err != nil { return err } @@ -117,3 +182,70 @@ type emptyReader struct { func (*emptyReader) Read([]byte) (int, error) { return 0, io.EOF } + +type StatInfo struct { + *types.Stat +} + +func (s *StatInfo) Name() string { + return filepath.Base(s.Stat.Path) +} +func (s *StatInfo) Size() int64 { + return s.Stat.Size_ +} +func (s *StatInfo) Mode() os.FileMode { + return os.FileMode(s.Stat.Mode) +} +func (s *StatInfo) ModTime() time.Time { + return time.Unix(s.Stat.ModTime/1e9, s.Stat.ModTime%1e9) +} +func (s *StatInfo) IsDir() bool { + return s.Mode().IsDir() +} +func (s *StatInfo) Sys() interface{} { + return s.Stat +} + +type DirEntryInfo struct { + *types.Stat + + entry gofs.DirEntry + path string + origpath string + seenFiles map[uint64]string +} + +func (s *DirEntryInfo) Name() string { + if s.Stat != nil { + return filepath.Base(s.Stat.Path) + } + return s.entry.Name() +} +func (s *DirEntryInfo) IsDir() bool { + if s.Stat != nil { + return s.Stat.IsDir() + } + return s.entry.IsDir() +} +func (s *DirEntryInfo) Type() gofs.FileMode { + if s.Stat != nil { + return gofs.FileMode(s.Stat.Mode) + } + return s.entry.Type() +} +func (s *DirEntryInfo) Info() (gofs.FileInfo, error) { + if s.Stat == nil { + fi, err := s.entry.Info() + if err != nil { + return nil, err + } + stat, err := mkstat(s.origpath, s.path, fi, s.seenFiles) + if err != nil { + return nil, err + } + s.Stat = stat + } + + st := *s.Stat + return &StatInfo{&st}, nil +} diff --git a/vendor/github.com/tonistiigi/fsutil/send.go b/vendor/github.com/tonistiigi/fsutil/send.go index f1c51b83652d..a044d04d8385 100644 --- a/vendor/github.com/tonistiigi/fsutil/send.go +++ b/vendor/github.com/tonistiigi/fsutil/send.go @@ -144,7 +144,11 @@ func (s *sender) sendFile(h *sendHandle) error { func (s *sender) walk(ctx context.Context) error { var i uint32 = 0 - err := s.fs.Walk(ctx, func(path string, fi os.FileInfo, err error) error { + err := s.fs.Walk(ctx, "/", func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + fi, err := entry.Info() if err != nil { return err } diff --git a/vendor/github.com/tonistiigi/fsutil/tarwriter.go b/vendor/github.com/tonistiigi/fsutil/tarwriter.go index bd46a2250ff8..06b7bda9bec0 100644 --- a/vendor/github.com/tonistiigi/fsutil/tarwriter.go +++ b/vendor/github.com/tonistiigi/fsutil/tarwriter.go @@ -15,10 +15,15 @@ import ( func WriteTar(ctx context.Context, fs FS, w io.Writer) error { tw := tar.NewWriter(w) - err := fs.Walk(ctx, func(path string, fi os.FileInfo, err error) error { + err := fs.Walk(ctx, "/", func(path string, entry os.DirEntry, err error) error { if err != nil && !errors.Is(err, os.ErrNotExist) { return err } + + fi, err := entry.Info() + if err != nil { + return err + } stat, ok := fi.Sys().(*types.Stat) if !ok { return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"}) diff --git a/vendor/modules.txt b/vendor/modules.txt index b511cb64ce82..cf458c3a02e8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -742,7 +742,7 @@ github.com/spdx/tools-golang/spdx/v2/v2_3 ## explicit; go 1.20 github.com/stretchr/testify/assert github.com/stretchr/testify/require -# github.com/tonistiigi/fsutil v0.0.0-20230629203738-36ef4d8c0dbb +# github.com/tonistiigi/fsutil v0.0.0-20230825212630-f09800878302 ## explicit; go 1.19 github.com/tonistiigi/fsutil github.com/tonistiigi/fsutil/copy