Skip to content

Commit

Permalink
session: modify FSSync provider to take fsutil.FS objects
Browse files Browse the repository at this point in the history 
This patch modifies the function signature of the FSSync provider to
take an fsutil.FS instead of a simple raw path resolved to the client's
root filesystem.

Internally, we were already creating an fsutil.FS to Send to the
buildkit server, however, this abstraction didn't reach the session
attachable parameters, so we couldn't provide our own custom FS
implementation.

The rationale behind this change is to allow providing more abstract
custom filesystem implementations to a BuildKit client. This way, we can
start to build from filesystems that might not be on disk - for example,
we could use our Static filesystem implementation in tests to prevent
creating lots of temporary directories, or we could use our Merge
filesystem implementation to allow easily creating variants of a single
context.

Signed-off-by: Justin Chadwell <[email protected]>
  • Loading branch information
@jedevc
jedevc committed Aug 8, 2023
1 parent 607f148 commit 2437e2f
Show file tree
Hide file tree
Showing 18 changed files with 496 additions and 191 deletions.
6 changes: 3 additions & 3 deletions cache/util/fsutil.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,17 +90,17 @@ type ReadDirRequest struct {
func ReadDir(ctx context.Context, mount snapshot.Mountable, req ReadDirRequest) ([]*fstypes.Stat, error) {
var (
rd []*fstypes.Stat
wo fsutil.WalkOpt
fo fsutil.FilterOpt
)
if req.IncludePattern != "" {
wo.IncludePatterns = append(wo.IncludePatterns, req.IncludePattern)
fo.IncludePatterns = append(fo.IncludePatterns, req.IncludePattern)
}
err := withMount(ctx, mount, func(root string) error {
fp, err := fs.RootPath(root, req.Path)
if err != nil {
return errors.WithStack(err)
}
return fsutil.Walk(ctx, fp, &wo, func(path string, info os.FileInfo, err error) error {
return fsutil.Walk(ctx, fp, &fo, func(path string, info os.FileInfo, err error) error {
if err != nil {
return errors.Wrapf(err, "walking %q", root)
}
Expand Down
40 changes: 26 additions & 14 deletions client/solve.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
return nil, errors.New("invalid with def and cb")
}

syncedDirs, err := prepareSyncedDirs(def, opt.LocalDirs)
syncedDirs, err := prepareSyncedFiles(def, opt.LocalDirs)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -361,26 +361,32 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
return res, nil
}

func prepareSyncedDirs(def *llb.Definition, localDirs map[string]string) (filesync.StaticDirSource, error) {
for _, d := range localDirs {
fi, err := os.Stat(d)
func prepareSyncedFiles(def *llb.Definition, localDirs map[string]string) (filesync.StaticDirSource, error) {
localFiles := make(map[string]fsutil.FS)
for name, d := range localDirs {
fs, err := fsutil.NewFS(d)
if err != nil {
return nil, errors.Wrapf(err, "could not find %s", d)
}
if !fi.IsDir() {
return nil, errors.Errorf("%s not a directory", d)
return nil, err
}
localFiles[name] = fs
}

resetUIDAndGID := func(p string, st *fstypes.Stat) fsutil.MapResult {
st.Uid = 0
st.Gid = 0
return fsutil.MapResultKeep
}

dirs := make(filesync.StaticDirSource, len(localDirs))
result := make(filesync.StaticDirSource, len(localFiles))
if def == nil {
for name, d := range localDirs {
dirs[name] = filesync.SyncedDir{Dir: d, Map: resetUIDAndGID}
for name, fs := range localFiles {
fs, err := fsutil.NewFilterFS(fs, &fsutil.FilterOpt{
Map: resetUIDAndGID,
})
if err != nil {
return nil, err
}
result[name] = fs
}
} else {
for _, dt := range def.Def {
Expand All @@ -391,16 +397,22 @@ func prepareSyncedDirs(def *llb.Definition, localDirs map[string]string) (filesy
if src := op.GetSource(); src != nil {
if strings.HasPrefix(src.Identifier, "local://") {
name := strings.TrimPrefix(src.Identifier, "local://")
d, ok := localDirs[name]
fs, ok := localFiles[name]
if !ok {
return nil, errors.Errorf("local directory %s not enabled", name)
}
dirs[name] = filesync.SyncedDir{Dir: d, Map: resetUIDAndGID}
fs, err := fsutil.NewFilterFS(fs, &fsutil.FilterOpt{
Map: resetUIDAndGID,
})
if err != nil {
return nil, err
}
result[name] = fs
}
}
}
}
return dirs, nil
return result, nil
}

func defaultSessionName() string {
Expand Down
4 changes: 2 additions & 2 deletions exporter/local/export.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,8 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source

if !e.opts.PlatformSplit {
// check for duplicate paths
err = outputFS.Walk(ctx, func(p string, fi os.FileInfo, err error) error {
if fi.IsDir() {
err = outputFS.Walk(ctx, "", func(p string, entry os.DirEntry, err error) error {
if entry.IsDir() {
return nil
}
if err != nil && !errors.Is(err, os.ErrNotExist) {
Expand Down
17 changes: 12 additions & 5 deletions exporter/local/fs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab
cleanup = lm.Unmount
}

walkOpt := &fsutil.WalkOpt{}
filterOpt := &fsutil.FilterOpt{}
var idMapFunc func(p string, st *fstypes.Stat) fsutil.MapResult

if idmap != nil {
Expand All @@ -116,7 +116,7 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab
}
}

walkOpt.Map = func(p string, st *fstypes.Stat) fsutil.MapResult {
filterOpt.Map = func(p string, st *fstypes.Stat) fsutil.MapResult {
res := fsutil.MapResultKeep
if idMapFunc != nil {
res = idMapFunc(p, st)
Expand All @@ -127,7 +127,14 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab
return res
}

outputFS := fsutil.NewFS(src, walkOpt)
outputFS, err := fsutil.NewFS(src)
if err != nil {
return nil, nil, err
}
outputFS, err = fsutil.NewFilterFS(outputFS, filterOpt)
if err != nil {
return nil, nil, err
}
attestations = attestation.Filter(attestations, nil, map[string][]byte{
result.AttestationInlineOnlyKey: []byte(strconv.FormatBool(true)),
})
Expand All @@ -137,11 +144,11 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab
}
if len(attestations) > 0 {
subjects := []intoto.Subject{}
err = outputFS.Walk(ctx, func(path string, info fs.FileInfo, err error) error {
err = outputFS.Walk(ctx, "", func(path string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
if !info.Mode().IsRegular() {
if !entry.Type().IsRegular() {
return nil
}
f, err := outputFS.Open(path)
Expand Down
2 changes: 2 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,3 +161,5 @@ require (
gopkg.in/yaml.v3 v3.0.1 // indirect
kernel.org/pub/linux/libs/security/libcap/psx v1.2.67 // indirect
)

replace github.com/tonistiigi/fsutil => github.com/jedevc/fsutil v0.0.0-20230808100152-f8658da7c74c
14 changes: 7 additions & 7 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59
github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
Expand Down Expand Up @@ -327,6 +328,7 @@ github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL
github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo=
github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y=
github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
github.com/containerd/continuity v0.3.1-0.20230206214859-2a963a2f56e8/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM=
github.com/containerd/continuity v0.4.1 h1:wQnVrjIyQ8vhU2sgOiL5T07jo+ouqc2bnKsv5/EqGhU=
github.com/containerd/continuity v0.4.1/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ=
github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
Expand Down Expand Up @@ -439,7 +441,6 @@ github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v0.0.0-20200511152416-a93e9eb0e95c/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v1.4.2-0.20180531152204-71cd53e4a197/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
Expand Down Expand Up @@ -790,6 +791,8 @@ github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07/go.mod h1:co9pwD
github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
github.com/jaguilar/vt100 v0.0.0-20150826170717-2703a27b14ea/go.mod h1:QMdK4dGB3YhEW2BmA1wgGpPYI3HZy/5gD705PXKUVSg=
github.com/jarcoal/httpmock v1.0.5/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
github.com/jedevc/fsutil v0.0.0-20230808100152-f8658da7c74c h1:KXyfIBdMcjWSKU6GmK6DUu+ZCnSUy+Bs36xSLzfKP0I=
github.com/jedevc/fsutil v0.0.0-20230808100152-f8658da7c74c/go.mod h1:SxX/oNQ/ag6Vaoli547ipFK9J7BZn5JqJG0JE8lf8bA=
github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s=
github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0=
Expand Down Expand Up @@ -908,7 +911,6 @@ github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo=
github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
github.com/moby/sys/mount v0.1.0/go.mod h1:FVQFLDRWwyBjDTBNQXDlWnSFREqOo3OKX9aqhmeoo74=
github.com/moby/sys/mount v0.1.1/go.mod h1:FVQFLDRWwyBjDTBNQXDlWnSFREqOo3OKX9aqhmeoo74=
github.com/moby/sys/mount v0.3.3 h1:fX1SVkXFJ47XWDoeFW4Sq7PdQJnV2QIDZAqjNqgEjUs=
github.com/moby/sys/mount v0.3.3/go.mod h1:PBaEorSNTLG5t/+4EgukEQVlAvVEc6ZjTySwKdqp5K0=
Expand Down Expand Up @@ -987,7 +989,6 @@ github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0
github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/opencontainers/runc v1.0.0-rc10/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/opencontainers/runc v1.0.0-rc92/go.mod h1:X1zlU4p7wOlX4+WRCz+hvlRv8phdL7UqbYD+vQwNMmE=
Expand Down Expand Up @@ -1204,9 +1205,6 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tommy-muehle/go-mnd v1.1.1/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig=
github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig=
github.com/tonistiigi/fsutil v0.0.0-20201103201449-0834f99b7b85/go.mod h1:a7cilN64dG941IOXfhJhlH0qB92hxJ9A1ewrdUmJ6xo=
github.com/tonistiigi/fsutil v0.0.0-20230629203738-36ef4d8c0dbb h1:uUe8rNyVXM8moActoBol6Xf6xX2GMr7SosR2EywMvGg=
github.com/tonistiigi/fsutil v0.0.0-20230629203738-36ef4d8c0dbb/go.mod h1:SxX/oNQ/ag6Vaoli547ipFK9J7BZn5JqJG0JE8lf8bA=
github.com/tonistiigi/go-actions-cache v0.0.0-20220404170428-0bdeb6e1eac7 h1:8eY6m1mjgyB8XySUR7WvebTM8D/Vs86jLJzD/Tw7zkc=
github.com/tonistiigi/go-actions-cache v0.0.0-20220404170428-0bdeb6e1eac7/go.mod h1:qqvyZqkfwkoJuPU/bw61bItaoO0SJ8YSW0vSVRRvsRg=
github.com/tonistiigi/go-archvariant v1.0.0 h1:5LC1eDWiBNflnTF1prCiX09yfNHIxDC/aukdhCdTyb0=
Expand Down Expand Up @@ -1450,6 +1448,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand Down Expand Up @@ -1522,7 +1521,6 @@ golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200917073148-efd3b9a0ff20/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201013081832-0aaa2718063a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand All @@ -1536,6 +1534,8 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220405210540-1e041c57c461/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down
20 changes: 9 additions & 11 deletions session/filesync/filesync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,20 +34,15 @@ type fsSyncProvider struct {
doneCh chan error
}

type SyncedDir struct {
Dir string
Map func(string, *fstypes.Stat) fsutil.MapResult
}

type DirSource interface {
LookupDir(string) (SyncedDir, bool)
LookupDir(string) (fsutil.FS, bool)
}

type StaticDirSource map[string]SyncedDir
type StaticDirSource map[string]fsutil.FS

var _ DirSource = StaticDirSource{}

func (dirs StaticDirSource) LookupDir(name string) (SyncedDir, bool) {
func (dirs StaticDirSource) LookupDir(name string) (fsutil.FS, bool) {
dir, found := dirs[name]
return dir, found
}
Expand Down Expand Up @@ -112,12 +107,15 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr
doneCh = sp.doneCh
sp.doneCh = nil
}
err := pr.sendFn(stream, fsutil.NewFS(dir.Dir, &fsutil.WalkOpt{
f, err := fsutil.NewFilterFS(dir, &fsutil.FilterOpt{
ExcludePatterns: excludes,
IncludePatterns: includes,
FollowPaths: followPaths,
Map: dir.Map,
}), progress)
})
if err != nil {
return err
}
err = pr.sendFn(stream, f, progress)
if doneCh != nil {
if err != nil {
doneCh <- err
Expand Down
7 changes: 5 additions & 2 deletions session/filesync/filesync_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/moby/buildkit/session/testutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/tonistiigi/fsutil"
"golang.org/x/sync/errgroup"
)

Expand All @@ -18,9 +19,11 @@ func TestFileSyncIncludePatterns(t *testing.T) {
t.Parallel()

tmpDir := t.TempDir()
tmpFS, err := fsutil.NewFS(tmpDir)
require.NoError(t, err)
destDir := t.TempDir()

err := os.WriteFile(filepath.Join(tmpDir, "foo"), []byte("content1"), 0600)
err = os.WriteFile(filepath.Join(tmpDir, "foo"), []byte("content1"), 0600)
require.NoError(t, err)

err = os.WriteFile(filepath.Join(tmpDir, "bar"), []byte("content2"), 0600)
Expand All @@ -32,7 +35,7 @@ func TestFileSyncIncludePatterns(t *testing.T) {
m, err := session.NewManager()
require.NoError(t, err)

fs := NewFSSyncProvider(StaticDirSource{"test0": {Dir: tmpDir}})
fs := NewFSSyncProvider(StaticDirSource{"test0": tmpFS})
s.Allow(fs)

dialer := session.Dialer(testutil.TestStream(testutil.Handler(m.HandleConn)))
Expand Down
21 changes: 10 additions & 11 deletions util/staticfs/merge.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"io"
"io/fs"
"os"
"path/filepath"

"github.com/tonistiigi/fsutil"
"golang.org/x/sync/errgroup"
Expand All @@ -26,9 +25,9 @@ func NewMergeFS(lower, upper fsutil.FS) *MergeFS {
}

type record struct {
path string
fi fs.FileInfo
err error
path string
entry fs.DirEntry
err error
}

func (r *record) key() string {
Expand All @@ -38,26 +37,26 @@ func (r *record) key() string {
return convertPathToKey(r.path)
}

func (mfs *MergeFS) Walk(ctx context.Context, fn filepath.WalkFunc) error {
func (mfs *MergeFS) Walk(ctx context.Context, target string, fn fs.WalkDirFunc) error {
ch1 := make(chan *record, 10)
ch2 := make(chan *record, 10)

eg, ctx := errgroup.WithContext(ctx)
eg.Go(func() error {
defer close(ch1)
return mfs.Lower.Walk(ctx, func(path string, info fs.FileInfo, err error) error {
return mfs.Lower.Walk(ctx, target, func(path string, entry fs.DirEntry, err error) error {
select {
case ch1 <- &record{path: path, fi: info, err: err}:
case ch1 <- &record{path: path, entry: entry, err: err}:
case <-ctx.Done():
}
return ctx.Err()
})
})
eg.Go(func() error {
defer close(ch2)
return mfs.Upper.Walk(ctx, func(path string, info fs.FileInfo, err error) error {
return mfs.Upper.Walk(ctx, target, func(path string, entry fs.DirEntry, err error) error {
select {
case ch2 <- &record{path: path, fi: info, err: err}:
case ch2 <- &record{path: path, entry: entry, err: err}:
case <-ctx.Done():
}
return ctx.Err()
Expand All @@ -75,13 +74,13 @@ func (mfs *MergeFS) Walk(ctx context.Context, fn filepath.WalkFunc) error {
break
}
if !ok2 || ok1 && key1 < key2 {
if err := fn(next1.path, next1.fi, next1.err); err != nil {
if err := fn(next1.path, next1.entry, next1.err); err != nil {
return err
}
next1, ok1 = <-ch1
key1 = next1.key()
} else if !ok1 || ok2 && key1 >= key2 {
if err := fn(next2.path, next2.fi, next2.err); err != nil {
if err := fn(next2.path, next2.entry, next2.err); err != nil {
return err
}
if ok1 && key1 == key2 {
Expand Down
Loading

0 comments on commit 2437e2f

Please sign in to comment.