From d9be60974b694a17e5c6c3e71fb7767e6bfe17e9 Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Sat, 27 Jan 2024 12:52:56 +0800 Subject: [PATCH] os: implement CopyFS Fixes #62484 Change-Id: I5d8950dedf86af48f42a641940b34e62aa2cddcb Reviewed-on: https://go-review.googlesource.com/c/go/+/558995 Auto-Submit: Ian Lance Taylor Reviewed-by: Ian Lance Taylor Reviewed-by: Dmitri Shuralyov LUCI-TryBot-Result: Go LUCI --- api/next/62484.txt | 1 + doc/next/6-stdlib/99-minor/os/62484.md | 2 + src/os/dir.go | 60 ++++++ src/os/os_test.go | 276 ++++++++++++++++++++++--- 4 files changed, 316 insertions(+), 23 deletions(-) create mode 100644 api/next/62484.txt create mode 100644 doc/next/6-stdlib/99-minor/os/62484.md diff --git a/api/next/62484.txt b/api/next/62484.txt new file mode 100644 index 00000000000000..7f5b5ca90c51f7 --- /dev/null +++ b/api/next/62484.txt @@ -0,0 +1 @@ +pkg os, func CopyFS(string, fs.FS) error #62484 diff --git a/doc/next/6-stdlib/99-minor/os/62484.md b/doc/next/6-stdlib/99-minor/os/62484.md new file mode 100644 index 00000000000000..81abb4bc684fa3 --- /dev/null +++ b/doc/next/6-stdlib/99-minor/os/62484.md @@ -0,0 +1,2 @@ +The [`CopyFS`](/os#CopyFS) function copies an [`io/fs.FS`](/io/fs#FS) +into the local filesystem. diff --git a/src/os/dir.go b/src/os/dir.go index 5306bcb3ba7c2b..5c15127bc170e6 100644 --- a/src/os/dir.go +++ b/src/os/dir.go @@ -5,6 +5,8 @@ package os import ( + "internal/safefilepath" + "io" "io/fs" "sort" ) @@ -123,3 +125,61 @@ func ReadDir(name string) ([]DirEntry, error) { sort.Slice(dirs, func(i, j int) bool { return dirs[i].Name() < dirs[j].Name() }) return dirs, err } + +// CopyFS copies the file system fsys into the directory dir, +// creating dir if necessary. +// +// Newly created directories and files have their default modes +// where any bits from the file in fsys that are not part of the +// standard read, write, and execute permissions will be zeroed +// out, and standard read and write permissions are set for owner, +// group, and others while retaining any existing execute bits from +// the file in fsys. +// +// Symbolic links in fsys are not supported, a *PathError with Err set +// to ErrInvalid is returned on symlink. +// +// Copying stops at and returns the first error encountered. +func CopyFS(dir string, fsys fs.FS) error { + return fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + fpath, err := safefilepath.FromFS(path) + if err != nil { + return err + } + newPath := joinPath(dir, fpath) + if d.IsDir() { + return MkdirAll(newPath, 0777) + } + + // TODO(panjf2000): handle symlinks with the help of fs.ReadLinkFS + // once https://go.dev/issue/49580 is done. + // we also need safefilepath.IsLocal from https://go.dev/cl/564295. + if !d.Type().IsRegular() { + return &PathError{Op: "CopyFS", Path: path, Err: ErrInvalid} + } + + r, err := fsys.Open(path) + if err != nil { + return err + } + defer r.Close() + info, err := r.Stat() + if err != nil { + return err + } + w, err := OpenFile(newPath, O_CREATE|O_TRUNC|O_WRONLY, 0666|info.Mode()&0777) + if err != nil { + return err + } + + if _, err := io.Copy(w, r); err != nil { + w.Close() + return &PathError{Op: "Copy", Path: newPath, Err: err} + } + return w.Close() + }) +} diff --git a/src/os/os_test.go b/src/os/os_test.go index 6adc3b547924be..e094edd8264f11 100644 --- a/src/os/os_test.go +++ b/src/os/os_test.go @@ -5,6 +5,7 @@ package os_test import ( + "bytes" "errors" "flag" "fmt" @@ -3030,35 +3031,44 @@ func TestOpenFileKeepsPermissions(t *testing.T) { } } -func TestDirFS(t *testing.T) { - t.Parallel() +func forceMFTUpdateOnWindows(t *testing.T, path string) { + t.Helper() + + if runtime.GOOS != "windows" { + return + } // On Windows, we force the MFT to update by reading the actual metadata from GetFileInformationByHandle and then // explicitly setting that. Otherwise it might get out of sync with FindFirstFile. See golang.org/issues/42637. - if runtime.GOOS == "windows" { - if err := filepath.WalkDir("./testdata/dirfs", func(path string, d fs.DirEntry, err error) error { - if err != nil { - t.Fatal(err) - } - info, err := d.Info() - if err != nil { - t.Fatal(err) - } - stat, err := Stat(path) // This uses GetFileInformationByHandle internally. - if err != nil { - t.Fatal(err) - } - if stat.ModTime() == info.ModTime() { - return nil - } - if err := Chtimes(path, stat.ModTime(), stat.ModTime()); err != nil { - t.Log(err) // We only log, not die, in case the test directory is not writable. - } - return nil - }); err != nil { + if err := filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { + if err != nil { + t.Fatal(err) + } + info, err := d.Info() + if err != nil { t.Fatal(err) } + stat, err := Stat(path) // This uses GetFileInformationByHandle internally. + if err != nil { + t.Fatal(err) + } + if stat.ModTime() == info.ModTime() { + return nil + } + if err := Chtimes(path, stat.ModTime(), stat.ModTime()); err != nil { + t.Log(err) // We only log, not die, in case the test directory is not writable. + } + return nil + }); err != nil { + t.Fatal(err) } +} + +func TestDirFS(t *testing.T) { + t.Parallel() + + forceMFTUpdateOnWindows(t, "./testdata/dirfs") + fsys := DirFS("./testdata/dirfs") if err := fstest.TestFS(fsys, "a", "b", "dir/x"); err != nil { t.Fatal(err) @@ -3358,3 +3368,223 @@ func TestRandomLen(t *testing.T) { } } } + +func TestCopyFS(t *testing.T) { + t.Parallel() + + // Test with disk filesystem. + forceMFTUpdateOnWindows(t, "./testdata/dirfs") + fsys := DirFS("./testdata/dirfs") + tmpDir := t.TempDir() + if err := CopyFS(tmpDir, fsys); err != nil { + t.Fatal("CopyFS:", err) + } + forceMFTUpdateOnWindows(t, tmpDir) + tmpFsys := DirFS(tmpDir) + if err := fstest.TestFS(tmpFsys, "a", "b", "dir/x"); err != nil { + t.Fatal("TestFS:", err) + } + if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error { + if d.IsDir() { + return nil + } + + data, err := fs.ReadFile(fsys, path) + if err != nil { + return err + } + newData, err := fs.ReadFile(tmpFsys, path) + if err != nil { + return err + } + if !bytes.Equal(data, newData) { + return errors.New("file " + path + " contents differ") + } + return nil + }); err != nil { + t.Fatal("comparing two directories:", err) + } + + // Test with memory filesystem. + fsys = fstest.MapFS{ + "william": {Data: []byte("Shakespeare\n")}, + "carl": {Data: []byte("Gauss\n")}, + "daVinci": {Data: []byte("Leonardo\n")}, + "einstein": {Data: []byte("Albert\n")}, + "dir/newton": {Data: []byte("Sir Isaac\n")}, + } + tmpDir = t.TempDir() + if err := CopyFS(tmpDir, fsys); err != nil { + t.Fatal("CopyFS:", err) + } + forceMFTUpdateOnWindows(t, tmpDir) + tmpFsys = DirFS(tmpDir) + if err := fstest.TestFS(tmpFsys, "william", "carl", "daVinci", "einstein", "dir/newton"); err != nil { + t.Fatal("TestFS:", err) + } + if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error { + if d.IsDir() { + return nil + } + + data, err := fs.ReadFile(fsys, path) + if err != nil { + return err + } + newData, err := fs.ReadFile(tmpFsys, path) + if err != nil { + return err + } + if !bytes.Equal(data, newData) { + return errors.New("file " + path + " contents differ") + } + return nil + }); err != nil { + t.Fatal("comparing two directories:", err) + } +} + +func TestCopyFSWithSymlinks(t *testing.T) { + // Test it with absolute and relative symlinks that point inside and outside the tree. + testenv.MustHaveSymlink(t) + + // Create a directory and file outside. + tmpDir := t.TempDir() + outsideDir, err := MkdirTemp(tmpDir, "copyfs") + if err != nil { + t.Fatalf("MkdirTemp: %v", err) + } + outsideFile := filepath.Join(outsideDir, "file.out.txt") + + if err := WriteFile(outsideFile, []byte("Testing CopyFS outside"), 0644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + // Create a directory and file inside. + testDataDir, err := filepath.Abs("./testdata/") + if err != nil { + t.Fatalf("filepath.Abs: %v", err) + } + insideDir := filepath.Join(testDataDir, "copyfs") + if err := Mkdir(insideDir, 0755); err != nil { + t.Fatalf("Mkdir: %v", err) + } + defer RemoveAll(insideDir) + insideFile := filepath.Join(insideDir, "file.in.txt") + if err := WriteFile(insideFile, []byte("Testing CopyFS inside"), 0644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + // Create directories for symlinks. + linkInDir := filepath.Join(insideDir, "in_symlinks") + if err := Mkdir(linkInDir, 0755); err != nil { + t.Fatalf("Mkdir: %v", err) + } + linkOutDir := filepath.Join(insideDir, "out_symlinks") + if err := Mkdir(linkOutDir, 0755); err != nil { + t.Fatalf("Mkdir: %v", err) + } + + // First, we create the absolute symlink pointing outside. + outLinkFile := filepath.Join(linkOutDir, "file.abs.out.link") + if err := Symlink(outsideFile, outLinkFile); err != nil { + t.Fatalf("Symlink: %v", err) + } + + // Then, we create the relative symlink pointing outside. + relOutsideFile, err := filepath.Rel(filepath.Join(linkOutDir, "."), outsideFile) + if err != nil { + t.Fatalf("filepath.Rel: %v", err) + } + relOutLinkFile := filepath.Join(linkOutDir, "file.rel.out.link") + if err := Symlink(relOutsideFile, relOutLinkFile); err != nil { + t.Fatalf("Symlink: %v", err) + } + + // Last, we create the relative symlink pointing inside. + relInsideFile, err := filepath.Rel(filepath.Join(linkInDir, "."), insideFile) + if err != nil { + t.Fatalf("filepath.Rel: %v", err) + } + relInLinkFile := filepath.Join(linkInDir, "file.rel.in.link") + if err := Symlink(relInsideFile, relInLinkFile); err != nil { + t.Fatalf("Symlink: %v", err) + } + + // Copy the directory tree and verify. + forceMFTUpdateOnWindows(t, insideDir) + fsys := DirFS(insideDir) + tmpDupDir, err := MkdirTemp(tmpDir, "copyfs_dup") + if err != nil { + t.Fatalf("MkdirTemp: %v", err) + } + + // TODO(panjf2000): symlinks are currently not supported, and a specific error + // will be returned. Verify that error and skip the subsequent test, + // revisit this once #49580 is closed. + if err := CopyFS(tmpDupDir, fsys); !errors.Is(err, ErrInvalid) { + t.Fatalf("got %v, want ErrInvalid", err) + } + t.Skip("skip the subsequent test and wait for #49580") + + forceMFTUpdateOnWindows(t, tmpDupDir) + tmpFsys := DirFS(tmpDupDir) + if err := fstest.TestFS(tmpFsys, "file.in.txt", "out_symlinks/file.abs.out.link", "out_symlinks/file.rel.out.link", "in_symlinks/file.rel.in.link"); err != nil { + t.Fatal("TestFS:", err) + } + if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error { + if d.IsDir() { + return nil + } + + fi, err := d.Info() + if err != nil { + return err + } + if filepath.Ext(path) == ".link" { + if fi.Mode()&ModeSymlink == 0 { + return errors.New("original file " + path + " should be a symlink") + } + tmpfi, err := fs.Stat(tmpFsys, path) + if err != nil { + return err + } + if tmpfi.Mode()&ModeSymlink != 0 { + return errors.New("copied file " + path + " should not be a symlink") + } + } + + data, err := fs.ReadFile(fsys, path) + if err != nil { + return err + } + newData, err := fs.ReadFile(tmpFsys, path) + if err != nil { + return err + } + if !bytes.Equal(data, newData) { + return errors.New("file " + path + " contents differ") + } + + var target string + switch fileName := filepath.Base(path); fileName { + case "file.abs.out.link", "file.rel.out.link": + target = outsideFile + case "file.rel.in.link": + target = insideFile + } + if len(target) > 0 { + targetData, err := ReadFile(target) + if err != nil { + return err + } + if !bytes.Equal(targetData, newData) { + return errors.New("file " + path + " contents differ from target") + } + } + + return nil + }); err != nil { + t.Fatal("comparing two directories:", err) + } +}