-
Notifications
You must be signed in to change notification settings - Fork 52
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #55 from nlewo/tar-graph
Build a graph of file trees before generating the tar stream
- Loading branch information
Showing
13 changed files
with
338 additions
and
153 deletions.
There are no files selected for viewing
Empty file.
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,116 @@ | ||
| package nix | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "github.com/nlewo/nix2container/types" | ||
| "os" | ||
| "path/filepath" | ||
| "reflect" | ||
| "sort" | ||
| ) | ||
|
|
||
| type fileNode struct { | ||
| // The file name on the FS | ||
| srcPath string | ||
| info *os.FileInfo | ||
| options *types.PathOptions | ||
| contents map[string]*fileNode | ||
| } | ||
|
|
||
| func initGraph() *fileNode { | ||
| root := &fileNode{ | ||
| contents: make(map[string]*fileNode), | ||
| } | ||
| return root | ||
| } | ||
|
|
||
| // addFileToGraph adds a file to the graph. A node of the graph | ||
| // represent a file. When addding a file, all parent directories of | ||
| // this file are added in the graph. | ||
| // | ||
| // The info and options are added to the node representing the file | ||
| // itself, ie. the leaf node. | ||
| // | ||
| // Note the graph describes the file tree of the tar stream, not the | ||
| // file tree read on the FS. This means transformations are done during | ||
| // the graph construction. | ||
| func addFileToGraph(root *fileNode, path string, info *os.FileInfo, options *types.PathOptions) error { | ||
| pathInTar := filePathToTarPath(path, options) | ||
| // A regex in the options could make the path becoming the | ||
| // empty string. In this case, we don't want to create | ||
| // anything in the graph. | ||
| if pathInTar == "" { | ||
| return nil | ||
| } | ||
|
|
||
| parts := splitPath(pathInTar) | ||
| current := root | ||
| for _, part := range parts { | ||
| if node, exists := current.contents[part]; exists { | ||
| current = node | ||
| } else { | ||
| current.contents[part] = &fileNode{ | ||
| contents: make(map[string]*fileNode), | ||
| } | ||
| current = current.contents[part] | ||
| } | ||
| } | ||
|
|
||
| if current.info != nil { | ||
| if (*current.info).Mode() != (*info).Mode() { | ||
| return fmt.Errorf("The file '%s' already exists in the graph with mode '%v' from '%s' while it is added again with mode '%v' by '%s'", | ||
| pathInTar, (*current.info).Mode(), current.srcPath, (*info).Mode(), path) | ||
| } | ||
| if (*current.info).Size() != (*info).Size() { | ||
| return fmt.Errorf("The file '%s' already exists in the graph with size '%d' from '%s' while it is added again with size '%d' by '%s'", | ||
| pathInTar, (*current.info).Size(), current.srcPath, (*info).Size(), path) | ||
| } | ||
| } | ||
| current.info = info | ||
|
|
||
| if current.options != nil && !reflect.DeepEqual(current.options.Perms, options.Perms) { | ||
| return fmt.Errorf("The file '%s' already exists in the tar with perms %#v but is overriden with perms %#v", | ||
| pathInTar, current.options.Perms, options.Perms) | ||
| } | ||
| current.options = options | ||
|
|
||
| current.srcPath = path | ||
| return nil | ||
| } | ||
|
|
||
| // If info is nil, dstPath is then a directory: this directory has | ||
| // been added to the graph but has not been walk by | ||
| // filepath.Walk. This for instance occurs when /nix/store/storepath1 | ||
| // is added: /nix/store is not walk by the filepath.Walk function. | ||
| type walkFunc func(srcPath, dstPath string, info *os.FileInfo, options *types.PathOptions) error | ||
|
|
||
| func walkGraph(root *fileNode, walkFn walkFunc) error { | ||
| return walkGraphFn("", root, walkFn) | ||
| } | ||
|
|
||
| func walkGraphFn(base string, root *fileNode, walkFn walkFunc) error { | ||
| keys := make([]string, len(root.contents)) | ||
| i := 0 | ||
| for k := range root.contents { | ||
| keys[i] = k | ||
| i++ | ||
| } | ||
| // Each subdirectory is sorted to avoid depending on the | ||
| // source file name order: we instead want to order file based | ||
| // on the name they have in the tar stream. | ||
| sort.Strings(keys) | ||
|
|
||
| for _, k := range keys { | ||
| dstPath := filepath.Join(base, k) | ||
| if k == "" { | ||
| dstPath = filepath.Join("/", k) | ||
| } | ||
| if err := walkFn(root.contents[k].srcPath, dstPath, root.contents[k].info, root.contents[k].options); err != nil { | ||
| return err | ||
| } | ||
| if err := walkGraphFn(dstPath, root.contents[k], walkFn); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| package nix | ||
|
|
||
| import ( | ||
| "github.com/nlewo/nix2container/types" | ||
| "github.com/stretchr/testify/assert" | ||
| "os" | ||
| "path/filepath" | ||
| "testing" | ||
| ) | ||
|
|
||
| func TestGraph(t *testing.T) { | ||
| g := initGraph() | ||
| err := addFileToGraph(g, "/nix", nil, nil) | ||
| assert.Equal(t, nil, err) | ||
| assert.Contains(t, g.contents, "") | ||
| assert.Contains(t, g.contents[""].contents, "nix") | ||
|
|
||
| g = initGraph() | ||
| err = addFileToGraph(g, "/nix/store/hash1", nil, nil) | ||
| assert.Equal(t, nil, err) | ||
| assert.Contains(t, g.contents, "") | ||
| assert.Contains(t, g.contents[""].contents, "nix") | ||
| assert.Contains(t, g.contents[""].contents["nix"].contents, "store") | ||
| assert.Contains(t, g.contents[""].contents["nix"].contents["store"].contents, "hash1") | ||
| err = addFileToGraph(g, "/nix/store/hash2", nil, nil) | ||
| assert.Equal(t, nil, err) | ||
| assert.Contains(t, g.contents, "") | ||
| assert.Contains(t, g.contents[""].contents, "nix") | ||
| assert.Contains(t, g.contents[""].contents["nix"].contents["store"].contents, "hash1") | ||
| assert.Contains(t, g.contents[""].contents["nix"].contents["store"].contents, "hash2") | ||
| } | ||
|
|
||
| func TestAddFileToGraphOverride(t *testing.T) { | ||
| g := initGraph() | ||
| err := addFileToGraph(g, "/nix/store/file1", nil, &types.PathOptions{ | ||
| Perms: []types.Perm{ | ||
| { | ||
| Regex: "*", | ||
| Uid: 1, | ||
| }, | ||
| }, | ||
| }) | ||
| assert.Equal(t, nil, err) | ||
| err = addFileToGraph(g, "/nix/store/file1", nil, &types.PathOptions{ | ||
| Perms: []types.Perm{ | ||
| { | ||
| Regex: "*", | ||
| Uid: 2, | ||
| }, | ||
| }, | ||
| }) | ||
| assert.Error(t, err) | ||
| } | ||
|
|
||
| func TestWalkGraph(t *testing.T) { | ||
| g := initGraph() | ||
| paths := make([]string, 5) | ||
| var idx int | ||
| pidx := &idx | ||
| err := addFileToGraph(g, "/nix/store/hash2", nil, nil) | ||
| assert.Equal(t, nil, err) | ||
| err = addFileToGraph(g, "/nix/store/hash1", nil, nil) | ||
| assert.Equal(t, nil, err) | ||
|
|
||
| err = walkGraph(g, func(srcPath, dstPath string, info *os.FileInfo, options *types.PathOptions) error { | ||
| paths[*pidx] = dstPath | ||
| *pidx = *pidx + 1 | ||
| return nil | ||
| }) | ||
| assert.Equal(t, nil, err) | ||
| assert.Equal(t, "/", paths[0]) | ||
| assert.Equal(t, "/nix", paths[1]) | ||
| assert.Equal(t, "/nix/store", paths[2]) | ||
| assert.Equal(t, "/nix/store/hash1", paths[3]) | ||
| assert.Equal(t, "/nix/store/hash2", paths[4]) | ||
| } | ||
|
|
||
| func TestWalkGraphOnDirectory(t *testing.T) { | ||
| graph := initGraph() | ||
| err := filepath.Walk("../data/graph-directory", | ||
| func(path string, info os.FileInfo, err error) error { | ||
| return addFileToGraph(graph, path, &info, nil) | ||
| }, | ||
| ) | ||
| assert.Equal(t, nil, err) | ||
| dstPaths := make([]string, 10) | ||
| srcPaths := make([]string, 10) | ||
| missingDirectories := make([]string, 10) | ||
| var idx int | ||
| pidx := &idx | ||
| err = walkGraph(graph, func(srcPath, dstPath string, info *os.FileInfo, options *types.PathOptions) error { | ||
| dstPaths[*pidx] = dstPath | ||
| srcPaths[*pidx] = srcPath | ||
| if info == nil { | ||
| missingDirectories[*pidx] = dstPath | ||
| } | ||
| *pidx = *pidx + 1 | ||
| return nil | ||
| }) | ||
| assert.Equal(t, nil, err) | ||
| assert.Equal(t, "..", dstPaths[0]) | ||
| assert.Equal(t, "../data", dstPaths[1]) | ||
| assert.Equal(t, "../data/graph-directory", dstPaths[2]) | ||
| assert.Equal(t, "../data/graph-directory/path1", dstPaths[3]) | ||
| assert.Equal(t, "../data/graph-directory/path1/path11", dstPaths[4]) | ||
| assert.Equal(t, "../data/graph-directory/path1/path11/file111", dstPaths[5]) | ||
| assert.Equal(t, "../data/graph-directory/path2", dstPaths[6]) | ||
| assert.Equal(t, "../data/graph-directory/path2/file21", dstPaths[7]) | ||
|
|
||
| assert.Equal(t, "", srcPaths[0]) | ||
| assert.Equal(t, "", srcPaths[1]) | ||
| assert.Equal(t, "../data/graph-directory", srcPaths[2]) | ||
| assert.Equal(t, "../data/graph-directory/path1", srcPaths[3]) | ||
| assert.Equal(t, "../data/graph-directory/path1/path11", srcPaths[4]) | ||
| assert.Equal(t, "../data/graph-directory/path1/path11/file111", srcPaths[5]) | ||
|
|
||
| assert.Equal(t, "..", missingDirectories[0]) | ||
| assert.Equal(t, "../data", missingDirectories[1]) | ||
| assert.Equal(t, "", missingDirectories[2]) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.