-
Notifications
You must be signed in to change notification settings - Fork 52
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Build a graph of file trees before generating the tar stream
Before this commit, nix2container read the file tree of store paths and write all file in the order they are walked by the os.Walk Go function. In this commit, the build process contains now two steps: 1. file trees are read and a graph representing these files is built 2. this graph is walk to write files to the tar stream This allows to easily transform the file tree (filename rewritting for instance) and also allow to easily detect duplicated files.
- Loading branch information
Showing
12 changed files
with
318 additions
and
153 deletions.
There are no files selected for viewing
Empty file.
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| package nix | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "github.com/nlewo/nix2container/types" | ||
| "os" | ||
| "path/filepath" | ||
| "reflect" | ||
| "sort" | ||
| ) | ||
|
|
||
| type fileNode struct { | ||
| // The file name on the FS | ||
| srcPath string | ||
| info *os.FileInfo | ||
| options *types.PathOptions | ||
| contents map[string]*fileNode | ||
| } | ||
|
|
||
| func initGraph() *fileNode { | ||
| root := &fileNode{ | ||
| contents: make(map[string]*fileNode), | ||
| } | ||
| return root | ||
| } | ||
|
|
||
| // addFileToGraph adds a file to the graph. A node of the graph | ||
| // represent a file. When addding a file, all parent directories of | ||
| // this file are added in the graph. | ||
| // | ||
| // The info and options are added to the node representing the file | ||
| // itself, ie. the leaf node. | ||
| // | ||
| // Note the graph describes the file tree of the tar stream, not the | ||
| // file tree read on the FS. This means transformations are done during | ||
| // the graph construction. | ||
| func addFileToGraph(root *fileNode, path string, info *os.FileInfo, options *types.PathOptions) error { | ||
| pathInTar := filePathToTarPath(path, options) | ||
| // A regex in the options could make the path becoming the | ||
| // empty string. In this case, we don't want to create | ||
| // anything in the graph. | ||
| if pathInTar == "" { | ||
| return nil | ||
| } | ||
|
|
||
| parts := splitPath(pathInTar) | ||
| current := root | ||
| for _, part := range parts { | ||
| if node, exists := current.contents[part]; exists { | ||
| current = node | ||
| } else { | ||
| current.contents[part] = &fileNode{ | ||
| contents: make(map[string]*fileNode), | ||
| } | ||
| current = current.contents[part] | ||
| } | ||
| } | ||
|
|
||
| if current.srcPath != "" && current.srcPath != path { | ||
| return fmt.Errorf("The file '%s' already exists in the tar with source path %s but is added again with the source path %s", | ||
| pathInTar, current.srcPath, path) | ||
| } | ||
| current.srcPath = path | ||
|
|
||
| if current.options != nil && !reflect.DeepEqual(current.options, options) { | ||
| return fmt.Errorf("The file '%s' already exists in the tar with options %#v but is overriden with options %#v", | ||
| pathInTar, current.options, options) | ||
| } | ||
| current.options = options | ||
|
|
||
| current.info = info | ||
| return nil | ||
| } | ||
|
|
||
| // If info is nil, dstPath is then a directory: this directory has | ||
| // been added to the graph but has not been walk by | ||
| // filepath.Walk. This for instance occurs when /nix/store/storepath1 | ||
| // is added: /nix/store is not walk by the filepath.Walk function. | ||
| type walkFunc func(srcPath, dstPath string, info *os.FileInfo, options *types.PathOptions) error | ||
|
|
||
| func walkGraph(root *fileNode, walkFn walkFunc) error { | ||
| return walkGraphFn("", root, walkFn) | ||
| } | ||
|
|
||
| func walkGraphFn(base string, root *fileNode, walkFn walkFunc) error { | ||
| keys := make([]string, len(root.contents)) | ||
| i := 0 | ||
| for k := range root.contents { | ||
| keys[i] = k | ||
| i++ | ||
| } | ||
| // Each subdirectory is sorted to avoid depending on the | ||
| // source file name order: we instead want to order file based | ||
| // on the name they have in the tar stream. | ||
| sort.Strings(keys) | ||
|
|
||
| for _, k := range keys { | ||
| dstPath := filepath.Join(base, k) | ||
| if k == "" { | ||
| dstPath = filepath.Join("/", k) | ||
| } | ||
| if err := walkFn(root.contents[k].srcPath, dstPath, root.contents[k].info, root.contents[k].options); err != nil { | ||
| return err | ||
| } | ||
| if err := walkGraphFn(dstPath, root.contents[k], walkFn); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| package nix | ||
|
|
||
| import ( | ||
| "github.com/nlewo/nix2container/types" | ||
| "github.com/stretchr/testify/assert" | ||
| "os" | ||
| "path/filepath" | ||
| "testing" | ||
| ) | ||
|
|
||
| func TestGraph(t *testing.T) { | ||
| g := initGraph() | ||
| err := addFileToGraph(g, "/nix", nil, nil) | ||
| assert.Equal(t, nil, err) | ||
| assert.Contains(t, g.contents, "") | ||
| assert.Contains(t, g.contents[""].contents, "nix") | ||
|
|
||
| g = initGraph() | ||
| err = addFileToGraph(g, "/nix/store/hash1", nil, nil) | ||
| assert.Equal(t, nil, err) | ||
| assert.Contains(t, g.contents, "") | ||
| assert.Contains(t, g.contents[""].contents, "nix") | ||
| assert.Contains(t, g.contents[""].contents["nix"].contents, "store") | ||
| assert.Contains(t, g.contents[""].contents["nix"].contents["store"].contents, "hash1") | ||
| err = addFileToGraph(g, "/nix/store/hash2", nil, nil) | ||
| assert.Equal(t, nil, err) | ||
| assert.Contains(t, g.contents, "") | ||
| assert.Contains(t, g.contents[""].contents, "nix") | ||
| assert.Contains(t, g.contents[""].contents["nix"].contents["store"].contents, "hash1") | ||
| assert.Contains(t, g.contents[""].contents["nix"].contents["store"].contents, "hash2") | ||
| } | ||
|
|
||
| func TestAddFileToGraphOverride(t *testing.T) { | ||
| g := initGraph() | ||
| err := addFileToGraph(g, "/nix/store/file1", nil, &types.PathOptions{ | ||
| Perms: []types.Perm{ | ||
| { | ||
| Regex: "*", | ||
| Uid: 1, | ||
| }, | ||
| }, | ||
| }) | ||
| assert.Equal(t, nil, err) | ||
| err = addFileToGraph(g, "/nix/store/file1", nil, &types.PathOptions{ | ||
| Perms: []types.Perm{ | ||
| { | ||
| Regex: "*", | ||
| Uid: 2, | ||
| }, | ||
| }, | ||
| }) | ||
| assert.Error(t, err) | ||
| } | ||
|
|
||
| func TestWalkGraph(t *testing.T) { | ||
| g := initGraph() | ||
| paths := make([]string, 5) | ||
| var idx int | ||
| pidx := &idx | ||
| err := addFileToGraph(g, "/nix/store/hash2", nil, nil) | ||
| assert.Equal(t, nil, err) | ||
| err = addFileToGraph(g, "/nix/store/hash1", nil, nil) | ||
| assert.Equal(t, nil, err) | ||
|
|
||
| err = walkGraph(g, func(srcPath, dstPath string, info *os.FileInfo, options *types.PathOptions) error { | ||
| paths[*pidx] = dstPath | ||
| *pidx = *pidx + 1 | ||
| return nil | ||
| }) | ||
| assert.Equal(t, nil, err) | ||
| assert.Equal(t, "/", paths[0]) | ||
| assert.Equal(t, "/nix", paths[1]) | ||
| assert.Equal(t, "/nix/store", paths[2]) | ||
| assert.Equal(t, "/nix/store/hash1", paths[3]) | ||
| assert.Equal(t, "/nix/store/hash2", paths[4]) | ||
| } | ||
|
|
||
| func TestWalkGraphOnDirectory(t *testing.T) { | ||
| graph := initGraph() | ||
| err := filepath.Walk("../data/graph-directory", | ||
| func(path string, info os.FileInfo, err error) error { | ||
| return addFileToGraph(graph, path, &info, nil) | ||
| }, | ||
| ) | ||
| assert.Equal(t, nil, err) | ||
| dstPaths := make([]string, 10) | ||
| srcPaths := make([]string, 10) | ||
| missingDirectories := make([]string, 10) | ||
| var idx int | ||
| pidx := &idx | ||
| err = walkGraph(graph, func(srcPath, dstPath string, info *os.FileInfo, options *types.PathOptions) error { | ||
| dstPaths[*pidx] = dstPath | ||
| srcPaths[*pidx] = srcPath | ||
| if info == nil { | ||
| missingDirectories[*pidx] = dstPath | ||
| } | ||
| *pidx = *pidx + 1 | ||
| return nil | ||
| }) | ||
| assert.Equal(t, nil, err) | ||
| assert.Equal(t, "..", dstPaths[0]) | ||
| assert.Equal(t, "../data", dstPaths[1]) | ||
| assert.Equal(t, "../data/graph-directory", dstPaths[2]) | ||
| assert.Equal(t, "../data/graph-directory/path1", dstPaths[3]) | ||
| assert.Equal(t, "../data/graph-directory/path1/path11", dstPaths[4]) | ||
| assert.Equal(t, "../data/graph-directory/path1/path11/file111", dstPaths[5]) | ||
| assert.Equal(t, "../data/graph-directory/path2", dstPaths[6]) | ||
| assert.Equal(t, "../data/graph-directory/path2/file21", dstPaths[7]) | ||
|
|
||
| assert.Equal(t, "", srcPaths[0]) | ||
| assert.Equal(t, "", srcPaths[1]) | ||
| assert.Equal(t, "../data/graph-directory", srcPaths[2]) | ||
| assert.Equal(t, "../data/graph-directory/path1", srcPaths[3]) | ||
| assert.Equal(t, "../data/graph-directory/path1/path11", srcPaths[4]) | ||
| assert.Equal(t, "../data/graph-directory/path1/path11/file111", srcPaths[5]) | ||
|
|
||
| assert.Equal(t, "..", missingDirectories[0]) | ||
| assert.Equal(t, "../data", missingDirectories[1]) | ||
| assert.Equal(t, "", missingDirectories[2]) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.