-
Notifications
You must be signed in to change notification settings - Fork 597
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Migrate location-related structs to the file package (#1751)
* migrate location structs to file package Signed-off-by: Alex Goodman <[email protected]> * replace source.Location refs with file package call Signed-off-by: Alex Goodman <[email protected]> * fix linting Signed-off-by: Alex Goodman <[email protected]> * remove hardlink test for file based catalogers Signed-off-by: Alex Goodman <[email protected]> * remove hardlink test for all-regular-files testing Signed-off-by: Alex Goodman <[email protected]> * migrate file resolver implementations to separate package Signed-off-by: Alex Goodman <[email protected]> * fix linting Signed-off-by: Alex Goodman <[email protected]> * [wip] migrate resolvers to internal Signed-off-by: Alex Goodman <[email protected]> * migrate resolvers to syft/internal Signed-off-by: Alex Goodman <[email protected]> --------- Signed-off-by: Alex Goodman <[email protected]> Signed-off-by: <>
- Loading branch information
Showing
313 changed files
with
2,317 additions
and
2,192 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| package filecontent | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
|
|
||
| "github.com/anchore/syft/syft/file" | ||
| ) | ||
|
|
||
| func TestContentsCataloger(t *testing.T) { | ||
| allFiles := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"} | ||
|
|
||
| tests := []struct { | ||
| name string | ||
| globs []string | ||
| maxSize int64 | ||
| files []string | ||
| expected map[file.Coordinates]string | ||
| }{ | ||
| { | ||
| name: "multi-pattern", | ||
| globs: []string{"test-fixtures/last/*.txt", "test-fixtures/*.txt"}, | ||
| files: allFiles, | ||
| expected: map[file.Coordinates]string{ | ||
| file.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh", | ||
| file.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh", | ||
| file.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh", | ||
| }, | ||
| }, | ||
| { | ||
| name: "no-patterns", | ||
| globs: []string{}, | ||
| files: []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"}, | ||
| expected: map[file.Coordinates]string{}, | ||
| }, | ||
| { | ||
| name: "all-txt", | ||
| globs: []string{"**/*.txt"}, | ||
| files: allFiles, | ||
| expected: map[file.Coordinates]string{ | ||
| file.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh", | ||
| file.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh", | ||
| file.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh", | ||
| }, | ||
| }, | ||
| { | ||
| name: "subpath", | ||
| globs: []string{"test-fixtures/*.txt"}, | ||
| files: allFiles, | ||
| expected: map[file.Coordinates]string{ | ||
| file.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh", | ||
| file.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh", | ||
| }, | ||
| }, | ||
| { | ||
| name: "size-filter", | ||
| maxSize: 42, | ||
| globs: []string{"**/*.txt"}, | ||
| files: allFiles, | ||
| expected: map[file.Coordinates]string{ | ||
| file.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh", | ||
| file.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh", | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| for _, test := range tests { | ||
| t.Run(test.name, func(t *testing.T) { | ||
| c, err := NewCataloger(test.globs, test.maxSize) | ||
| assert.NoError(t, err) | ||
|
|
||
| resolver := file.NewMockResolverForPaths(test.files...) | ||
| actual, err := c.Catalog(resolver) | ||
| assert.NoError(t, err) | ||
| assert.Equal(t, test.expected, actual, "mismatched contents") | ||
|
|
||
| }) | ||
| } | ||
| } |
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| package filedigest | ||
|
|
||
| import ( | ||
| "crypto" | ||
| "errors" | ||
|
|
||
| "github.com/wagoodman/go-partybus" | ||
| "github.com/wagoodman/go-progress" | ||
|
|
||
| stereoscopeFile "github.com/anchore/stereoscope/pkg/file" | ||
| "github.com/anchore/syft/internal" | ||
| "github.com/anchore/syft/internal/bus" | ||
| "github.com/anchore/syft/internal/log" | ||
| "github.com/anchore/syft/syft/event" | ||
| "github.com/anchore/syft/syft/file" | ||
| internal2 "github.com/anchore/syft/syft/file/cataloger/internal" | ||
| ) | ||
|
|
||
| var ErrUndigestableFile = errors.New("undigestable file") | ||
|
|
||
| type Cataloger struct { | ||
| hashes []crypto.Hash | ||
| } | ||
|
|
||
| func NewCataloger(hashes []crypto.Hash) *Cataloger { | ||
| return &Cataloger{ | ||
| hashes: hashes, | ||
| } | ||
| } | ||
|
|
||
| func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordinates) (map[file.Coordinates][]file.Digest, error) { | ||
| results := make(map[file.Coordinates][]file.Digest) | ||
| var locations []file.Location | ||
|
|
||
| if len(coordinates) == 0 { | ||
| locations = internal2.AllRegularFiles(resolver) | ||
| } else { | ||
| for _, c := range coordinates { | ||
| locations = append(locations, file.NewLocationFromCoordinates(c)) | ||
| } | ||
| } | ||
|
|
||
| stage, prog := digestsCatalogingProgress(int64(len(locations))) | ||
| for _, location := range locations { | ||
| stage.Current = location.RealPath | ||
| result, err := i.catalogLocation(resolver, location) | ||
|
|
||
| if errors.Is(err, ErrUndigestableFile) { | ||
| continue | ||
| } | ||
|
|
||
| if internal.IsErrPathPermission(err) { | ||
| log.Debugf("file digests cataloger skipping %q: %+v", location.RealPath, err) | ||
| continue | ||
| } | ||
|
|
||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| prog.Increment() | ||
| results[location.Coordinates] = result | ||
| } | ||
| log.Debugf("file digests cataloger processed %d files", prog.Current()) | ||
| prog.SetCompleted() | ||
| return results, nil | ||
| } | ||
|
|
||
| func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Location) ([]file.Digest, error) { | ||
| meta, err := resolver.FileMetadataByLocation(location) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| // we should only attempt to report digests for files that are regular files (don't attempt to resolve links) | ||
| if meta.Type != stereoscopeFile.TypeRegular { | ||
| return nil, ErrUndigestableFile | ||
| } | ||
|
|
||
| contentReader, err := resolver.FileContentsByLocation(location) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| defer internal.CloseAndLogError(contentReader, location.VirtualPath) | ||
|
|
||
| digests, err := file.NewDigestsFromFile(contentReader, i.hashes) | ||
| if err != nil { | ||
| return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err} | ||
| } | ||
|
|
||
| return digests, nil | ||
| } | ||
|
|
||
| func digestsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) { | ||
| stage := &progress.Stage{} | ||
| prog := progress.NewManual(locations) | ||
|
|
||
| bus.Publish(partybus.Event{ | ||
| Type: event.FileDigestsCatalogerStarted, | ||
| Value: struct { | ||
| progress.Stager | ||
| progress.Progressable | ||
| }{ | ||
| Stager: progress.Stager(stage), | ||
| Progressable: prog, | ||
| }, | ||
| }) | ||
|
|
||
| return stage, prog | ||
| } |
Oops, something went wrong.