From 248d8ec491c5b79410b151337c60219d02b87e9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sat, 16 Oct 2021 16:24:49 +0200 Subject: [PATCH] hugofs: Add includeFiles and excludeFiles to mount configuration Fixes #9042 --- hugofs/fileinfo.go | 12 ++++++ hugofs/glob/glob.go | 11 +++++- hugofs/rootmapping_fs.go | 26 +++++++++++-- hugofs/rootmapping_fs_test.go | 69 +++++++++++++++++++++++++++++++++++ hugolib/filesystems/basefs.go | 19 ++++++++-- modules/config.go | 5 +++ 6 files changed, 135 insertions(+), 7 deletions(-) diff --git a/hugofs/fileinfo.go b/hugofs/fileinfo.go index fcf35d9564f..ea6ac4fd37a 100644 --- a/hugofs/fileinfo.go +++ b/hugofs/fileinfo.go @@ -23,6 +23,8 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/hugofs/files" "golang.org/x/text/unicode/norm" @@ -76,6 +78,9 @@ type FileMeta struct { Fs afero.Fs OpenFunc func() (afero.File, error) JoinStatFunc func(name string) (FileMetaInfo, error) + + // Include only files or directories that match. + InclusionFilter *glob.FilenameFilter } func (m *FileMeta) Copy() *FileMeta { @@ -95,10 +100,17 @@ func (m *FileMeta) Merge(from *FileMeta) { for i := 0; i < dstv.NumField(); i++ { v := dstv.Field(i) + if !v.CanSet() { + continue + } if !hreflect.IsTruthfulValue(v) { v.Set(srcv.Field(i)) } } + + if m.InclusionFilter == nil { + m.InclusionFilter = from.InclusionFilter + } } func (f *FileMeta) Open() (afero.File, error) { diff --git a/hugofs/glob/glob.go b/hugofs/glob/glob.go index 6dd0df5ed4a..3ba4d5c18ef 100644 --- a/hugofs/glob/glob.go +++ b/hugofs/glob/glob.go @@ -170,7 +170,7 @@ type FilenameFilter struct { } // NewFilenameFilter creates a new Glob where the Match method will -// return true if the file should be exluded. +// return true if the file should be included. // Note that the inclusions will be checked first. func NewFilenameFilter(inclusions, exclusions []string) (*FilenameFilter, error) { filter := &FilenameFilter{isWindows: isWindows} @@ -193,6 +193,15 @@ func NewFilenameFilter(inclusions, exclusions []string) (*FilenameFilter, error) return filter, nil } +// MustNewFilenameFilter invokes NewFilenameFilter and panics on error. +func MustNewFilenameFilter(inclusions, exclusions []string) *FilenameFilter { + filter, err := NewFilenameFilter(inclusions, exclusions) + if err != nil { + panic(err) + } + return filter +} + // NewFilenameFilterForInclusionFunc create a new filter using the provided inclusion func. func NewFilenameFilterForInclusionFunc(shouldInclude func(filename string) bool) *FilenameFilter { return &FilenameFilter{shouldInclude: shouldInclude, isWindows: isWindows} diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index 6441693adab..299e853814c 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -367,6 +367,9 @@ func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error) for _, fi := range direntries { meta := fi.(FileMetaInfo).Meta() + if !rm.Meta.InclusionFilter.Match(meta.Filename) { + continue + } meta.Merge(rm.Meta) if fi.IsDir() { name := fi.Name() @@ -508,7 +511,13 @@ func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) { } fileCount := 0 + var wasFiltered bool for _, root := range roots { + // TODO1 relevant? + if !root.Meta.InclusionFilter.Match(name) { + wasFiltered = true + continue + } if !root.fi.IsDir() { fileCount++ } @@ -518,6 +527,9 @@ func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) { } if fileCount == 0 { + if wasFiltered { + return nil, os.ErrNotExist + } // Dir only. return []FileMetaInfo{newDirNameOnlyFileInfo(name, roots[0].Meta, fs.virtualDirOpener(name))}, nil } @@ -531,6 +543,9 @@ func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) { } func (fs *RootMappingFs) statRoot(root RootMapping, name string) (FileMetaInfo, bool, error) { + if !root.Meta.InclusionFilter.Match(name) { + return nil, false, os.ErrNotExist + } filename := root.filename(name) fi, b, err := lstatIfPossible(fs.Fs, filename) @@ -586,16 +601,21 @@ func (f *rootMappingFile) Name() string { func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) { if f.File != nil { + fis, err := f.File.Readdir(count) if err != nil { return nil, err } - for i, fi := range fis { - fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta) + var result []os.FileInfo + for _, fi := range fis { + if f.meta.InclusionFilter.Match(fi.(FileMetaInfo).Meta().Filename) { + result = append(result, decorateFileInfo(fi, f.fs, nil, "", "", f.meta)) + } } - return fis, nil + return result, nil } + return f.fs.collectDirEntries(f.name) } diff --git a/hugofs/rootmapping_fs_test.go b/hugofs/rootmapping_fs_test.go index e83a46a873d..c650e8f110d 100644 --- a/hugofs/rootmapping_fs_test.go +++ b/hugofs/rootmapping_fs_test.go @@ -20,6 +20,8 @@ import ( "sort" "testing" + "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/config" qt "github.com/frankban/quicktest" @@ -483,3 +485,70 @@ func TestRootMappingFsOsBase(t *testing.T) { c.Assert(getDirnames("static/a/b/c"), qt.DeepEquals, []string{"d4", "f-1.txt", "f-2.txt", "f-3.txt", "ms-1.txt"}) } + +func TestRootMappingFileFilter(t *testing.T) { + c := qt.New(t) + fs := NewBaseFileDecorator(afero.NewMemMapFs()) + + for _, lang := range []string{"no", "en", "fr"} { + for i := 1; i <= 3; i++ { + c.Assert(afero.WriteFile(fs, filepath.Join(lang, fmt.Sprintf("my%s%d.txt", lang, i)), []byte("some text file for"+lang), 0755), qt.IsNil) + } + } + + for _, lang := range []string{"no", "en", "fr"} { + for i := 1; i <= 3; i++ { + c.Assert(afero.WriteFile(fs, filepath.Join(lang, "sub", fmt.Sprintf("mysub%s%d.txt", lang, i)), []byte("some text file for"+lang), 0755), qt.IsNil) + } + } + + rm := []RootMapping{ + { + From: "content", + To: "no", + Meta: &FileMeta{Lang: "no", InclusionFilter: glob.MustNewFilenameFilter(nil, []string{"**.txt"})}, + }, + { + From: "content", + To: "en", + Meta: &FileMeta{Lang: "en"}, + }, + { + From: "content", + To: "fr", + Meta: &FileMeta{Lang: "fr", InclusionFilter: glob.MustNewFilenameFilter(nil, []string{"**.txt"})}, + }, + } + + rfs, err := NewRootMappingFs(fs, rm...) + c.Assert(err, qt.IsNil) + + assertExists := func(filename string, shouldExist bool) { + c.Helper() + filename = filepath.Clean(filename) + _, err1 := rfs.Stat(filename) + f, err2 := rfs.Open(filename) + if shouldExist { + c.Assert(err1, qt.IsNil) + c.Assert(err2, qt.IsNil) + c.Assert(f.Close(), qt.IsNil) + } else { + c.Assert(err1, qt.Not(qt.IsNil)) + c.Assert(err2, qt.Not(qt.IsNil)) + } + } + + assertExists("content/myno1.txt", false) + assertExists("content/myen1.txt", true) + assertExists("content/myfr1.txt", false) + + dirEntriesSub, err := afero.ReadDir(rfs, filepath.Join("content", "sub")) + c.Assert(err, qt.IsNil) + c.Assert(len(dirEntriesSub), qt.Equals, 3) + + dirEntries, err := afero.ReadDir(rfs, "content") + + c.Assert(err, qt.IsNil) + c.Assert(len(dirEntries), qt.Equals, 4) + +} diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index dcfee34ffed..54e3a663267 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -24,6 +24,10 @@ import ( "strings" "sync" + "github.com/gohugoio/hugo/hugofs/glob" + + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/hugofs/files" @@ -597,6 +601,14 @@ func (b *sourceFilesystemsBuilder) createModFs( mountWeight++ } + inclusionFilter, err := glob.NewFilenameFilter( + types.ToStringSlicePreserveString(mount.IncludeFiles), + types.ToStringSlicePreserveString(mount.ExcludeFiles), + ) + if err != nil { + return err + } + base, filename := absPathify(mount.Source) rm := hugofs.RootMapping{ @@ -605,9 +617,10 @@ func (b *sourceFilesystemsBuilder) createModFs( ToBasedir: base, Module: md.Module.Path(), Meta: &hugofs.FileMeta{ - Watch: md.Watch(), - Weight: mountWeight, - Classifier: files.ContentClassContent, + Watch: md.Watch(), + Weight: mountWeight, + Classifier: files.ContentClassContent, + InclusionFilter: inclusionFilter, }, } diff --git a/modules/config.go b/modules/config.go index 45a2f22ee1f..f80a456cfc9 100644 --- a/modules/config.go +++ b/modules/config.go @@ -379,6 +379,11 @@ type Mount struct { Lang string // any language code associated with this mount. + // Include only files matching the given Glob patterns (string or slice). + IncludeFiles interface{} + + // Exclude all files matching the given Glob patterns (string or slice). + ExcludeFiles interface{} } func (m Mount) Component() string {