commit 471ed91c60cd36645794925cb4892cc820eae626
parent 94a5bac5b29bbba1ca4809752fe3fd04a58547b6
Author: Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Date: Sat, 16 Oct 2021 16:24:49 +0200
hugofs: Add includeFiles and excludeFiles to mount configuration
Fixes #9042
Diffstat:
15 files changed, 795 insertions(+), 131 deletions(-)
diff --git a/create/content.go b/create/content.go
@@ -137,8 +137,9 @@ func (b *contentBuilder) buildDir() error {
if !b.dirMap.siteUsed {
// We don't need to build everything.
contentInclusionFilter = glob.NewFilenameFilterForInclusionFunc(func(filename string) bool {
+ filename = strings.TrimPrefix(filename, string(os.PathSeparator))
for _, cn := range contentTargetFilenames {
- if strings.HasPrefix(cn, filename) {
+ if strings.Contains(cn, filename) {
return true
}
}
@@ -205,7 +206,8 @@ func (b *contentBuilder) buildFile() error {
if !usesSite {
// We don't need to build everything.
contentInclusionFilter = glob.NewFilenameFilterForInclusionFunc(func(filename string) bool {
- return strings.HasPrefix(contentPlaceholderAbsFilename, filename)
+ filename = strings.TrimPrefix(filename, string(os.PathSeparator))
+ return strings.Contains(contentPlaceholderAbsFilename, filename)
})
}
diff --git a/docs/content/en/hugo-modules/configuration.md b/docs/content/en/hugo-modules/configuration.md
@@ -155,3 +155,15 @@ target
lang
: The language code, e.g. "en". Only relevant for `content` mounts, and `static` mounts when in multihost mode.
+includeFiles (string or slice)
+: One or more [glob](https://github.com/gobwas/glob) patterns matching files or directories to include. If `excludeFiles` is not set, the files matching `includeFiles` will be the files mounted.
+
+The glob patterns are matched to the filenames starting from the `source` root, they should have Unix styled slashes even on Windows, `/` matches the mount root and `**` can be used as a super-asterisk to match recursively down all directories, e.g `/posts/**.jpg`.
+
+The search is case-insensitive.
+
+{{< new-in "0.89.0" >}}
+
+excludeFiles (string or slice)
+: One or more glob patterns matching files to exclude.
+
diff --git a/hugofs/fileinfo.go b/hugofs/fileinfo.go
@@ -23,6 +23,8 @@ import (
"strings"
"time"
+ "github.com/gohugoio/hugo/hugofs/glob"
+
"github.com/gohugoio/hugo/hugofs/files"
"golang.org/x/text/unicode/norm"
@@ -76,6 +78,9 @@ type FileMeta struct {
Fs afero.Fs
OpenFunc func() (afero.File, error)
JoinStatFunc func(name string) (FileMetaInfo, error)
+
+ // Include only files or directories that match.
+ InclusionFilter *glob.FilenameFilter
}
func (m *FileMeta) Copy() *FileMeta {
@@ -95,10 +100,17 @@ func (m *FileMeta) Merge(from *FileMeta) {
for i := 0; i < dstv.NumField(); i++ {
v := dstv.Field(i)
+ if !v.CanSet() {
+ continue
+ }
if !hreflect.IsTruthfulValue(v) {
v.Set(srcv.Field(i))
}
}
+
+ if m.InclusionFilter == nil {
+ m.InclusionFilter = from.InclusionFilter
+ }
}
func (f *FileMeta) Open() (afero.File, error) {
diff --git a/hugofs/filename_filter_fs.go b/hugofs/filename_filter_fs.go
@@ -0,0 +1,170 @@
+// Copyright 2021 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+ "os"
+ "strings"
+ "syscall"
+ "time"
+
+ "github.com/gohugoio/hugo/hugofs/glob"
+ "github.com/spf13/afero"
+)
+
+func newFilenameFilterFs(fs afero.Fs, base string, filter *glob.FilenameFilter) afero.Fs {
+ return &filenameFilterFs{
+ fs: fs,
+ base: base,
+ filter: filter,
+ }
+}
+
+// filenameFilterFs is a filesystem that filters by filename.
+type filenameFilterFs struct {
+ base string
+ fs afero.Fs
+
+ filter *glob.FilenameFilter
+}
+
+func (fs *filenameFilterFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
+ fi, b, err := fs.fs.(afero.Lstater).LstatIfPossible(name)
+ if err != nil {
+ return nil, false, err
+ }
+ if !fs.filter.Match(name, fi.IsDir()) {
+ return nil, false, os.ErrNotExist
+ }
+ return fi, b, nil
+}
+
+func (fs *filenameFilterFs) Open(name string) (afero.File, error) {
+ fi, err := fs.fs.Stat(name)
+ if err != nil {
+ return nil, err
+ }
+
+ if !fs.filter.Match(name, fi.IsDir()) {
+ return nil, os.ErrNotExist
+ }
+
+ f, err := fs.fs.Open(name)
+ if err != nil {
+ return nil, err
+ }
+
+ if !fi.IsDir() {
+ return f, nil
+ }
+
+ return &filenameFilterDir{
+ File: f,
+ base: fs.base,
+ filter: fs.filter,
+ }, nil
+}
+
+func (fs *filenameFilterFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
+ return fs.Open(name)
+}
+
+func (fs *filenameFilterFs) Stat(name string) (os.FileInfo, error) {
+ fi, _, err := fs.LstatIfPossible(name)
+ return fi, err
+}
+
+func (fs *filenameFilterFs) getOpener(name string) func() (afero.File, error) {
+ return func() (afero.File, error) {
+ return fs.Open(name)
+ }
+}
+
+type filenameFilterDir struct {
+ afero.File
+ base string
+ filter *glob.FilenameFilter
+}
+
+func (f *filenameFilterDir) Readdir(count int) ([]os.FileInfo, error) {
+ fis, err := f.File.Readdir(-1)
+ if err != nil {
+ return nil, err
+ }
+
+ var result []os.FileInfo
+ for _, fi := range fis {
+ fim := fi.(FileMetaInfo)
+ if f.filter.Match(strings.TrimPrefix(fim.Meta().Filename, f.base), fim.IsDir()) {
+ result = append(result, fi)
+ }
+ }
+
+ return result, nil
+}
+
+func (f *filenameFilterDir) Readdirnames(count int) ([]string, error) {
+ dirsi, err := f.Readdir(count)
+ if err != nil {
+ return nil, err
+ }
+
+ dirs := make([]string, len(dirsi))
+ for i, d := range dirsi {
+ dirs[i] = d.Name()
+ }
+ return dirs, nil
+}
+
+func (fs *filenameFilterFs) Chmod(n string, m os.FileMode) error {
+ return syscall.EPERM
+}
+
+func (fs *filenameFilterFs) Chtimes(n string, a, m time.Time) error {
+ return syscall.EPERM
+}
+
+func (fs *filenameFilterFs) Chown(n string, uid, gid int) error {
+ return syscall.EPERM
+}
+
+func (fs *filenameFilterFs) ReadDir(name string) ([]os.FileInfo, error) {
+ panic("not implemented")
+}
+
+func (fs *filenameFilterFs) Remove(n string) error {
+ return syscall.EPERM
+}
+
+func (fs *filenameFilterFs) RemoveAll(p string) error {
+ return syscall.EPERM
+}
+
+func (fs *filenameFilterFs) Rename(o, n string) error {
+ return syscall.EPERM
+}
+func (fs *filenameFilterFs) Create(n string) (afero.File, error) {
+ return nil, syscall.EPERM
+}
+func (fs *filenameFilterFs) Name() string {
+ return "FinameFilterFS"
+}
+
+func (fs *filenameFilterFs) Mkdir(n string, p os.FileMode) error {
+ return syscall.EPERM
+}
+
+func (fs *filenameFilterFs) MkdirAll(n string, p os.FileMode) error {
+ return syscall.EPERM
+}
diff --git a/hugofs/filename_filter_fs_test.go b/hugofs/filename_filter_fs_test.go
@@ -0,0 +1,83 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/gohugoio/hugo/hugofs/glob"
+
+ "github.com/spf13/afero"
+
+ qt "github.com/frankban/quicktest"
+)
+
+func TestFilenameFilterFs(t *testing.T) {
+ c := qt.New(t)
+
+ base := filepath.FromSlash("/mybase")
+
+ fs := NewBaseFileDecorator(afero.NewMemMapFs())
+
+ for _, letter := range []string{"a", "b", "c"} {
+ for i := 1; i <= 3; i++ {
+ c.Assert(afero.WriteFile(fs, filepath.Join(base, letter, fmt.Sprintf("my%d.txt", i)), []byte("some text file for"+letter), 0755), qt.IsNil)
+ c.Assert(afero.WriteFile(fs, filepath.Join(base, letter, fmt.Sprintf("my%d.json", i)), []byte("some json file for"+letter), 0755), qt.IsNil)
+ }
+ }
+
+ fs = afero.NewBasePathFs(fs, base)
+
+ filter, err := glob.NewFilenameFilter(nil, []string{"/b/**.txt"})
+ c.Assert(err, qt.IsNil)
+
+ fs = newFilenameFilterFs(fs, base, filter)
+
+ assertExists := func(filename string, shouldExist bool) {
+ filename = filepath.Clean(filename)
+ _, err1 := fs.Stat(filename)
+ f, err2 := fs.Open(filename)
+ if shouldExist {
+ c.Assert(err1, qt.IsNil)
+ c.Assert(err2, qt.IsNil)
+ defer f.Close()
+
+ } else {
+ for _, err := range []error{err1, err2} {
+ c.Assert(err, qt.Not(qt.IsNil))
+ c.Assert(errors.Is(err, os.ErrNotExist), qt.IsTrue)
+ }
+ }
+ }
+
+ assertExists("/a/my1.txt", true)
+ assertExists("/b/my1.txt", false)
+
+ dirB, err := fs.Open("/b")
+ defer dirB.Close()
+ c.Assert(err, qt.IsNil)
+ dirBEntries, err := dirB.Readdirnames(-1)
+ c.Assert(dirBEntries, qt.DeepEquals, []string{"my1.json", "my2.json", "my3.json"})
+
+ dirC, err := fs.Open("/c")
+ defer dirC.Close()
+ c.Assert(err, qt.IsNil)
+ dirCEntries, err := dirC.Readdirnames(-1)
+ c.Assert(dirCEntries, qt.DeepEquals, []string{"my1.json", "my1.txt", "my2.json", "my2.txt", "my3.json", "my3.txt"})
+
+}
diff --git a/hugofs/glob/filename_filter.go b/hugofs/glob/filename_filter.go
@@ -0,0 +1,159 @@
+// Copyright 2021 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package glob
+
+import (
+ "path"
+ "path/filepath"
+ "strings"
+
+ "github.com/gobwas/glob"
+)
+
+type FilenameFilter struct {
+ shouldInclude func(filename string) bool
+ inclusions []glob.Glob
+ dirInclusions []glob.Glob
+ exclusions []glob.Glob
+ isWindows bool
+}
+
+func normalizeFilenameGlobPattern(s string) string {
+ // Use Unix separators even on Windows.
+ s = filepath.ToSlash(s)
+ if !strings.HasPrefix(s, "/") {
+ s = "/" + s
+ }
+ return s
+}
+
+// NewFilenameFilter creates a new Glob where the Match method will
+// return true if the file should be included.
+// Note that the inclusions will be checked first.
+func NewFilenameFilter(inclusions, exclusions []string) (*FilenameFilter, error) {
+ if inclusions == nil && exclusions == nil {
+ return nil, nil
+ }
+ filter := &FilenameFilter{isWindows: isWindows}
+
+ for _, include := range inclusions {
+ include = normalizeFilenameGlobPattern(include)
+ g, err := filenamesGlobCache.GetGlob(include)
+ if err != nil {
+ return nil, err
+ }
+ filter.inclusions = append(filter.inclusions, g)
+
+ // For mounts that do directory walking (e.g. content) we
+ // must make sure that all directories up to this inclusion also
+ // gets included.
+ dir := path.Dir(include)
+ parts := strings.Split(dir, "/")
+ for i, _ := range parts {
+ pattern := "/" + filepath.Join(parts[:i+1]...)
+ g, err := filenamesGlobCache.GetGlob(pattern)
+ if err != nil {
+ return nil, err
+ }
+ filter.dirInclusions = append(filter.dirInclusions, g)
+ }
+ }
+
+ for _, exclude := range exclusions {
+ exclude = normalizeFilenameGlobPattern(exclude)
+ g, err := filenamesGlobCache.GetGlob(exclude)
+ if err != nil {
+ return nil, err
+ }
+ filter.exclusions = append(filter.exclusions, g)
+ }
+
+ return filter, nil
+}
+
+// MustNewFilenameFilter invokes NewFilenameFilter and panics on error.
+func MustNewFilenameFilter(inclusions, exclusions []string) *FilenameFilter {
+ filter, err := NewFilenameFilter(inclusions, exclusions)
+ if err != nil {
+ panic(err)
+ }
+ return filter
+}
+
+// NewFilenameFilterForInclusionFunc create a new filter using the provided inclusion func.
+func NewFilenameFilterForInclusionFunc(shouldInclude func(filename string) bool) *FilenameFilter {
+ return &FilenameFilter{shouldInclude: shouldInclude, isWindows: isWindows}
+}
+
+// Match returns whether filename should be included.
+func (f *FilenameFilter) Match(filename string, isDir bool) bool {
+ if f == nil {
+ return true
+ }
+ return f.doMatch(filename, isDir)
+ /*if f.shouldInclude == nil {
+ fmt.Printf("Match: %q (%t) => %t\n", filename, isDir, isMatch)
+ }
+ return isMatch*/
+}
+
+func (f *FilenameFilter) doMatch(filename string, isDir bool) bool {
+ if f == nil {
+ return true
+ }
+
+ if !strings.HasPrefix(filename, filepathSeparator) {
+ filename = filepathSeparator + filename
+ }
+
+ if f.shouldInclude != nil {
+ if f.shouldInclude(filename) {
+ return true
+ }
+ if f.isWindows {
+ // The Glob matchers below handles this by themselves,
+ // for the shouldInclude we need to take some extra steps
+ // to make this robust.
+ winFilename := filepath.FromSlash(filename)
+ if filename != winFilename {
+ if f.shouldInclude(winFilename) {
+ return true
+ }
+ }
+ }
+
+ }
+
+ for _, inclusion := range f.inclusions {
+ if inclusion.Match(filename) {
+ return true
+ }
+ }
+
+ if isDir && f.inclusions != nil {
+ for _, inclusion := range f.dirInclusions {
+ if inclusion.Match(filename) {
+ return true
+ }
+ }
+ }
+
+ for _, exclusion := range f.exclusions {
+ if exclusion.Match(filename) {
+ return false
+ }
+ }
+
+ return f.inclusions == nil && f.shouldInclude == nil
+}
diff --git a/hugofs/glob/filename_filter_test.go b/hugofs/glob/filename_filter_test.go
@@ -0,0 +1,70 @@
+// Copyright 2021 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package glob
+
+import (
+ "path/filepath"
+ "strings"
+ "testing"
+
+ qt "github.com/frankban/quicktest"
+)
+
+func TestFilenameFilter(t *testing.T) {
+ c := qt.New(t)
+
+ excludeAlmostAllJSON, err := NewFilenameFilter([]string{"/a/b/c/foo.json"}, []string{"**.json"})
+ c.Assert(err, qt.IsNil)
+ c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/data/my.json"), false), qt.Equals, false)
+ c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c/foo.json"), false), qt.Equals, true)
+ c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c/foo.bar"), false), qt.Equals, false)
+ c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c"), true), qt.Equals, true)
+ c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b"), true), qt.Equals, true)
+ c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a"), true), qt.Equals, true)
+ c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/"), true), qt.Equals, true)
+ c.Assert(excludeAlmostAllJSON.Match("", true), qt.Equals, true)
+
+ excludeAllButFooJSON, err := NewFilenameFilter([]string{"/a/**/foo.json"}, []string{"**.json"})
+ c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/data/my.json"), false), qt.Equals, false)
+ c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/c/d/e/foo.json"), false), qt.Equals, true)
+ c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/c"), true), qt.Equals, true)
+ c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/"), true), qt.Equals, true)
+ c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/"), true), qt.Equals, true)
+ c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/b"), true), qt.Equals, false)
+ c.Assert(err, qt.IsNil)
+
+ nopFilter, err := NewFilenameFilter(nil, nil)
+ c.Assert(err, qt.IsNil)
+ c.Assert(nopFilter.Match("ab.txt", false), qt.Equals, true)
+
+ includeOnlyFilter, err := NewFilenameFilter([]string{"**.json", "**.jpg"}, nil)
+ c.Assert(err, qt.IsNil)
+ c.Assert(includeOnlyFilter.Match("ab.json", false), qt.Equals, true)
+ c.Assert(includeOnlyFilter.Match("ab.jpg", false), qt.Equals, true)
+ c.Assert(includeOnlyFilter.Match("ab.gif", false), qt.Equals, false)
+
+ exlcudeOnlyFilter, err := NewFilenameFilter(nil, []string{"**.json", "**.jpg"})
+ c.Assert(err, qt.IsNil)
+ c.Assert(exlcudeOnlyFilter.Match("ab.json", false), qt.Equals, false)
+ c.Assert(exlcudeOnlyFilter.Match("ab.jpg", false), qt.Equals, false)
+ c.Assert(exlcudeOnlyFilter.Match("ab.gif", false), qt.Equals, true)
+
+ var nilFilter *FilenameFilter
+ c.Assert(nilFilter.Match("ab.gif", false), qt.Equals, true)
+
+ funcFilter := NewFilenameFilterForInclusionFunc(func(s string) bool { return strings.HasSuffix(s, ".json") })
+ c.Assert(funcFilter.Match("ab.json", false), qt.Equals, true)
+ c.Assert(funcFilter.Match("ab.bson", false), qt.Equals, false)
+
+}
diff --git a/hugofs/glob/glob.go b/hugofs/glob/glob.go
@@ -1,4 +1,4 @@
-// Copyright 2019 The Hugo Authors. All rights reserved.
+// Copyright 2021 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
package glob
import (
+ "os"
"path"
"path/filepath"
"runtime"
@@ -24,6 +25,8 @@ import (
"github.com/gobwas/glob/syntax"
)
+const filepathSeparator = string(os.PathSeparator)
+
var (
isWindows = runtime.GOOS == "windows"
defaultGlobCache = &globCache{
@@ -33,7 +36,7 @@ var (
}
filenamesGlobCache = &globCache{
- isCaseSensitive: true, // TODO(bep) bench
+ isCaseSensitive: false, // As long as the search strings are all lower case, this does not allocate.
isWindows: isWindows,
cache: make(map[string]globErr),
}
@@ -161,78 +164,3 @@ func HasGlobChar(s string) bool {
}
return false
}
-
-type FilenameFilter struct {
- shouldInclude func(filename string) bool
- inclusions []glob.Glob
- exclusions []glob.Glob
- isWindows bool
-}
-
-// NewFilenameFilter creates a new Glob where the Match method will
-// return true if the file should be exluded.
-// Note that the inclusions will be checked first.
-func NewFilenameFilter(inclusions, exclusions []string) (*FilenameFilter, error) {
- filter := &FilenameFilter{isWindows: isWindows}
-
- for _, include := range inclusions {
- g, err := filenamesGlobCache.GetGlob(filepath.FromSlash(include))
- if err != nil {
- return nil, err
- }
- filter.inclusions = append(filter.inclusions, g)
- }
- for _, exclude := range exclusions {
- g, err := filenamesGlobCache.GetGlob(filepath.FromSlash(exclude))
- if err != nil {
- return nil, err
- }
- filter.exclusions = append(filter.exclusions, g)
- }
-
- return filter, nil
-}
-
-// NewFilenameFilterForInclusionFunc create a new filter using the provided inclusion func.
-func NewFilenameFilterForInclusionFunc(shouldInclude func(filename string) bool) *FilenameFilter {
- return &FilenameFilter{shouldInclude: shouldInclude, isWindows: isWindows}
-}
-
-// Match returns whether filename should be included.
-func (f *FilenameFilter) Match(filename string) bool {
- if f == nil {
- return true
- }
-
- if f.shouldInclude != nil {
- if f.shouldInclude(filename) {
- return true
- }
- if f.isWindows {
- // The Glob matchers below handles this by themselves,
- // for the shouldInclude we need to take some extra steps
- // to make this robust.
- winFilename := filepath.FromSlash(filename)
- if filename != winFilename {
- if f.shouldInclude(winFilename) {
- return true
- }
- }
- }
-
- }
-
- for _, inclusion := range f.inclusions {
- if inclusion.Match(filename) {
- return true
- }
- }
-
- for _, exclusion := range f.exclusions {
- if exclusion.Match(filename) {
- return false
- }
- }
-
- return f.inclusions == nil && f.shouldInclude == nil
-}
diff --git a/hugofs/glob/glob_test.go b/hugofs/glob/glob_test.go
@@ -1,4 +1,4 @@
-// Copyright 2019 The Hugo Authors. All rights reserved.
+// Copyright 2021 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -15,7 +15,6 @@ package glob
import (
"path/filepath"
- "strings"
"testing"
qt "github.com/frankban/quicktest"
@@ -67,51 +66,38 @@ func TestNormalizePath(t *testing.T) {
}
func TestGetGlob(t *testing.T) {
- c := qt.New(t)
- g, err := GetGlob("**.JSON")
- c.Assert(err, qt.IsNil)
- c.Assert(g.Match("data/my.json"), qt.Equals, true)
+ for _, cache := range []*globCache{defaultGlobCache, filenamesGlobCache} {
+ c := qt.New(t)
+ g, err := cache.GetGlob("**.JSON")
+ c.Assert(err, qt.IsNil)
+ c.Assert(g.Match("data/my.jSon"), qt.Equals, true)
+ }
}
-func TestFilenameFilter(t *testing.T) {
- c := qt.New(t)
-
- excludeAlmostAllJSON, err := NewFilenameFilter([]string{"a/b/c/foo.json"}, []string{"**.json"})
- c.Assert(err, qt.IsNil)
- c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("data/my.json")), qt.Equals, false)
- c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("a/b/c/foo.json")), qt.Equals, true)
- c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("a/b/c/foo.bar")), qt.Equals, false)
-
- nopFilter, err := NewFilenameFilter(nil, nil)
- c.Assert(err, qt.IsNil)
- c.Assert(nopFilter.Match("ab.txt"), qt.Equals, true)
-
- includeOnlyFilter, err := NewFilenameFilter([]string{"**.json", "**.jpg"}, nil)
- c.Assert(err, qt.IsNil)
- c.Assert(includeOnlyFilter.Match("ab.json"), qt.Equals, true)
- c.Assert(includeOnlyFilter.Match("ab.jpg"), qt.Equals, true)
- c.Assert(includeOnlyFilter.Match("ab.gif"), qt.Equals, false)
-
- exlcudeOnlyFilter, err := NewFilenameFilter(nil, []string{"**.json", "**.jpg"})
- c.Assert(err, qt.IsNil)
- c.Assert(exlcudeOnlyFilter.Match("ab.json"), qt.Equals, false)
- c.Assert(exlcudeOnlyFilter.Match("ab.jpg"), qt.Equals, false)
- c.Assert(exlcudeOnlyFilter.Match("ab.gif"), qt.Equals, true)
-
- var nilFilter *FilenameFilter
- c.Assert(nilFilter.Match("ab.gif"), qt.Equals, true)
+func BenchmarkGetGlob(b *testing.B) {
- funcFilter := NewFilenameFilterForInclusionFunc(func(s string) bool { return strings.HasSuffix(s, ".json") })
- c.Assert(funcFilter.Match("ab.json"), qt.Equals, true)
- c.Assert(funcFilter.Match("ab.bson"), qt.Equals, false)
+ runBench := func(name string, cache *globCache, search string) {
+ b.Run(name, func(b *testing.B) {
+ g, err := GetGlob("**/foo")
+ if err != nil {
+ b.Fatal(err)
+ }
+ for i := 0; i < b.N; i++ {
+ _ = g.Match(search)
+ }
+ })
+ }
-}
+ runBench("Default cache", defaultGlobCache, "abcde")
+ runBench("Filenames cache, lowercase searchs", filenamesGlobCache, "abcde")
+ runBench("Filenames cache, mixed case searchs", filenamesGlobCache, "abCDe")
-func BenchmarkGetGlob(b *testing.B) {
- for i := 0; i < b.N; i++ {
- _, err := GetGlob("**/foo")
- if err != nil {
- b.Fatal(err)
+ b.Run("GetGlob", func(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ _, err := GetGlob("**/foo")
+ if err != nil {
+ b.Fatal(err)
+ }
}
- }
+ })
}
diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go
@@ -142,6 +142,13 @@ func (r RootMapping) filename(name string) string {
return filepath.Join(r.To, strings.TrimPrefix(name, r.From))
}
+func (r RootMapping) trimFrom(name string) string {
+ if name == "" {
+ return ""
+ }
+ return strings.TrimPrefix(name, r.From)
+}
+
// A RootMappingFs maps several roots into one. Note that the root of this filesystem
// is directories only, and they will be returned in Readdir and Readdirnames
// in the order given.
@@ -170,7 +177,12 @@ func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) {
p = strings.TrimLeft(p, filepathSeparator)
return p
})
- fs := decorateDirs(bfs, r.Meta)
+
+ fs := bfs
+ if r.Meta.InclusionFilter != nil {
+ fs = newFilenameFilterFs(fs, r.To, r.Meta.InclusionFilter)
+ }
+ fs = decorateDirs(fs, r.Meta)
fi, err := fs.Stat("")
if err != nil {
return nil, errors.Wrap(err, "RootMappingFs.Dirs")
@@ -368,6 +380,10 @@ func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error)
for _, fi := range direntries {
meta := fi.(FileMetaInfo).Meta()
meta.Merge(rm.Meta)
+ if !rm.Meta.InclusionFilter.Match(strings.TrimPrefix(meta.Filename, meta.SourceRoot), fi.IsDir()) {
+ continue
+ }
+
if fi.IsDir() {
name := fi.Name()
if seen[name] {
@@ -508,7 +524,14 @@ func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) {
}
fileCount := 0
+ var wasFiltered bool
for _, root := range roots {
+ meta := root.fi.Meta()
+ if !meta.InclusionFilter.Match(strings.TrimPrefix(meta.Filename, meta.SourceRoot), root.fi.IsDir()) {
+ wasFiltered = true
+ continue
+ }
+
if !root.fi.IsDir() {
fileCount++
}
@@ -518,6 +541,9 @@ func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) {
}
if fileCount == 0 {
+ if wasFiltered {
+ return nil, os.ErrNotExist
+ }
// Dir only.
return []FileMetaInfo{newDirNameOnlyFileInfo(name, roots[0].Meta, fs.virtualDirOpener(name))}, nil
}
@@ -531,6 +557,9 @@ func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) {
}
func (fs *RootMappingFs) statRoot(root RootMapping, name string) (FileMetaInfo, bool, error) {
+ if !root.Meta.InclusionFilter.Match(root.trimFrom(name), root.fi.IsDir()) {
+ return nil, false, os.ErrNotExist
+ }
filename := root.filename(name)
fi, b, err := lstatIfPossible(fs.Fs, filename)
@@ -586,16 +615,23 @@ func (f *rootMappingFile) Name() string {
func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) {
if f.File != nil {
+
fis, err := f.File.Readdir(count)
if err != nil {
return nil, err
}
- for i, fi := range fis {
- fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta)
+ var result []os.FileInfo
+ for _, fi := range fis {
+ fim := decorateFileInfo(fi, f.fs, nil, "", "", f.meta)
+ meta := fim.Meta()
+ if f.meta.InclusionFilter.Match(strings.TrimPrefix(meta.Filename, meta.SourceRoot), fim.IsDir()) {
+ result = append(result, fim)
+ }
}
- return fis, nil
+ return result, nil
}
+
return f.fs.collectDirEntries(f.name)
}
diff --git a/hugofs/rootmapping_fs_test.go b/hugofs/rootmapping_fs_test.go
@@ -20,6 +20,8 @@ import (
"sort"
"testing"
+ "github.com/gohugoio/hugo/hugofs/glob"
+
"github.com/gohugoio/hugo/config"
qt "github.com/frankban/quicktest"
@@ -483,3 +485,70 @@ func TestRootMappingFsOsBase(t *testing.T) {
c.Assert(getDirnames("static/a/b/c"), qt.DeepEquals, []string{"d4", "f-1.txt", "f-2.txt", "f-3.txt", "ms-1.txt"})
}
+
+func TestRootMappingFileFilter(t *testing.T) {
+ c := qt.New(t)
+ fs := NewBaseFileDecorator(afero.NewMemMapFs())
+
+ for _, lang := range []string{"no", "en", "fr"} {
+ for i := 1; i <= 3; i++ {
+ c.Assert(afero.WriteFile(fs, filepath.Join(lang, fmt.Sprintf("my%s%d.txt", lang, i)), []byte("some text file for"+lang), 0755), qt.IsNil)
+ }
+ }
+
+ for _, lang := range []string{"no", "en", "fr"} {
+ for i := 1; i <= 3; i++ {
+ c.Assert(afero.WriteFile(fs, filepath.Join(lang, "sub", fmt.Sprintf("mysub%s%d.txt", lang, i)), []byte("some text file for"+lang), 0755), qt.IsNil)
+ }
+ }
+
+ rm := []RootMapping{
+ {
+ From: "content",
+ To: "no",
+ Meta: &FileMeta{Lang: "no", InclusionFilter: glob.MustNewFilenameFilter(nil, []string{"**.txt"})},
+ },
+ {
+ From: "content",
+ To: "en",
+ Meta: &FileMeta{Lang: "en"},
+ },
+ {
+ From: "content",
+ To: "fr",
+ Meta: &FileMeta{Lang: "fr", InclusionFilter: glob.MustNewFilenameFilter(nil, []string{"**.txt"})},
+ },
+ }
+
+ rfs, err := NewRootMappingFs(fs, rm...)
+ c.Assert(err, qt.IsNil)
+
+ assertExists := func(filename string, shouldExist bool) {
+ c.Helper()
+ filename = filepath.Clean(filename)
+ _, err1 := rfs.Stat(filename)
+ f, err2 := rfs.Open(filename)
+ if shouldExist {
+ c.Assert(err1, qt.IsNil)
+ c.Assert(err2, qt.IsNil)
+ c.Assert(f.Close(), qt.IsNil)
+ } else {
+ c.Assert(err1, qt.Not(qt.IsNil))
+ c.Assert(err2, qt.Not(qt.IsNil))
+ }
+ }
+
+ assertExists("content/myno1.txt", false)
+ assertExists("content/myen1.txt", true)
+ assertExists("content/myfr1.txt", false)
+
+ dirEntriesSub, err := afero.ReadDir(rfs, filepath.Join("content", "sub"))
+ c.Assert(err, qt.IsNil)
+ c.Assert(len(dirEntriesSub), qt.Equals, 3)
+
+ dirEntries, err := afero.ReadDir(rfs, "content")
+
+ c.Assert(err, qt.IsNil)
+ c.Assert(len(dirEntries), qt.Equals, 4)
+
+}
diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go
@@ -25,6 +25,9 @@ import (
"sync"
"github.com/gohugoio/hugo/htesting"
+ "github.com/gohugoio/hugo/hugofs/glob"
+
+ "github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/common/loggers"
"github.com/rogpeppe/go-internal/lockedfile"
@@ -127,7 +130,8 @@ func (b *BaseFs) RelContentDir(filename string) string {
return filename
}
-// AbsProjectContentDir tries to create a TODO1
+// AbsProjectContentDir tries to construct a filename below the most
+// relevant content directory.
func (b *BaseFs) AbsProjectContentDir(filename string) (string, string) {
isAbs := filepath.IsAbs(filename)
for _, dir := range b.SourceFilesystems.Content.Dirs {
@@ -623,6 +627,14 @@ func (b *sourceFilesystemsBuilder) createModFs(
mountWeight++
}
+ inclusionFilter, err := glob.NewFilenameFilter(
+ types.ToStringSlicePreserveString(mount.IncludeFiles),
+ types.ToStringSlicePreserveString(mount.ExcludeFiles),
+ )
+ if err != nil {
+ return err
+ }
+
base, filename := absPathify(mount.Source)
rm := hugofs.RootMapping{
@@ -631,9 +643,10 @@ func (b *sourceFilesystemsBuilder) createModFs(
ToBasedir: base,
Module: md.Module.Path(),
Meta: &hugofs.FileMeta{
- Watch: md.Watch(),
- Weight: mountWeight,
- Classifier: files.ContentClassContent,
+ Watch: md.Watch(),
+ Weight: mountWeight,
+ Classifier: files.ContentClassContent,
+ InclusionFilter: inclusionFilter,
},
}
diff --git a/hugolib/mount_filters_test.go b/hugolib/mount_filters_test.go
@@ -0,0 +1,119 @@
+// Copyright 2021 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/gohugoio/hugo/common/loggers"
+
+ "github.com/gohugoio/hugo/hugofs/files"
+
+ "github.com/gohugoio/hugo/htesting"
+ "github.com/gohugoio/hugo/hugofs"
+
+ qt "github.com/frankban/quicktest"
+)
+
+func TestMountFilters(t *testing.T) {
+ t.Parallel()
+ b := newTestSitesBuilder(t)
+ workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-test-mountfilters")
+ b.Assert(err, qt.IsNil)
+ defer clean()
+
+ for _, component := range files.ComponentFolders {
+ b.Assert(os.MkdirAll(filepath.Join(workingDir, component), 0777), qt.IsNil)
+ }
+ b.WithWorkingDir(workingDir).WithLogger(loggers.NewInfoLogger())
+ b.WithConfigFile("toml", fmt.Sprintf(`
+workingDir = %q
+
+[module]
+[[module.mounts]]
+source = 'content'
+target = 'content'
+excludeFiles = "/a/c/**"
+[[module.mounts]]
+source = 'static'
+target = 'static'
+[[module.mounts]]
+source = 'layouts'
+target = 'layouts'
+excludeFiles = "/**/foo.html"
+[[module.mounts]]
+source = 'data'
+target = 'data'
+includeFiles = "/mydata/**"
+[[module.mounts]]
+source = 'assets'
+target = 'assets'
+excludeFiles = "/**exclude.*"
+[[module.mounts]]
+source = 'i18n'
+target = 'i18n'
+[[module.mounts]]
+source = 'archetypes'
+target = 'archetypes'
+
+
+`, workingDir))
+
+ b.WithContent("/a/b/p1.md", "---\ntitle: Include\n---")
+ b.WithContent("/a/c/p2.md", "---\ntitle: Exclude\n---")
+
+ b.WithSourceFile(
+ "data/mydata/b.toml", `b1='bval'`,
+ "data/nodata/c.toml", `c1='bval'`,
+ "layouts/partials/foo.html", `foo`,
+ "assets/exclude.txt", `foo`,
+ "assets/js/exclude.js", `foo`,
+ "assets/js/include.js", `foo`,
+ "assets/js/exclude.js", `foo`,
+ )
+
+ b.WithTemplatesAdded("index.html", `
+
+Data: {{ site.Data }}:END
+
+Template: {{ templates.Exists "partials/foo.html" }}:END
+Resource1: {{ resources.Get "js/include.js" }}:END
+Resource2: {{ resources.Get "js/exclude.js" }}:END
+Resource3: {{ resources.Get "exclude.txt" }}:END
+Resources: {{ resources.Match "**.js" }}
+`)
+
+ b.Build(BuildCfg{})
+
+ assertExists := func(name string, shouldExist bool) {
+ b.Helper()
+ b.Assert(b.CheckExists(filepath.Join(workingDir, name)), qt.Equals, shouldExist)
+ }
+
+ assertExists("public/a/b/p1/index.html", true)
+ assertExists("public/a/c/p2/index.html", false)
+
+ b.AssertFileContent(filepath.Join(workingDir, "public", "index.html"), `
+Data: map[mydata:map[b:map[b1:bval]]]:END
+Template: false
+Resource1: js/include.js:END
+Resource2: :END
+Resource3: :END
+Resources: [js/include.js]
+`)
+
+}
diff --git a/modules/config.go b/modules/config.go
@@ -379,6 +379,11 @@ type Mount struct {
Lang string // any language code associated with this mount.
+ // Include only files matching the given Glob patterns (string or slice).
+ IncludeFiles interface{}
+
+ // Exclude all files matching the given Glob patterns (string or slice).
+ ExcludeFiles interface{}
}
func (m Mount) Component() string {
diff --git a/source/sourceSpec.go b/source/sourceSpec.go
@@ -74,7 +74,7 @@ func NewSourceSpec(ps *helpers.PathSpec, inclusionFilter *glob.FilenameFilter, f
}
}
shouldInclude := func(filename string) bool {
- if !inclusionFilter.Match(filename) {
+ if !inclusionFilter.Match(filename, false) {
return false
}
for _, r := range regexps {