hugofs: Add includeFiles and excludeFiles to mount configuration

Fixes #9042

hugofs/fileinfo.go

@@ -23,6 +23,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/hugofs/glob"
+
 	"github.com/gohugoio/hugo/hugofs/files"
 	"golang.org/x/text/unicode/norm"
 
@@ -76,6 +78,9 @@ type FileMeta struct {
 	Fs           afero.Fs
 	OpenFunc     func() (afero.File, error)
 	JoinStatFunc func(name string) (FileMetaInfo, error)
+
+	// Include only files or directories that match.
+	InclusionFilter *glob.FilenameFilter
 }
 
 func (m *FileMeta) Copy() *FileMeta {
@@ -95,10 +100,17 @@ func (m *FileMeta) Merge(from *FileMeta) {
 
 	for i := 0; i < dstv.NumField(); i++ {
 		v := dstv.Field(i)
+		if !v.CanSet() {
+			continue
+		}
 		if !hreflect.IsTruthfulValue(v) {
 			v.Set(srcv.Field(i))
 		}
 	}
+
+	if m.InclusionFilter == nil {
+		m.InclusionFilter = from.InclusionFilter
+	}
 }
 
 func (f *FileMeta) Open() (afero.File, error) {

hugofs/filename_filter_fs.go

@@ -0,0 +1,170 @@
+// Copyright 2021 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"os"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/gohugoio/hugo/hugofs/glob"
+	"github.com/spf13/afero"
+)
+
+func newFilenameFilterFs(fs afero.Fs, base string, filter *glob.FilenameFilter) afero.Fs {
+	return &filenameFilterFs{
+		fs:     fs,
+		base:   base,
+		filter: filter,
+	}
+}
+
+// filenameFilterFs is a filesystem that filters by filename.
+type filenameFilterFs struct {
+	base string
+	fs   afero.Fs
+
+	filter *glob.FilenameFilter
+}
+
+func (fs *filenameFilterFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
+	fi, b, err := fs.fs.(afero.Lstater).LstatIfPossible(name)
+	if err != nil {
+		return nil, false, err
+	}
+	if !fs.filter.Match(name, fi.IsDir()) {
+		return nil, false, os.ErrNotExist
+	}
+	return fi, b, nil
+}
+
+func (fs *filenameFilterFs) Open(name string) (afero.File, error) {
+	fi, err := fs.fs.Stat(name)
+	if err != nil {
+		return nil, err
+	}
+
+	if !fs.filter.Match(name, fi.IsDir()) {
+		return nil, os.ErrNotExist
+	}
+
+	f, err := fs.fs.Open(name)
+	if err != nil {
+		return nil, err
+	}
+
+	if !fi.IsDir() {
+		return f, nil
+	}
+
+	return &filenameFilterDir{
+		File:   f,
+		base:   fs.base,
+		filter: fs.filter,
+	}, nil
+}
+
+func (fs *filenameFilterFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
+	return fs.Open(name)
+}
+
+func (fs *filenameFilterFs) Stat(name string) (os.FileInfo, error) {
+	fi, _, err := fs.LstatIfPossible(name)
+	return fi, err
+}
+
+func (fs *filenameFilterFs) getOpener(name string) func() (afero.File, error) {
+	return func() (afero.File, error) {
+		return fs.Open(name)
+	}
+}
+
+type filenameFilterDir struct {
+	afero.File
+	base   string
+	filter *glob.FilenameFilter
+}
+
+func (f *filenameFilterDir) Readdir(count int) ([]os.FileInfo, error) {
+	fis, err := f.File.Readdir(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	var result []os.FileInfo
+	for _, fi := range fis {
+		fim := fi.(FileMetaInfo)
+		if f.filter.Match(strings.TrimPrefix(fim.Meta().Filename, f.base), fim.IsDir()) {
+			result = append(result, fi)
+		}
+	}
+
+	return result, nil
+}
+
+func (f *filenameFilterDir) Readdirnames(count int) ([]string, error) {
+	dirsi, err := f.Readdir(count)
+	if err != nil {
+		return nil, err
+	}
+
+	dirs := make([]string, len(dirsi))
+	for i, d := range dirsi {
+		dirs[i] = d.Name()
+	}
+	return dirs, nil
+}
+
+func (fs *filenameFilterFs) Chmod(n string, m os.FileMode) error {
+	return syscall.EPERM
+}
+
+func (fs *filenameFilterFs) Chtimes(n string, a, m time.Time) error {
+	return syscall.EPERM
+}
+
+func (fs *filenameFilterFs) Chown(n string, uid, gid int) error {
+	return syscall.EPERM
+}
+
+func (fs *filenameFilterFs) ReadDir(name string) ([]os.FileInfo, error) {
+	panic("not implemented")
+}
+
+func (fs *filenameFilterFs) Remove(n string) error {
+	return syscall.EPERM
+}
+
+func (fs *filenameFilterFs) RemoveAll(p string) error {
+	return syscall.EPERM
+}
+
+func (fs *filenameFilterFs) Rename(o, n string) error {
+	return syscall.EPERM
+}
+func (fs *filenameFilterFs) Create(n string) (afero.File, error) {
+	return nil, syscall.EPERM
+}
+func (fs *filenameFilterFs) Name() string {
+	return "FinameFilterFS"
+}
+
+func (fs *filenameFilterFs) Mkdir(n string, p os.FileMode) error {
+	return syscall.EPERM
+}
+
+func (fs *filenameFilterFs) MkdirAll(n string, p os.FileMode) error {
+	return syscall.EPERM
+}

hugofs/filename_filter_fs_test.go

@@ -0,0 +1,83 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/gohugoio/hugo/hugofs/glob"
+
+	"github.com/spf13/afero"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestFilenameFilterFs(t *testing.T) {
+	c := qt.New(t)
+
+	base := filepath.FromSlash("/mybase")
+
+	fs := NewBaseFileDecorator(afero.NewMemMapFs())
+
+	for _, letter := range []string{"a", "b", "c"} {
+		for i := 1; i <= 3; i++ {
+			c.Assert(afero.WriteFile(fs, filepath.Join(base, letter, fmt.Sprintf("my%d.txt", i)), []byte("some text file for"+letter), 0755), qt.IsNil)
+			c.Assert(afero.WriteFile(fs, filepath.Join(base, letter, fmt.Sprintf("my%d.json", i)), []byte("some json file for"+letter), 0755), qt.IsNil)
+		}
+	}
+
+	fs = afero.NewBasePathFs(fs, base)
+
+	filter, err := glob.NewFilenameFilter(nil, []string{"/b/**.txt"})
+	c.Assert(err, qt.IsNil)
+
+	fs = newFilenameFilterFs(fs, base, filter)
+
+	assertExists := func(filename string, shouldExist bool) {
+		filename = filepath.Clean(filename)
+		_, err1 := fs.Stat(filename)
+		f, err2 := fs.Open(filename)
+		if shouldExist {
+			c.Assert(err1, qt.IsNil)
+			c.Assert(err2, qt.IsNil)
+			defer f.Close()
+
+		} else {
+			for _, err := range []error{err1, err2} {
+				c.Assert(err, qt.Not(qt.IsNil))
+				c.Assert(errors.Is(err, os.ErrNotExist), qt.IsTrue)
+			}
+		}
+	}
+
+	assertExists("/a/my1.txt", true)
+	assertExists("/b/my1.txt", false)
+
+	dirB, err := fs.Open("/b")
+	defer dirB.Close()
+	c.Assert(err, qt.IsNil)
+	dirBEntries, err := dirB.Readdirnames(-1)
+	c.Assert(dirBEntries, qt.DeepEquals, []string{"my1.json", "my2.json", "my3.json"})
+
+	dirC, err := fs.Open("/c")
+	defer dirC.Close()
+	c.Assert(err, qt.IsNil)
+	dirCEntries, err := dirC.Readdirnames(-1)
+	c.Assert(dirCEntries, qt.DeepEquals, []string{"my1.json", "my1.txt", "my2.json", "my2.txt", "my3.json", "my3.txt"})
+
+}

hugofs/glob/filename_filter.go

@@ -0,0 +1,159 @@
+// Copyright 2021 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package glob
+
+import (
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/gobwas/glob"
+)
+
+type FilenameFilter struct {
+	shouldInclude func(filename string) bool
+	inclusions    []glob.Glob
+	dirInclusions []glob.Glob
+	exclusions    []glob.Glob
+	isWindows     bool
+}
+
+func normalizeFilenameGlobPattern(s string) string {
+	// Use Unix separators even on Windows.
+	s = filepath.ToSlash(s)
+	if !strings.HasPrefix(s, "/") {
+		s = "/" + s
+	}
+	return s
+}
+
+// NewFilenameFilter creates a new Glob where the Match method will
+// return true if the file should be included.
+// Note that the inclusions will be checked first.
+func NewFilenameFilter(inclusions, exclusions []string) (*FilenameFilter, error) {
+	if inclusions == nil && exclusions == nil {
+		return nil, nil
+	}
+	filter := &FilenameFilter{isWindows: isWindows}
+
+	for _, include := range inclusions {
+		include = normalizeFilenameGlobPattern(include)
+		g, err := filenamesGlobCache.GetGlob(include)
+		if err != nil {
+			return nil, err
+		}
+		filter.inclusions = append(filter.inclusions, g)
+
+		// For mounts that do directory walking (e.g. content) we
+		// must make sure that all directories up to this inclusion also
+		// gets included.
+		dir := path.Dir(include)
+		parts := strings.Split(dir, "/")
+		for i, _ := range parts {
+			pattern := "/" + filepath.Join(parts[:i+1]...)
+			g, err := filenamesGlobCache.GetGlob(pattern)
+			if err != nil {
+				return nil, err
+			}
+			filter.dirInclusions = append(filter.dirInclusions, g)
+		}
+	}
+
+	for _, exclude := range exclusions {
+		exclude = normalizeFilenameGlobPattern(exclude)
+		g, err := filenamesGlobCache.GetGlob(exclude)
+		if err != nil {
+			return nil, err
+		}
+		filter.exclusions = append(filter.exclusions, g)
+	}
+
+	return filter, nil
+}
+
+// MustNewFilenameFilter invokes NewFilenameFilter and panics on error.
+func MustNewFilenameFilter(inclusions, exclusions []string) *FilenameFilter {
+	filter, err := NewFilenameFilter(inclusions, exclusions)
+	if err != nil {
+		panic(err)
+	}
+	return filter
+}
+
+// NewFilenameFilterForInclusionFunc create a new filter using the provided inclusion func.
+func NewFilenameFilterForInclusionFunc(shouldInclude func(filename string) bool) *FilenameFilter {
+	return &FilenameFilter{shouldInclude: shouldInclude, isWindows: isWindows}
+}
+
+// Match returns whether filename should be included.
+func (f *FilenameFilter) Match(filename string, isDir bool) bool {
+	if f == nil {
+		return true
+	}
+	return f.doMatch(filename, isDir)
+	/*if f.shouldInclude == nil {
+		fmt.Printf("Match: %q (%t) => %t\n", filename, isDir, isMatch)
+	}
+	return isMatch*/
+}
+
+func (f *FilenameFilter) doMatch(filename string, isDir bool) bool {
+	if f == nil {
+		return true
+	}
+
+	if !strings.HasPrefix(filename, filepathSeparator) {
+		filename = filepathSeparator + filename
+	}
+
+	if f.shouldInclude != nil {
+		if f.shouldInclude(filename) {
+			return true
+		}
+		if f.isWindows {
+			// The Glob matchers below handles this by themselves,
+			// for the shouldInclude we need to take some extra steps
+			// to make this robust.
+			winFilename := filepath.FromSlash(filename)
+			if filename != winFilename {
+				if f.shouldInclude(winFilename) {
+					return true
+				}
+			}
+		}
+
+	}
+
+	for _, inclusion := range f.inclusions {
+		if inclusion.Match(filename) {
+			return true
+		}
+	}
+
+	if isDir && f.inclusions != nil {
+		for _, inclusion := range f.dirInclusions {
+			if inclusion.Match(filename) {
+				return true
+			}
+		}
+	}
+
+	for _, exclusion := range f.exclusions {
+		if exclusion.Match(filename) {
+			return false
+		}
+	}
+
+	return f.inclusions == nil && f.shouldInclude == nil
+}

hugofs/glob/filename_filter_test.go

@@ -0,0 +1,70 @@
+// Copyright 2021 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package glob
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestFilenameFilter(t *testing.T) {
+	c := qt.New(t)
+
+	excludeAlmostAllJSON, err := NewFilenameFilter([]string{"/a/b/c/foo.json"}, []string{"**.json"})
+	c.Assert(err, qt.IsNil)
+	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/data/my.json"), false), qt.Equals, false)
+	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c/foo.json"), false), qt.Equals, true)
+	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c/foo.bar"), false), qt.Equals, false)
+	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c"), true), qt.Equals, true)
+	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b"), true), qt.Equals, true)
+	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a"), true), qt.Equals, true)
+	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/"), true), qt.Equals, true)
+	c.Assert(excludeAlmostAllJSON.Match("", true), qt.Equals, true)
+
+	excludeAllButFooJSON, err := NewFilenameFilter([]string{"/a/**/foo.json"}, []string{"**.json"})
+	c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/data/my.json"), false), qt.Equals, false)
+	c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/c/d/e/foo.json"), false), qt.Equals, true)
+	c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/c"), true), qt.Equals, true)
+	c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/"), true), qt.Equals, true)
+	c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/"), true), qt.Equals, true)
+	c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/b"), true), qt.Equals, false)
+	c.Assert(err, qt.IsNil)
+
+	nopFilter, err := NewFilenameFilter(nil, nil)
+	c.Assert(err, qt.IsNil)
+	c.Assert(nopFilter.Match("ab.txt", false), qt.Equals, true)
+
+	includeOnlyFilter, err := NewFilenameFilter([]string{"**.json", "**.jpg"}, nil)
+	c.Assert(err, qt.IsNil)
+	c.Assert(includeOnlyFilter.Match("ab.json", false), qt.Equals, true)
+	c.Assert(includeOnlyFilter.Match("ab.jpg", false), qt.Equals, true)
+	c.Assert(includeOnlyFilter.Match("ab.gif", false), qt.Equals, false)
+
+	exlcudeOnlyFilter, err := NewFilenameFilter(nil, []string{"**.json", "**.jpg"})
+	c.Assert(err, qt.IsNil)
+	c.Assert(exlcudeOnlyFilter.Match("ab.json", false), qt.Equals, false)
+	c.Assert(exlcudeOnlyFilter.Match("ab.jpg", false), qt.Equals, false)
+	c.Assert(exlcudeOnlyFilter.Match("ab.gif", false), qt.Equals, true)
+
+	var nilFilter *FilenameFilter
+	c.Assert(nilFilter.Match("ab.gif", false), qt.Equals, true)
+
+	funcFilter := NewFilenameFilterForInclusionFunc(func(s string) bool { return strings.HasSuffix(s, ".json") })
+	c.Assert(funcFilter.Match("ab.json", false), qt.Equals, true)
+	c.Assert(funcFilter.Match("ab.bson", false), qt.Equals, false)
+
+}

hugofs/glob/glob.go

@@ -1,4 +1,4 @@
-// Copyright 2019 The Hugo Authors. All rights reserved.
+// Copyright 2021 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
 package glob
 
 import (
+	"os"
 	"path"
 	"path/filepath"
 	"runtime"
@@ -24,6 +25,8 @@ import (
 	"github.com/gobwas/glob/syntax"
 )
 
+const filepathSeparator = string(os.PathSeparator)
+
 var (
 	isWindows        = runtime.GOOS == "windows"
 	defaultGlobCache = &globCache{
@@ -33,7 +36,7 @@ var (
 	}
 
 	filenamesGlobCache = &globCache{
-		isCaseSensitive: true, // TODO(bep) bench
+		isCaseSensitive: false, // As long as the search strings are all lower case, this does not allocate.
 		isWindows:       isWindows,
 		cache:           make(map[string]globErr),
 	}
@@ -161,78 +164,3 @@ func HasGlobChar(s string) bool {
 	}
 	return false
 }
-
-type FilenameFilter struct {
-	shouldInclude func(filename string) bool
-	inclusions    []glob.Glob
-	exclusions    []glob.Glob
-	isWindows     bool
-}
-
-// NewFilenameFilter creates a new Glob where the Match method will
-// return true if the file should be exluded.
-// Note that the inclusions will be checked first.
-func NewFilenameFilter(inclusions, exclusions []string) (*FilenameFilter, error) {
-	filter := &FilenameFilter{isWindows: isWindows}
-
-	for _, include := range inclusions {
-		g, err := filenamesGlobCache.GetGlob(filepath.FromSlash(include))
-		if err != nil {
-			return nil, err
-		}
-		filter.inclusions = append(filter.inclusions, g)
-	}
-	for _, exclude := range exclusions {
-		g, err := filenamesGlobCache.GetGlob(filepath.FromSlash(exclude))
-		if err != nil {
-			return nil, err
-		}
-		filter.exclusions = append(filter.exclusions, g)
-	}
-
-	return filter, nil
-}
-
-// NewFilenameFilterForInclusionFunc create a new filter using the provided inclusion func.
-func NewFilenameFilterForInclusionFunc(shouldInclude func(filename string) bool) *FilenameFilter {
-	return &FilenameFilter{shouldInclude: shouldInclude, isWindows: isWindows}
-}
-
-// Match returns whether filename should be included.
-func (f *FilenameFilter) Match(filename string) bool {
-	if f == nil {
-		return true
-	}
-
-	if f.shouldInclude != nil {
-		if f.shouldInclude(filename) {
-			return true
-		}
-		if f.isWindows {
-			// The Glob matchers below handles this by themselves,
-			// for the shouldInclude we need to take some extra steps
-			// to make this robust.
-			winFilename := filepath.FromSlash(filename)
-			if filename != winFilename {
-				if f.shouldInclude(winFilename) {
-					return true
-				}
-			}
-		}
-
-	}
-
-	for _, inclusion := range f.inclusions {
-		if inclusion.Match(filename) {
-			return true
-		}
-	}
-
-	for _, exclusion := range f.exclusions {
-		if exclusion.Match(filename) {
-			return false
-		}
-	}
-
-	return f.inclusions == nil && f.shouldInclude == nil
-}

hugofs/glob/glob_test.go

@@ -1,4 +1,4 @@
-// Copyright 2019 The Hugo Authors. All rights reserved.
+// Copyright 2021 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,7 +15,6 @@ package glob
 
 import (
 	"path/filepath"
-	"strings"
 	"testing"
 
 	qt "github.com/frankban/quicktest"
@@ -67,51 +66,38 @@ func TestNormalizePath(t *testing.T) {
 }
 
 func TestGetGlob(t *testing.T) {
-	c := qt.New(t)
-	g, err := GetGlob("**.JSON")
-	c.Assert(err, qt.IsNil)
-	c.Assert(g.Match("data/my.json"), qt.Equals, true)
-}
-
-func TestFilenameFilter(t *testing.T) {
-	c := qt.New(t)
-
-	excludeAlmostAllJSON, err := NewFilenameFilter([]string{"a/b/c/foo.json"}, []string{"**.json"})
-	c.Assert(err, qt.IsNil)
-	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("data/my.json")), qt.Equals, false)
-	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("a/b/c/foo.json")), qt.Equals, true)
-	c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("a/b/c/foo.bar")), qt.Equals, false)
-
-	nopFilter, err := NewFilenameFilter(nil, nil)
-	c.Assert(err, qt.IsNil)
-	c.Assert(nopFilter.Match("ab.txt"), qt.Equals, true)
-
-	includeOnlyFilter, err := NewFilenameFilter([]string{"**.json", "**.jpg"}, nil)
-	c.Assert(err, qt.IsNil)
-	c.Assert(includeOnlyFilter.Match("ab.json"), qt.Equals, true)
-	c.Assert(includeOnlyFilter.Match("ab.jpg"), qt.Equals, true)
-	c.Assert(includeOnlyFilter.Match("ab.gif"), qt.Equals, false)
-
-	exlcudeOnlyFilter, err := NewFilenameFilter(nil, []string{"**.json", "**.jpg"})
-	c.Assert(err, qt.IsNil)
-	c.Assert(exlcudeOnlyFilter.Match("ab.json"), qt.Equals, false)
-	c.Assert(exlcudeOnlyFilter.Match("ab.jpg"), qt.Equals, false)
-	c.Assert(exlcudeOnlyFilter.Match("ab.gif"), qt.Equals, true)
-
-	var nilFilter *FilenameFilter
-	c.Assert(nilFilter.Match("ab.gif"), qt.Equals, true)
-
-	funcFilter := NewFilenameFilterForInclusionFunc(func(s string) bool { return strings.HasSuffix(s, ".json") })
-	c.Assert(funcFilter.Match("ab.json"), qt.Equals, true)
-	c.Assert(funcFilter.Match("ab.bson"), qt.Equals, false)
-
+	for _, cache := range []*globCache{defaultGlobCache, filenamesGlobCache} {
+		c := qt.New(t)
+		g, err := cache.GetGlob("**.JSON")
+		c.Assert(err, qt.IsNil)
+		c.Assert(g.Match("data/my.jSon"), qt.Equals, true)
+	}
 }
 
 func BenchmarkGetGlob(b *testing.B) {
-	for i := 0; i < b.N; i++ {
-		_, err := GetGlob("**/foo")
-		if err != nil {
-			b.Fatal(err)
-		}
+
+	runBench := func(name string, cache *globCache, search string) {
+		b.Run(name, func(b *testing.B) {
+			g, err := GetGlob("**/foo")
+			if err != nil {
+				b.Fatal(err)
+			}
+			for i := 0; i < b.N; i++ {
+				_ = g.Match(search)
+			}
+		})
 	}
+
+	runBench("Default cache", defaultGlobCache, "abcde")
+	runBench("Filenames cache, lowercase searchs", filenamesGlobCache, "abcde")
+	runBench("Filenames cache, mixed case searchs", filenamesGlobCache, "abCDe")
+
+	b.Run("GetGlob", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			_, err := GetGlob("**/foo")
+			if err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
 }

hugofs/rootmapping_fs.go

@@ -142,6 +142,13 @@ func (r RootMapping) filename(name string) string {
 	return filepath.Join(r.To, strings.TrimPrefix(name, r.From))
 }
 
+func (r RootMapping) trimFrom(name string) string {
+	if name == "" {
+		return ""
+	}
+	return strings.TrimPrefix(name, r.From)
+}
+
 // A RootMappingFs maps several roots into one. Note that the root of this filesystem
 // is directories only, and they will be returned in Readdir and Readdirnames
 // in the order given.
@@ -170,7 +177,12 @@ func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) {
 			p = strings.TrimLeft(p, filepathSeparator)
 			return p
 		})
-		fs := decorateDirs(bfs, r.Meta)
+
+		fs := bfs
+		if r.Meta.InclusionFilter != nil {
+			fs = newFilenameFilterFs(fs, r.To, r.Meta.InclusionFilter)
+		}
+		fs = decorateDirs(fs, r.Meta)
 		fi, err := fs.Stat("")
 		if err != nil {
 			return nil, errors.Wrap(err, "RootMappingFs.Dirs")
@@ -368,6 +380,10 @@ func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error)
 		for _, fi := range direntries {
 			meta := fi.(FileMetaInfo).Meta()
 			meta.Merge(rm.Meta)
+			if !rm.Meta.InclusionFilter.Match(strings.TrimPrefix(meta.Filename, meta.SourceRoot), fi.IsDir()) {
+				continue
+			}
+
 			if fi.IsDir() {
 				name := fi.Name()
 				if seen[name] {
@@ -508,7 +524,14 @@ func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) {
 	}
 
 	fileCount := 0
+	var wasFiltered bool
 	for _, root := range roots {
+		meta := root.fi.Meta()
+		if !meta.InclusionFilter.Match(strings.TrimPrefix(meta.Filename, meta.SourceRoot), root.fi.IsDir()) {
+			wasFiltered = true
+			continue
+		}
+
 		if !root.fi.IsDir() {
 			fileCount++
 		}
@@ -518,6 +541,9 @@ func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) {
 	}
 
 	if fileCount == 0 {
+		if wasFiltered {
+			return nil, os.ErrNotExist
+		}
 		// Dir only.
 		return []FileMetaInfo{newDirNameOnlyFileInfo(name, roots[0].Meta, fs.virtualDirOpener(name))}, nil
 	}
@@ -531,6 +557,9 @@ func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) {
 }
 
 func (fs *RootMappingFs) statRoot(root RootMapping, name string) (FileMetaInfo, bool, error) {
+	if !root.Meta.InclusionFilter.Match(root.trimFrom(name), root.fi.IsDir()) {
+		return nil, false, os.ErrNotExist
+	}
 	filename := root.filename(name)
 
 	fi, b, err := lstatIfPossible(fs.Fs, filename)
@@ -586,16 +615,23 @@ func (f *rootMappingFile) Name() string {
 
 func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) {
 	if f.File != nil {
+
 		fis, err := f.File.Readdir(count)
 		if err != nil {
 			return nil, err
 		}
 
-		for i, fi := range fis {
-			fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta)
+		var result []os.FileInfo
+		for _, fi := range fis {
+			fim := decorateFileInfo(fi, f.fs, nil, "", "", f.meta)
+			meta := fim.Meta()
+			if f.meta.InclusionFilter.Match(strings.TrimPrefix(meta.Filename, meta.SourceRoot), fim.IsDir()) {
+				result = append(result, fim)
+			}
 		}
-		return fis, nil
+		return result, nil
 	}
+
 	return f.fs.collectDirEntries(f.name)
 }
 

hugofs/rootmapping_fs_test.go

@@ -20,6 +20,8 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/gohugoio/hugo/hugofs/glob"
+
 	"github.com/gohugoio/hugo/config"
 
 	qt "github.com/frankban/quicktest"
@@ -483,3 +485,70 @@ func TestRootMappingFsOsBase(t *testing.T) {
 
 	c.Assert(getDirnames("static/a/b/c"), qt.DeepEquals, []string{"d4", "f-1.txt", "f-2.txt", "f-3.txt", "ms-1.txt"})
 }
+
+func TestRootMappingFileFilter(t *testing.T) {
+	c := qt.New(t)
+	fs := NewBaseFileDecorator(afero.NewMemMapFs())
+
+	for _, lang := range []string{"no", "en", "fr"} {
+		for i := 1; i <= 3; i++ {
+			c.Assert(afero.WriteFile(fs, filepath.Join(lang, fmt.Sprintf("my%s%d.txt", lang, i)), []byte("some text file for"+lang), 0755), qt.IsNil)
+		}
+	}
+
+	for _, lang := range []string{"no", "en", "fr"} {
+		for i := 1; i <= 3; i++ {
+			c.Assert(afero.WriteFile(fs, filepath.Join(lang, "sub", fmt.Sprintf("mysub%s%d.txt", lang, i)), []byte("some text file for"+lang), 0755), qt.IsNil)
+		}
+	}
+
+	rm := []RootMapping{
+		{
+			From: "content",
+			To:   "no",
+			Meta: &FileMeta{Lang: "no", InclusionFilter: glob.MustNewFilenameFilter(nil, []string{"**.txt"})},
+		},
+		{
+			From: "content",
+			To:   "en",
+			Meta: &FileMeta{Lang: "en"},
+		},
+		{
+			From: "content",
+			To:   "fr",
+			Meta: &FileMeta{Lang: "fr", InclusionFilter: glob.MustNewFilenameFilter(nil, []string{"**.txt"})},
+		},
+	}
+
+	rfs, err := NewRootMappingFs(fs, rm...)
+	c.Assert(err, qt.IsNil)
+
+	assertExists := func(filename string, shouldExist bool) {
+		c.Helper()
+		filename = filepath.Clean(filename)
+		_, err1 := rfs.Stat(filename)
+		f, err2 := rfs.Open(filename)
+		if shouldExist {
+			c.Assert(err1, qt.IsNil)
+			c.Assert(err2, qt.IsNil)
+			c.Assert(f.Close(), qt.IsNil)
+		} else {
+			c.Assert(err1, qt.Not(qt.IsNil))
+			c.Assert(err2, qt.Not(qt.IsNil))
+		}
+	}
+
+	assertExists("content/myno1.txt", false)
+	assertExists("content/myen1.txt", true)
+	assertExists("content/myfr1.txt", false)
+
+	dirEntriesSub, err := afero.ReadDir(rfs, filepath.Join("content", "sub"))
+	c.Assert(err, qt.IsNil)
+	c.Assert(len(dirEntriesSub), qt.Equals, 3)
+
+	dirEntries, err := afero.ReadDir(rfs, "content")
+
+	c.Assert(err, qt.IsNil)
+	c.Assert(len(dirEntries), qt.Equals, 4)
+
+}