Add Hugo Modules

This commit implements Hugo Modules.

This is a broad subject, but some keywords include:

* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`,  `hugo mod get`,  `hugo mod graph`,  `hugo mod tidy`, and  `hugo mod vendor`.

All of the above is backed by Go Modules.

Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092

hugofs/basepath_real_filename_fs.go

@@ -1,91 +0,0 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hugofs
-
-import (
-	"os"
-
-	"github.com/spf13/afero"
-)
-
-// RealFilenameInfo is a thin wrapper around os.FileInfo adding the real filename.
-type RealFilenameInfo interface {
-	os.FileInfo
-
-	// This is the real filename to the file in the underlying filesystem.
-	RealFilename() string
-}
-
-type realFilenameInfo struct {
-	os.FileInfo
-	realFilename string
-}
-
-func (f *realFilenameInfo) RealFilename() string {
-	return f.realFilename
-}
-
-// NewBasePathRealFilenameFs returns a new BasePathRealFilenameFs instance
-// using base.
-func NewBasePathRealFilenameFs(base *afero.BasePathFs) *BasePathRealFilenameFs {
-	return &BasePathRealFilenameFs{BasePathFs: base}
-}
-
-// BasePathRealFilenameFs is a thin wrapper around afero.BasePathFs that
-// provides the real filename in Stat and LstatIfPossible.
-type BasePathRealFilenameFs struct {
-	*afero.BasePathFs
-}
-
-// Stat returns the os.FileInfo structure describing a given file.  If there is
-// an error, it will be of type *os.PathError.
-func (b *BasePathRealFilenameFs) Stat(name string) (os.FileInfo, error) {
-	fi, err := b.BasePathFs.Stat(name)
-	if err != nil {
-		return nil, err
-	}
-
-	if _, ok := fi.(RealFilenameInfo); ok {
-		return fi, nil
-	}
-
-	filename, err := b.RealPath(name)
-	if err != nil {
-		return nil, &os.PathError{Op: "stat", Path: name, Err: err}
-	}
-
-	return &realFilenameInfo{FileInfo: fi, realFilename: filename}, nil
-}
-
-// LstatIfPossible returns the os.FileInfo structure describing a given file.
-// It attempts to use Lstat if supported or defers to the os.  In addition to
-// the FileInfo, a boolean is returned telling whether Lstat was called.
-func (b *BasePathRealFilenameFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
-
-	fi, ok, err := b.BasePathFs.LstatIfPossible(name)
-	if err != nil {
-		return nil, false, err
-	}
-
-	if _, ok := fi.(RealFilenameInfo); ok {
-		return fi, ok, nil
-	}
-
-	filename, err := b.RealPath(name)
-	if err != nil {
-		return nil, false, &os.PathError{Op: "lstat", Path: name, Err: err}
-	}
-
-	return &realFilenameInfo{FileInfo: fi, realFilename: filename}, ok, nil
-}

hugofs/decorators.go

@@ -0,0 +1,205 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/spf13/afero"
+)
+
+func decorateDirs(fs afero.Fs, meta FileMeta) afero.Fs {
+	ffs := &baseFileDecoratorFs{Fs: fs}
+
+	decorator := func(fi os.FileInfo, name string) (os.FileInfo, error) {
+		if !fi.IsDir() {
+			// Leave regular files as they are.
+			return fi, nil
+		}
+
+		return decorateFileInfo(fi, fs, nil, "", "", meta), nil
+	}
+
+	ffs.decorate = decorator
+
+	return ffs
+
+}
+
+func decoratePath(fs afero.Fs, createPath func(name string) string) afero.Fs {
+
+	ffs := &baseFileDecoratorFs{Fs: fs}
+
+	decorator := func(fi os.FileInfo, name string) (os.FileInfo, error) {
+		path := createPath(name)
+
+		return decorateFileInfo(fi, fs, nil, "", path, nil), nil
+	}
+
+	ffs.decorate = decorator
+
+	return ffs
+
+}
+
+// DecorateBasePathFs adds Path info to files and directories in the
+// provided BasePathFs, using the base as base.
+func DecorateBasePathFs(base *afero.BasePathFs) afero.Fs {
+	basePath, _ := base.RealPath("")
+	if !strings.HasSuffix(basePath, filepathSeparator) {
+		basePath += filepathSeparator
+	}
+
+	ffs := &baseFileDecoratorFs{Fs: base}
+
+	decorator := func(fi os.FileInfo, name string) (os.FileInfo, error) {
+		path := strings.TrimPrefix(name, basePath)
+
+		return decorateFileInfo(fi, base, nil, "", path, nil), nil
+	}
+
+	ffs.decorate = decorator
+
+	return ffs
+}
+
+// NewBaseFileDecorator decorates the given Fs to provide the real filename
+// and an Opener func. If
+func NewBaseFileDecorator(fs afero.Fs) afero.Fs {
+
+	ffs := &baseFileDecoratorFs{Fs: fs}
+
+	decorator := func(fi os.FileInfo, filename string) (os.FileInfo, error) {
+		// Store away the original in case it's a symlink.
+		meta := FileMeta{metaKeyName: fi.Name()}
+		isSymlink := isSymlink(fi)
+		if isSymlink {
+			meta[metaKeyOriginalFilename] = filename
+			link, err := filepath.EvalSymlinks(filename)
+			if err != nil {
+				return nil, err
+			}
+
+			fi, err = fs.Stat(link)
+			if err != nil {
+				return nil, err
+			}
+
+			filename = link
+			meta[metaKeyIsSymlink] = true
+
+		}
+
+		opener := func() (afero.File, error) {
+			return ffs.open(filename)
+
+		}
+
+		return decorateFileInfo(fi, ffs, opener, filename, "", meta), nil
+	}
+
+	ffs.decorate = decorator
+	return ffs
+}
+
+type baseFileDecoratorFs struct {
+	afero.Fs
+	decorate func(fi os.FileInfo, filename string) (os.FileInfo, error)
+}
+
+func (fs *baseFileDecoratorFs) Stat(name string) (os.FileInfo, error) {
+	fi, err := fs.Fs.Stat(name)
+	if err != nil {
+		return nil, err
+	}
+
+	return fs.decorate(fi, name)
+
+}
+
+func (fs *baseFileDecoratorFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
+	var (
+		fi  os.FileInfo
+		err error
+		ok  bool
+	)
+
+	if lstater, isLstater := fs.Fs.(afero.Lstater); isLstater {
+		fi, ok, err = lstater.LstatIfPossible(name)
+	} else {
+		fi, err = fs.Fs.Stat(name)
+	}
+
+	if err != nil {
+		return nil, false, err
+	}
+
+	fi, err = fs.decorate(fi, name)
+
+	return fi, ok, err
+}
+
+func (fs *baseFileDecoratorFs) Open(name string) (afero.File, error) {
+	return fs.open(name)
+}
+
+func (fs *baseFileDecoratorFs) open(name string) (afero.File, error) {
+	f, err := fs.Fs.Open(name)
+	if err != nil {
+		return nil, err
+	}
+	return &baseFileDecoratorFile{File: f, fs: fs}, nil
+}
+
+type baseFileDecoratorFile struct {
+	afero.File
+	fs *baseFileDecoratorFs
+}
+
+func (l *baseFileDecoratorFile) Readdir(c int) (ofi []os.FileInfo, err error) {
+	dirnames, err := l.File.Readdirnames(c)
+	if err != nil {
+		return nil, err
+	}
+
+	fisp := make([]os.FileInfo, 0, len(dirnames))
+
+	for _, dirname := range dirnames {
+		filename := dirname
+
+		if l.Name() != "" && l.Name() != filepathSeparator {
+			filename = filepath.Join(l.Name(), dirname)
+		}
+
+		// We need to resolve any symlink info.
+		fi, _, err := lstatIfPossible(l.fs.Fs, filename)
+		if err != nil {
+			if os.IsNotExist(err) {
+				continue
+			}
+			return nil, err
+		}
+		fi, err = l.fs.decorate(fi, filename)
+		if err != nil {
+			return nil, errors.Wrap(err, "decorate")
+		}
+		fisp = append(fisp, fi)
+	}
+
+	return fisp, err
+}

hugofs/fileinfo.go

@@ -0,0 +1,297 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package hugofs provides the file systems used by Hugo.
+package hugofs
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/gohugoio/hugo/hugofs/files"
+	"golang.org/x/text/unicode/norm"
+
+	"github.com/pkg/errors"
+
+	"github.com/spf13/cast"
+
+	"github.com/gohugoio/hugo/common/hreflect"
+
+	"github.com/spf13/afero"
+)
+
+const (
+	metaKeyFilename                   = "filename"
+	metaKeyOriginalFilename           = "originalFilename"
+	metaKeyName                       = "name"
+	metaKeyPath                       = "path"
+	metaKeyPathWalk                   = "pathWalk"
+	metaKeyLang                       = "lang"
+	metaKeyWeight                     = "weight"
+	metaKeyOrdinal                    = "ordinal"
+	metaKeyFs                         = "fs"
+	metaKeyOpener                     = "opener"
+	metaKeyIsOrdered                  = "isOrdered"
+	metaKeyIsSymlink                  = "isSymlink"
+	metaKeySkipDir                    = "skipDir"
+	metaKeyClassifier                 = "classifier"
+	metaKeyTranslationBaseName        = "translationBaseName"
+	metaKeyTranslationBaseNameWithExt = "translationBaseNameWithExt"
+	metaKeyTranslations               = "translations"
+	metaKeyDecoraterPath              = "decoratorPath"
+)
+
+type FileMeta map[string]interface{}
+
+func (f FileMeta) GetInt(key string) int {
+	return cast.ToInt(f[key])
+}
+
+func (f FileMeta) GetString(key string) string {
+	return cast.ToString(f[key])
+}
+
+func (f FileMeta) GetBool(key string) bool {
+	return cast.ToBool(f[key])
+}
+
+func (f FileMeta) Filename() string {
+	return f.stringV(metaKeyFilename)
+}
+
+func (f FileMeta) OriginalFilename() string {
+	return f.stringV(metaKeyOriginalFilename)
+}
+
+func (f FileMeta) SkipDir() bool {
+	return f.GetBool(metaKeySkipDir)
+}
+func (f FileMeta) TranslationBaseName() string {
+	return f.stringV(metaKeyTranslationBaseName)
+}
+
+func (f FileMeta) TranslationBaseNameWithExt() string {
+	return f.stringV(metaKeyTranslationBaseNameWithExt)
+}
+
+func (f FileMeta) Translations() []string {
+	return cast.ToStringSlice(f[metaKeyTranslations])
+}
+
+func (f FileMeta) Name() string {
+	return f.stringV(metaKeyName)
+}
+
+func (f FileMeta) Classifier() string {
+	c := f.stringV(metaKeyClassifier)
+	if c != "" {
+		return c
+	}
+
+	return files.ContentClassFile // For sorting
+}
+
+func (f FileMeta) Lang() string {
+	return f.stringV(metaKeyLang)
+}
+
+func (f FileMeta) Path() string {
+	return f.stringV(metaKeyPath)
+}
+
+func (f FileMeta) Weight() int {
+	return f.GetInt(metaKeyWeight)
+}
+
+func (f FileMeta) Ordinal() int {
+	return f.GetInt(metaKeyOrdinal)
+}
+
+func (f FileMeta) IsOrdered() bool {
+	return f.GetBool(metaKeyIsOrdered)
+}
+
+// IsSymlink returns whether this comes from a symlinked file or directory.
+func (f FileMeta) IsSymlink() bool {
+	return f.GetBool(metaKeyIsSymlink)
+}
+
+func (f FileMeta) Watch() bool {
+	if v, found := f["watch"]; found {
+		return v.(bool)
+	}
+	return false
+}
+
+func (f FileMeta) Fs() afero.Fs {
+	if v, found := f[metaKeyFs]; found {
+		return v.(afero.Fs)
+	}
+	return nil
+}
+
+func (f FileMeta) GetOpener() func() (afero.File, error) {
+	o, found := f[metaKeyOpener]
+	if !found {
+		return nil
+	}
+	return o.(func() (afero.File, error))
+}
+
+func (f FileMeta) Open() (afero.File, error) {
+	v, found := f[metaKeyOpener]
+	if !found {
+		return nil, errors.New("file opener not found")
+	}
+	return v.(func() (afero.File, error))()
+}
+
+func (f FileMeta) stringV(key string) string {
+	if v, found := f[key]; found {
+		return v.(string)
+	}
+	return ""
+}
+
+func (f FileMeta) setIfNotZero(key string, val interface{}) {
+	if !hreflect.IsTruthful(val) {
+		return
+	}
+	f[key] = val
+}
+
+type FileMetaInfo interface {
+	os.FileInfo
+	Meta() FileMeta
+}
+
+type fileInfoMeta struct {
+	os.FileInfo
+	m FileMeta
+}
+
+func (fi *fileInfoMeta) Meta() FileMeta {
+	return fi.m
+}
+
+func NewFileMetaInfo(fi os.FileInfo, m FileMeta) FileMetaInfo {
+
+	if fim, ok := fi.(FileMetaInfo); ok {
+		mergeFileMeta(fim.Meta(), m)
+	}
+	return &fileInfoMeta{FileInfo: fi, m: m}
+}
+
+// Merge metadata, last entry wins.
+func mergeFileMeta(from, to FileMeta) {
+	if from == nil {
+		return
+	}
+	for k, v := range from {
+		if _, found := to[k]; !found {
+			to[k] = v
+		}
+	}
+}
+
+type dirNameOnlyFileInfo struct {
+	name string
+}
+
+func (fi *dirNameOnlyFileInfo) Name() string {
+	return fi.name
+}
+
+func (fi *dirNameOnlyFileInfo) Size() int64 {
+	panic("not implemented")
+}
+
+func (fi *dirNameOnlyFileInfo) Mode() os.FileMode {
+	return os.ModeDir
+}
+
+func (fi *dirNameOnlyFileInfo) ModTime() time.Time {
+	return time.Time{}
+}
+
+func (fi *dirNameOnlyFileInfo) IsDir() bool {
+	return true
+}
+
+func (fi *dirNameOnlyFileInfo) Sys() interface{} {
+	return nil
+}
+
+func newDirNameOnlyFileInfo(name string, isOrdered bool, fileOpener func() (afero.File, error)) FileMetaInfo {
+	name = normalizeFilename(name)
+	_, base := filepath.Split(name)
+	return NewFileMetaInfo(&dirNameOnlyFileInfo{name: base}, FileMeta{
+		metaKeyFilename:  name,
+		metaKeyIsOrdered: isOrdered,
+		metaKeyOpener:    fileOpener})
+}
+
+func decorateFileInfo(
+	fi os.FileInfo,
+	fs afero.Fs, opener func() (afero.File, error),
+	filename, filepath string, inMeta FileMeta) FileMetaInfo {
+
+	var meta FileMeta
+	var fim FileMetaInfo
+
+	filepath = strings.TrimPrefix(filepath, filepathSeparator)
+
+	var ok bool
+	if fim, ok = fi.(FileMetaInfo); ok {
+		meta = fim.Meta()
+	} else {
+		meta = make(FileMeta)
+		fim = NewFileMetaInfo(fi, meta)
+	}
+
+	meta.setIfNotZero(metaKeyOpener, opener)
+	meta.setIfNotZero(metaKeyFs, fs)
+	meta.setIfNotZero(metaKeyPath, normalizeFilename(filepath))
+	meta.setIfNotZero(metaKeyFilename, normalizeFilename(filename))
+
+	mergeFileMeta(inMeta, meta)
+
+	return fim
+
+}
+
+func isSymlink(fi os.FileInfo) bool {
+	return fi != nil && fi.Mode()&os.ModeSymlink == os.ModeSymlink
+}
+
+func fileInfosToFileMetaInfos(fis []os.FileInfo) []FileMetaInfo {
+	fims := make([]FileMetaInfo, len(fis))
+	for i, v := range fis {
+		fims[i] = v.(FileMetaInfo)
+	}
+	return fims
+}
+
+func normalizeFilename(filename string) string {
+	if filename == "" {
+		return ""
+	}
+	if runtime.GOOS == "darwin" {
+		// When a file system is HFS+, its filepath is in NFD form.
+		return norm.NFC.String(filename)
+	}
+	return filename
+}

hugofs/files/classifier.go

@@ -0,0 +1,121 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package files
+
+import (
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+var (
+	// This should be the only list of valid extensions for content files.
+	contentFileExtensions = []string{
+		"html", "htm",
+		"mdown", "markdown", "md",
+		"asciidoc", "adoc", "ad",
+		"rest", "rst",
+		"mmark",
+		"org",
+		"pandoc", "pdc"}
+
+	contentFileExtensionsSet map[string]bool
+)
+
+func init() {
+	contentFileExtensionsSet = make(map[string]bool)
+	for _, ext := range contentFileExtensions {
+		contentFileExtensionsSet[ext] = true
+	}
+}
+
+func IsContentFile(filename string) bool {
+	return contentFileExtensionsSet[strings.TrimPrefix(filepath.Ext(filename), ".")]
+}
+
+func IsContentExt(ext string) bool {
+	return contentFileExtensionsSet[ext]
+}
+
+const (
+	ContentClassLeaf    = "leaf"
+	ContentClassBranch  = "branch"
+	ContentClassFile    = "zfile" // Sort below
+	ContentClassContent = "zcontent"
+)
+
+func ClassifyContentFile(filename string) string {
+	if !IsContentFile(filename) {
+		return ContentClassFile
+	}
+	if strings.HasPrefix(filename, "_index.") {
+		return ContentClassBranch
+	}
+
+	if strings.HasPrefix(filename, "index.") {
+		return ContentClassLeaf
+	}
+
+	return ContentClassContent
+}
+
+const (
+	ComponentFolderArchetypes = "archetypes"
+	ComponentFolderStatic     = "static"
+	ComponentFolderLayouts    = "layouts"
+	ComponentFolderContent    = "content"
+	ComponentFolderData       = "data"
+	ComponentFolderAssets     = "assets"
+	ComponentFolderI18n       = "i18n"
+
+	FolderResources = "resources"
+)
+
+var (
+	ComponentFolders = []string{
+		ComponentFolderArchetypes,
+		ComponentFolderStatic,
+		ComponentFolderLayouts,
+		ComponentFolderContent,
+		ComponentFolderData,
+		ComponentFolderAssets,
+		ComponentFolderI18n,
+	}
+
+	componentFoldersSet = make(map[string]bool)
+)
+
+func init() {
+	sort.Strings(ComponentFolders)
+	for _, f := range ComponentFolders {
+		componentFoldersSet[f] = true
+	}
+}
+
+// ResolveComponentFolder returns "content" from "content/blog/foo.md" etc.
+func ResolveComponentFolder(filename string) string {
+	filename = strings.TrimPrefix(filename, string(os.PathSeparator))
+	for _, cf := range ComponentFolders {
+		if strings.HasPrefix(filename, cf) {
+			return cf
+		}
+	}
+
+	return ""
+}
+
+func IsComponentFolder(name string) bool {
+	return componentFoldersSet[name]
+}

hugofs/files/classifier_test.go

@@ -0,0 +1,49 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package files
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestIsContentFile(t *testing.T) {
+	assert := require.New(t)
+
+	assert.True(IsContentFile(filepath.FromSlash("my/file.md")))
+	assert.True(IsContentFile(filepath.FromSlash("my/file.ad")))
+	assert.False(IsContentFile(filepath.FromSlash("textfile.txt")))
+	assert.True(IsContentExt("md"))
+	assert.False(IsContentExt("json"))
+}
+
+func TestComponentFolders(t *testing.T) {
+	assert := require.New(t)
+
+	// It's important that these are absolutely right and not changed.
+	assert.Equal(len(ComponentFolders), len(componentFoldersSet))
+	assert.True(IsComponentFolder("archetypes"))
+	assert.True(IsComponentFolder("layouts"))
+	assert.True(IsComponentFolder("data"))
+	assert.True(IsComponentFolder("i18n"))
+	assert.True(IsComponentFolder("assets"))
+	assert.False(IsComponentFolder("resources"))
+	assert.True(IsComponentFolder("static"))
+	assert.True(IsComponentFolder("content"))
+	assert.False(IsComponentFolder("foo"))
+	assert.False(IsComponentFolder(""))
+
+}

hugofs/filter_fs.go

@@ -0,0 +1,341 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/gohugoio/hugo/hugofs/files"
+
+	"github.com/spf13/afero"
+)
+
+var (
+	_ afero.Fs      = (*FilterFs)(nil)
+	_ afero.Lstater = (*FilterFs)(nil)
+	_ afero.File    = (*filterDir)(nil)
+)
+
+func NewLanguageFs(langs map[string]int, fs afero.Fs) (afero.Fs, error) {
+
+	applyMeta := func(fs *FilterFs, name string, fis []os.FileInfo) {
+
+		for i, fi := range fis {
+			if fi.IsDir() {
+				filename := filepath.Join(name, fi.Name())
+				fis[i] = decorateFileInfo(fi, fs, fs.getOpener(filename), "", "", nil)
+				continue
+			}
+
+			meta := fi.(FileMetaInfo).Meta()
+			lang := meta.Lang()
+
+			fileLang, translationBaseName, translationBaseNameWithExt := langInfoFrom(langs, fi.Name())
+			weight := 0
+
+			if fileLang != "" {
+				weight = 1
+				if fileLang == lang {
+					// Give priority to myfile.sv.txt inside the sv filesystem.
+					weight++
+				}
+				lang = fileLang
+			}
+
+			fim := NewFileMetaInfo(fi, FileMeta{
+				metaKeyLang:                       lang,
+				metaKeyWeight:                     weight,
+				metaKeyOrdinal:                    langs[lang],
+				metaKeyTranslationBaseName:        translationBaseName,
+				metaKeyTranslationBaseNameWithExt: translationBaseNameWithExt,
+				metaKeyClassifier:                 files.ClassifyContentFile(fi.Name()),
+			})
+
+			fis[i] = fim
+		}
+	}
+
+	all := func(fis []os.FileInfo) {
+		// Maps translation base name to a list of language codes.
+		translations := make(map[string][]string)
+		trackTranslation := func(meta FileMeta) {
+			name := meta.TranslationBaseNameWithExt()
+			translations[name] = append(translations[name], meta.Lang())
+		}
+		for _, fi := range fis {
+			if fi.IsDir() {
+				continue
+			}
+			meta := fi.(FileMetaInfo).Meta()
+
+			trackTranslation(meta)
+
+		}
+
+		for _, fi := range fis {
+			fim := fi.(FileMetaInfo)
+			langs := translations[fim.Meta().TranslationBaseNameWithExt()]
+			if len(langs) > 0 {
+				fim.Meta()["translations"] = sortAndremoveStringDuplicates(langs)
+			}
+		}
+	}
+
+	return &FilterFs{
+		fs:             fs,
+		applyPerSource: applyMeta,
+		applyAll:       all,
+	}, nil
+
+}
+
+func NewFilterFs(fs afero.Fs) (afero.Fs, error) {
+
+	applyMeta := func(fs *FilterFs, name string, fis []os.FileInfo) {
+		for i, fi := range fis {
+			if fi.IsDir() {
+				fis[i] = decorateFileInfo(fi, fs, fs.getOpener(fi.(FileMetaInfo).Meta().Filename()), "", "", nil)
+			}
+		}
+	}
+
+	ffs := &FilterFs{
+		fs:             fs,
+		applyPerSource: applyMeta,
+	}
+
+	return ffs, nil
+
+}
+
+// FilterFs is an ordered composite filesystem.
+type FilterFs struct {
+	fs afero.Fs
+
+	applyPerSource func(fs *FilterFs, name string, fis []os.FileInfo)
+	applyAll       func(fis []os.FileInfo)
+}
+
+func (fs *FilterFs) Chmod(n string, m os.FileMode) error {
+	return syscall.EPERM
+}
+
+func (fs *FilterFs) Chtimes(n string, a, m time.Time) error {
+	return syscall.EPERM
+}
+
+func (fs *FilterFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
+	fi, b, err := lstatIfPossible(fs.fs, name)
+
+	if err != nil {
+		return nil, false, err
+	}
+
+	if fi.IsDir() {
+		return decorateFileInfo(fi, fs, fs.getOpener(name), "", "", nil), false, nil
+	}
+
+	fs.applyFilters(name, -1, fi)
+
+	return fi, b, nil
+
+}
+
+func (fs *FilterFs) Mkdir(n string, p os.FileMode) error {
+	return syscall.EPERM
+}
+
+func (fs *FilterFs) MkdirAll(n string, p os.FileMode) error {
+	return syscall.EPERM
+}
+
+func (fs *FilterFs) Name() string {
+	return "WeightedFileSystem"
+}
+
+func (fs *FilterFs) Open(name string) (afero.File, error) {
+	f, err := fs.fs.Open(name)
+	if err != nil {
+		return nil, err
+	}
+
+	return &filterDir{
+		File: f,
+		ffs:  fs,
+	}, nil
+
+}
+
+func (fs *FilterFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
+	panic("not implemented")
+}
+
+func (fs *FilterFs) ReadDir(name string) ([]os.FileInfo, error) {
+	panic("not implemented")
+}
+
+func (fs *FilterFs) Remove(n string) error {
+	return syscall.EPERM
+}
+
+func (fs *FilterFs) RemoveAll(p string) error {
+	return syscall.EPERM
+}
+
+func (fs *FilterFs) Rename(o, n string) error {
+	return syscall.EPERM
+}
+
+func (fs *FilterFs) Stat(name string) (os.FileInfo, error) {
+	fi, _, err := fs.LstatIfPossible(name)
+	return fi, err
+}
+
+func (fs *FilterFs) Create(n string) (afero.File, error) {
+	return nil, syscall.EPERM
+}
+
+func (fs *FilterFs) getOpener(name string) func() (afero.File, error) {
+	return func() (afero.File, error) {
+		return fs.Open(name)
+	}
+}
+
+func (fs *FilterFs) applyFilters(name string, count int, fis ...os.FileInfo) ([]os.FileInfo, error) {
+	if fs.applyPerSource != nil {
+		fs.applyPerSource(fs, name, fis)
+	}
+
+	seen := make(map[string]bool)
+	var duplicates []int
+	for i, dir := range fis {
+		if !dir.IsDir() {
+			continue
+		}
+		if seen[dir.Name()] {
+			duplicates = append(duplicates, i)
+		} else {
+			seen[dir.Name()] = true
+		}
+	}
+
+	// Remove duplicate directories, keep first.
+	if len(duplicates) > 0 {
+		for i := len(duplicates) - 1; i >= 0; i-- {
+			idx := duplicates[i]
+			fis = append(fis[:idx], fis[idx+1:]...)
+		}
+	}
+
+	if fs.applyAll != nil {
+		fs.applyAll(fis)
+	}
+
+	if count > 0 && len(fis) >= count {
+		return fis[:count], nil
+	}
+
+	return fis, nil
+
+}
+
+type filterDir struct {
+	afero.File
+	ffs *FilterFs
+}
+
+func (f *filterDir) Readdir(count int) ([]os.FileInfo, error) {
+	fis, err := f.File.Readdir(-1)
+	if err != nil {
+		return nil, err
+	}
+	return f.ffs.applyFilters(f.Name(), count, fis...)
+}
+
+func (f *filterDir) Readdirnames(count int) ([]string, error) {
+	dirsi, err := f.Readdir(count)
+	if err != nil {
+		return nil, err
+	}
+
+	dirs := make([]string, len(dirsi))
+	for i, d := range dirsi {
+		dirs[i] = d.Name()
+	}
+	return dirs, nil
+}
+
+// Try to extract the language from the given filename.
+// Any valid language identificator in the name will win over the
+// language set on the file system, e.g. "mypost.en.md".
+func langInfoFrom(languages map[string]int, name string) (string, string, string) {
+	var lang string
+
+	baseName := filepath.Base(name)
+	ext := filepath.Ext(baseName)
+	translationBaseName := baseName
+
+	if ext != "" {
+		translationBaseName = strings.TrimSuffix(translationBaseName, ext)
+	}
+
+	fileLangExt := filepath.Ext(translationBaseName)
+	fileLang := strings.TrimPrefix(fileLangExt, ".")
+
+	if _, found := languages[fileLang]; found {
+		lang = fileLang
+		translationBaseName = strings.TrimSuffix(translationBaseName, fileLangExt)
+	}
+
+	translationBaseNameWithExt := translationBaseName
+
+	if ext != "" {
+		translationBaseNameWithExt += ext
+	}
+
+	return lang, translationBaseName, translationBaseNameWithExt
+
+}
+
+func printFs(fs afero.Fs, path string, w io.Writer) {
+	if fs == nil {
+		return
+	}
+	afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {
+		fmt.Println("p:::", path)
+		return nil
+	})
+}
+
+func sortAndremoveStringDuplicates(s []string) []string {
+	ss := sort.StringSlice(s)
+	ss.Sort()
+	i := 0
+	for j := 1; j < len(s); j++ {
+		if !ss.Less(i, j) {
+			continue
+		}
+		i++
+		s[i] = s[j]
+	}
+
+	return s[:i+1]
+}

hugofs/filter_fs_test.go

@@ -0,0 +1,48 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestLangInfoFrom(t *testing.T) {
+
+	langs := map[string]int{
+		"sv": 10,
+		"en": 20,
+	}
+
+	assert := require.New(t)
+
+	tests := []struct {
+		input    string
+		expected []string
+	}{
+		{"page.sv.md", []string{"sv", "page", "page.md"}},
+		{"page.en.md", []string{"en", "page", "page.md"}},
+		{"page.no.md", []string{"", "page.no", "page.no.md"}},
+		{filepath.FromSlash("tc-lib-color/class-Com.Tecnick.Color.Css"), []string{"", "class-Com.Tecnick.Color", "class-Com.Tecnick.Color.Css"}},
+		{filepath.FromSlash("class-Com.Tecnick.Color.sv.Css"), []string{"sv", "class-Com.Tecnick.Color", "class-Com.Tecnick.Color.Css"}},
+	}
+
+	for _, test := range tests {
+		v1, v2, v3 := langInfoFrom(langs, test.input)
+		assert.Equal(test.expected, []string{v1, v2, v3})
+	}
+
+}

hugofs/fs.go

@@ -21,8 +21,10 @@ import (
 	"github.com/spf13/afero"
 )
 
-// Os points to an Os Afero file system.
-var Os = &afero.OsFs{}
+var (
+	// Os points to the (real) Os filesystem.
+	Os = &afero.OsFs{}
+)
 
 // Fs abstracts the file system to separate source and destination file systems
 // and allows both to be mocked for testing.

hugofs/language_composite_fs.go

@@ -14,6 +14,9 @@
 package hugofs
 
 import (
+	"os"
+	"path"
+
 	"github.com/spf13/afero"
 )
 
@@ -30,8 +33,8 @@ type languageCompositeFs struct {
 // This is a hybrid filesystem. To get a specific file in Open, Stat etc., use the full filename
 // to the target filesystem. This information is available in Readdir, Stat etc. via the
 // special LanguageFileInfo FileInfo implementation.
-func NewLanguageCompositeFs(base afero.Fs, overlay *LanguageFs) afero.Fs {
-	return afero.NewReadOnlyFs(&languageCompositeFs{afero.NewCopyOnWriteFs(base, overlay).(*afero.CopyOnWriteFs)})
+func NewLanguageCompositeFs(base, overlay afero.Fs) afero.Fs {
+	return &languageCompositeFs{afero.NewCopyOnWriteFs(base, overlay).(*afero.CopyOnWriteFs)}
 }
 
 // Open takes the full path to the file in the target filesystem. If it is a directory, it gets merged
@@ -49,3 +52,36 @@ func (fs *languageCompositeFs) Open(name string) (afero.File, error) {
 	}
 	return f, nil
 }
+
+// LanguageDirsMerger implements the afero.DirsMerger interface, which is used
+// to merge two directories.
+var LanguageDirsMerger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) {
+	m := make(map[string]FileMetaInfo)
+
+	getKey := func(fim FileMetaInfo) string {
+		return path.Join(fim.Meta().Lang(), fim.Name())
+	}
+
+	for _, fi := range lofi {
+		fim := fi.(FileMetaInfo)
+		m[getKey(fim)] = fim
+	}
+
+	for _, fi := range bofi {
+		fim := fi.(FileMetaInfo)
+		key := getKey(fim)
+		_, found := m[key]
+		if !found {
+			m[key] = fim
+		}
+	}
+
+	merged := make([]os.FileInfo, len(m))
+	i := 0
+	for _, v := range m {
+		merged[i] = v
+		i++
+	}
+
+	return merged, nil
+}

hugofs/language_composite_fs_test.go

@@ -1,107 +0,0 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hugofs
-
-import (
-	"path/filepath"
-
-	"strings"
-
-	"testing"
-
-	"github.com/spf13/afero"
-	"github.com/stretchr/testify/require"
-)
-
-func TestCompositeLanguagFsTest(t *testing.T) {
-	assert := require.New(t)
-
-	languages := map[string]bool{
-		"sv": true,
-		"en": true,
-		"nn": true,
-	}
-	msv := afero.NewMemMapFs()
-	baseSv := "/content/sv"
-	lfssv := NewLanguageFs("sv", languages, afero.NewBasePathFs(msv, baseSv))
-	mnn := afero.NewMemMapFs()
-	baseNn := "/content/nn"
-	lfsnn := NewLanguageFs("nn", languages, afero.NewBasePathFs(mnn, baseNn))
-	men := afero.NewMemMapFs()
-	baseEn := "/content/en"
-	lfsen := NewLanguageFs("en", languages, afero.NewBasePathFs(men, baseEn))
-
-	// The order will be sv, en, nn
-	composite := NewLanguageCompositeFs(lfsnn, lfsen)
-	composite = NewLanguageCompositeFs(composite, lfssv)
-
-	afero.WriteFile(msv, filepath.Join(baseSv, "f1.txt"), []byte("some sv"), 0755)
-	afero.WriteFile(mnn, filepath.Join(baseNn, "f1.txt"), []byte("some nn"), 0755)
-	afero.WriteFile(men, filepath.Join(baseEn, "f1.txt"), []byte("some en"), 0755)
-
-	// Swedish is the top layer.
-	assertLangFile(t, composite, "f1.txt", "sv")
-
-	afero.WriteFile(msv, filepath.Join(baseSv, "f2.en.txt"), []byte("some sv"), 0755)
-	afero.WriteFile(mnn, filepath.Join(baseNn, "f2.en.txt"), []byte("some nn"), 0755)
-	afero.WriteFile(men, filepath.Join(baseEn, "f2.en.txt"), []byte("some en"), 0755)
-
-	// English is in the middle, but the most specific language match wins.
-	//assertLangFile(t, composite, "f2.en.txt", "en")
-
-	// Fetch some specific language versions
-	assertLangFile(t, composite, filepath.Join(baseNn, "f2.en.txt"), "nn")
-	assertLangFile(t, composite, filepath.Join(baseEn, "f2.en.txt"), "en")
-	assertLangFile(t, composite, filepath.Join(baseSv, "f2.en.txt"), "sv")
-
-	// Read the root
-	f, err := composite.Open("/")
-	assert.NoError(err)
-	defer f.Close()
-	files, err := f.Readdir(-1)
-	assert.NoError(err)
-	assert.Equal(4, len(files))
-	expected := map[string]bool{
-		filepath.FromSlash("/content/en/f1.txt"):    true,
-		filepath.FromSlash("/content/nn/f1.txt"):    true,
-		filepath.FromSlash("/content/sv/f1.txt"):    true,
-		filepath.FromSlash("/content/en/f2.en.txt"): true,
-	}
-	got := make(map[string]bool)
-
-	for _, fi := range files {
-		fil, ok := fi.(*LanguageFileInfo)
-		assert.True(ok)
-		got[fil.Filename()] = true
-	}
-	assert.Equal(expected, got)
-}
-
-func assertLangFile(t testing.TB, fs afero.Fs, filename, match string) {
-	f, err := fs.Open(filename)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer f.Close()
-	b, err := afero.ReadAll(f)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	s := string(b)
-	if !strings.Contains(s, match) {
-		t.Fatalf("got %q expected it to contain %q", s, match)
-
-	}
-}

hugofs/language_fs.go

@@ -1,346 +0,0 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hugofs
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/spf13/afero"
-)
-
-const hugoFsMarker = "__hugofs"
-
-var (
-	_ LanguageAnnouncer = (*LanguageFileInfo)(nil)
-	_ FilePather        = (*LanguageFileInfo)(nil)
-	_ afero.Lstater     = (*LanguageFs)(nil)
-)
-
-// LanguageAnnouncer is aware of its language.
-type LanguageAnnouncer interface {
-	Lang() string
-	TranslationBaseName() string
-}
-
-// FilePather is aware of its file's location.
-type FilePather interface {
-	// Filename gets the full path and filename to the file.
-	Filename() string
-
-	// Path gets the content relative path including file name and extension.
-	// The directory is relative to the content root where "content" is a broad term.
-	Path() string
-
-	// RealName is FileInfo.Name in its original form.
-	RealName() string
-
-	BaseDir() string
-}
-
-// LanguageDirsMerger implements the afero.DirsMerger interface, which is used
-// to merge two directories.
-var LanguageDirsMerger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) {
-	m := make(map[string]*LanguageFileInfo)
-
-	for _, fi := range lofi {
-		fil, ok := fi.(*LanguageFileInfo)
-		if !ok {
-			return nil, fmt.Errorf("received %T, expected *LanguageFileInfo", fi)
-		}
-		m[fil.virtualName] = fil
-	}
-
-	for _, fi := range bofi {
-		fil, ok := fi.(*LanguageFileInfo)
-		if !ok {
-			return nil, fmt.Errorf("received %T, expected *LanguageFileInfo", fi)
-		}
-		existing, found := m[fil.virtualName]
-
-		if !found || existing.weight < fil.weight {
-			m[fil.virtualName] = fil
-		}
-	}
-
-	merged := make([]os.FileInfo, len(m))
-	i := 0
-	for _, v := range m {
-		merged[i] = v
-		i++
-	}
-
-	return merged, nil
-}
-
-// LanguageFileInfo is a super-set of os.FileInfo with additional information
-// about the file in relation to its Hugo language.
-type LanguageFileInfo struct {
-	os.FileInfo
-	lang                string
-	baseDir             string
-	realFilename        string
-	relFilename         string
-	name                string
-	realName            string
-	virtualName         string
-	translationBaseName string
-
-	// We add some weight to the files in their own language's content directory.
-	weight int
-}
-
-// Filename returns a file's real filename including the base (ie.
-// "/my/base/sect/page.md").
-func (fi *LanguageFileInfo) Filename() string {
-	return fi.realFilename
-}
-
-// Path returns a file's filename relative to the base (ie. "sect/page.md").
-func (fi *LanguageFileInfo) Path() string {
-	return fi.relFilename
-}
-
-// RealName returns a file's real base name (ie. "page.md").
-func (fi *LanguageFileInfo) RealName() string {
-	return fi.realName
-}
-
-// BaseDir returns a file's base directory (ie. "/my/base").
-func (fi *LanguageFileInfo) BaseDir() string {
-	return fi.baseDir
-}
-
-// Lang returns a file's language (ie. "sv").
-func (fi *LanguageFileInfo) Lang() string {
-	return fi.lang
-}
-
-// TranslationBaseName returns the base filename without any extension or language
-// identifiers (ie. "page").
-func (fi *LanguageFileInfo) TranslationBaseName() string {
-	return fi.translationBaseName
-}
-
-// Name is the name of the file within this filesystem without any path info.
-// It will be marked with language information so we can identify it as ours
-// (ie. "__hugofs_sv_page.md").
-func (fi *LanguageFileInfo) Name() string {
-	return fi.name
-}
-
-type languageFile struct {
-	afero.File
-	fs *LanguageFs
-}
-
-// Readdir creates FileInfo entries by calling Lstat if possible.
-func (l *languageFile) Readdir(c int) (ofi []os.FileInfo, err error) {
-	names, err := l.File.Readdirnames(c)
-	if err != nil {
-		return nil, err
-	}
-
-	fis := make([]os.FileInfo, len(names))
-
-	for i, name := range names {
-		fi, _, err := l.fs.LstatIfPossible(filepath.Join(l.Name(), name))
-
-		if err != nil {
-			return nil, err
-		}
-		fis[i] = fi
-	}
-
-	return fis, err
-}
-
-// LanguageFs represents a language filesystem.
-type LanguageFs struct {
-	// This Fs is usually created with a BasePathFs
-	basePath   string
-	lang       string
-	nameMarker string
-	languages  map[string]bool
-	afero.Fs
-}
-
-// NewLanguageFs creates a new language filesystem.
-func NewLanguageFs(lang string, languages map[string]bool, fs afero.Fs) *LanguageFs {
-	if lang == "" {
-		panic("no lang set for the language fs")
-	}
-	var basePath string
-
-	if bfs, ok := fs.(*afero.BasePathFs); ok {
-		basePath, _ = bfs.RealPath("")
-	}
-
-	marker := hugoFsMarker + "_" + lang + "_"
-
-	return &LanguageFs{lang: lang, languages: languages, basePath: basePath, Fs: fs, nameMarker: marker}
-}
-
-// Lang returns a language filesystem's language (ie. "sv").
-func (fs *LanguageFs) Lang() string {
-	return fs.lang
-}
-
-// Stat returns the os.FileInfo of a given file.
-func (fs *LanguageFs) Stat(name string) (os.FileInfo, error) {
-	name, err := fs.realName(name)
-	if err != nil {
-		return nil, err
-	}
-
-	fi, err := fs.Fs.Stat(name)
-	if err != nil {
-		return nil, err
-	}
-
-	return fs.newLanguageFileInfo(name, fi)
-}
-
-// Open opens the named file for reading.
-func (fs *LanguageFs) Open(name string) (afero.File, error) {
-	name, err := fs.realName(name)
-	if err != nil {
-		return nil, err
-	}
-	f, err := fs.Fs.Open(name)
-
-	if err != nil {
-		return nil, err
-	}
-	return &languageFile{File: f, fs: fs}, nil
-}
-
-// LstatIfPossible returns the os.FileInfo structure describing a given file.
-// It attempts to use Lstat if supported or defers to the os.  In addition to
-// the FileInfo, a boolean is returned telling whether Lstat was called.
-func (fs *LanguageFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
-	name, err := fs.realName(name)
-	if err != nil {
-		return nil, false, err
-	}
-
-	var fi os.FileInfo
-	var b bool
-
-	if lif, ok := fs.Fs.(afero.Lstater); ok {
-		fi, b, err = lif.LstatIfPossible(name)
-	} else {
-		fi, err = fs.Fs.Stat(name)
-	}
-
-	if err != nil {
-		return nil, b, err
-	}
-
-	lfi, err := fs.newLanguageFileInfo(name, fi)
-
-	return lfi, b, err
-}
-
-func (fs *LanguageFs) realPath(name string) (string, error) {
-	if baseFs, ok := fs.Fs.(*afero.BasePathFs); ok {
-		return baseFs.RealPath(name)
-	}
-	return name, nil
-}
-
-func (fs *LanguageFs) realName(name string) (string, error) {
-	if strings.Contains(name, hugoFsMarker) {
-		if !strings.Contains(name, fs.nameMarker) {
-			return "", os.ErrNotExist
-		}
-		return strings.Replace(name, fs.nameMarker, "", 1), nil
-	}
-
-	if fs.basePath == "" {
-		return name, nil
-	}
-
-	return strings.TrimPrefix(name, fs.basePath), nil
-}
-
-func (fs *LanguageFs) newLanguageFileInfo(filename string, fi os.FileInfo) (*LanguageFileInfo, error) {
-	filename = filepath.Clean(filename)
-	_, name := filepath.Split(filename)
-
-	realName := name
-	virtualName := name
-
-	realPath, err := fs.realPath(filename)
-	if err != nil {
-		return nil, err
-	}
-
-	lang := fs.Lang()
-
-	baseNameNoExt := ""
-
-	if !fi.IsDir() {
-
-		// Try to extract the language from the file name.
-		// Any valid language identificator in the name will win over the
-		// language set on the file system, e.g. "mypost.en.md".
-		baseName := filepath.Base(name)
-		ext := filepath.Ext(baseName)
-		baseNameNoExt = baseName
-
-		if ext != "" {
-			baseNameNoExt = strings.TrimSuffix(baseNameNoExt, ext)
-		}
-
-		fileLangExt := filepath.Ext(baseNameNoExt)
-		fileLang := strings.TrimPrefix(fileLangExt, ".")
-
-		if fs.languages[fileLang] {
-			lang = fileLang
-			baseNameNoExt = strings.TrimSuffix(baseNameNoExt, fileLangExt)
-		}
-
-		// This connects the filename to the filesystem, not the language.
-		virtualName = baseNameNoExt + "." + lang + ext
-
-		name = fs.nameMarker + name
-	}
-
-	weight := 1
-	// If this file's language belongs in this directory, add some weight to it
-	// to make it more important.
-	if lang == fs.Lang() {
-		weight = 2
-	}
-
-	if fi.IsDir() {
-		// For directories we always want to start from the union view.
-		realPath = strings.TrimPrefix(realPath, fs.basePath)
-	}
-
-	return &LanguageFileInfo{
-		lang:                lang,
-		weight:              weight,
-		realFilename:        realPath,
-		realName:            realName,
-		relFilename:         strings.TrimPrefix(strings.TrimPrefix(realPath, fs.basePath), string(os.PathSeparator)),
-		name:                name,
-		virtualName:         virtualName,
-		translationBaseName: baseNameNoExt,
-		baseDir:             fs.basePath,
-		FileInfo:            fi}, nil
-}

hugofs/language_fs_test.go

@@ -1,100 +0,0 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hugofs
-
-import (
-	"path/filepath"
-	"testing"
-
-	"github.com/spf13/afero"
-	"github.com/stretchr/testify/require"
-)
-
-func TestLanguagFs(t *testing.T) {
-	languages := map[string]bool{
-		"sv": true,
-	}
-	base := filepath.FromSlash("/my/base")
-	assert := require.New(t)
-	m := afero.NewMemMapFs()
-	bfs := afero.NewBasePathFs(m, base)
-	lfs := NewLanguageFs("sv", languages, bfs)
-	assert.NotNil(lfs)
-	assert.Equal("sv", lfs.Lang())
-	err := afero.WriteFile(lfs, filepath.FromSlash("sect/page.md"), []byte("abc"), 0777)
-	assert.NoError(err)
-	fi, err := lfs.Stat(filepath.FromSlash("sect/page.md"))
-	assert.NoError(err)
-	assert.Equal("__hugofs_sv_page.md", fi.Name())
-
-	languager, ok := fi.(LanguageAnnouncer)
-	assert.True(ok)
-
-	assert.Equal("sv", languager.Lang())
-
-	lfi, ok := fi.(*LanguageFileInfo)
-	assert.True(ok)
-	assert.Equal(filepath.FromSlash("/my/base/sect/page.md"), lfi.Filename())
-	assert.Equal(filepath.FromSlash("sect/page.md"), lfi.Path())
-	assert.Equal("page.sv.md", lfi.virtualName)
-	assert.Equal("__hugofs_sv_page.md", lfi.Name())
-	assert.Equal("page.md", lfi.RealName())
-	assert.Equal(filepath.FromSlash("/my/base"), lfi.BaseDir())
-	assert.Equal("sv", lfi.Lang())
-	assert.Equal("page", lfi.TranslationBaseName())
-}
-
-// Issue 4559
-func TestFilenamesHandling(t *testing.T) {
-	languages := map[string]bool{
-		"sv": true,
-	}
-	base := filepath.FromSlash("/my/base")
-	assert := require.New(t)
-	m := afero.NewMemMapFs()
-	bfs := afero.NewBasePathFs(m, base)
-	lfs := NewLanguageFs("sv", languages, bfs)
-	assert.NotNil(lfs)
-	assert.Equal("sv", lfs.Lang())
-
-	for _, test := range []struct {
-		filename string
-		check    func(fi *LanguageFileInfo)
-	}{
-		{"tc-lib-color/class-Com.Tecnick.Color.Css", func(fi *LanguageFileInfo) {
-			assert.Equal("class-Com.Tecnick.Color", fi.TranslationBaseName())
-			assert.Equal(filepath.FromSlash("/my/base"), fi.BaseDir())
-			assert.Equal(filepath.FromSlash("tc-lib-color/class-Com.Tecnick.Color.Css"), fi.Path())
-			assert.Equal("class-Com.Tecnick.Color.Css", fi.RealName())
-			assert.Equal(filepath.FromSlash("/my/base/tc-lib-color/class-Com.Tecnick.Color.Css"), fi.Filename())
-		}},
-		{"tc-lib-color/class-Com.Tecnick.Color.sv.Css", func(fi *LanguageFileInfo) {
-			assert.Equal("class-Com.Tecnick.Color", fi.TranslationBaseName())
-			assert.Equal("class-Com.Tecnick.Color.sv.Css", fi.RealName())
-			assert.Equal(filepath.FromSlash("/my/base/tc-lib-color/class-Com.Tecnick.Color.sv.Css"), fi.Filename())
-		}},
-	} {
-		err := afero.WriteFile(lfs, filepath.FromSlash(test.filename), []byte("abc"), 0777)
-		assert.NoError(err)
-		fi, err := lfs.Stat(filepath.FromSlash(test.filename))
-		assert.NoError(err)
-
-		lfi, ok := fi.(*LanguageFileInfo)
-		assert.True(ok)
-		assert.Equal("sv", lfi.Lang())
-		test.check(lfi)
-
-	}
-
-}

hugofs/nolstat_fs.go

@@ -1,39 +0,0 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hugofs
-
-import (
-	"os"
-
-	"github.com/spf13/afero"
-)
-
-var (
-	_ afero.Fs = (*noLstatFs)(nil)
-)
-
-type noLstatFs struct {
-	afero.Fs
-}
-
-// NewNoLstatFs creates a new filesystem with no Lstat support.
-func NewNoLstatFs(fs afero.Fs) afero.Fs {
-	return &noLstatFs{Fs: fs}
-}
-
-// LstatIfPossible always delegates to Stat.
-func (fs *noLstatFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
-	fi, err := fs.Stat(name)
-	return fi, false, err
-}

hugofs/nosymlink_fs.go

@@ -0,0 +1,85 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"errors"
+	"os"
+
+	"github.com/spf13/afero"
+)
+
+var (
+	ErrPermissionSymlink = errors.New("symlinks not allowed in this filesystem")
+)
+
+func NewNoSymlinkFs(fs afero.Fs) afero.Fs {
+	return &noSymlinkFs{Fs: fs}
+}
+
+// noSymlinkFs is a filesystem that prevents symlinking.
+type noSymlinkFs struct {
+	afero.Fs
+}
+
+func (fs *noSymlinkFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
+	return fs.stat(name)
+}
+
+func (fs *noSymlinkFs) Stat(name string) (os.FileInfo, error) {
+	fi, _, err := fs.stat(name)
+	return fi, err
+}
+
+func (fs *noSymlinkFs) stat(name string) (os.FileInfo, bool, error) {
+
+	var (
+		fi       os.FileInfo
+		wasLstat bool
+		err      error
+	)
+
+	if lstater, ok := fs.Fs.(afero.Lstater); ok {
+		fi, wasLstat, err = lstater.LstatIfPossible(name)
+	} else {
+
+		fi, err = fs.Fs.Stat(name)
+	}
+
+	var metaIsSymlink bool
+
+	if fim, ok := fi.(FileMetaInfo); ok {
+		metaIsSymlink = fim.Meta().IsSymlink()
+	}
+
+	if metaIsSymlink || isSymlink(fi) {
+		return nil, wasLstat, ErrPermissionSymlink
+	}
+
+	return fi, wasLstat, err
+}
+
+func (fs *noSymlinkFs) Open(name string) (afero.File, error) {
+	if _, _, err := fs.stat(name); err != nil {
+		return nil, err
+	}
+	return fs.Fs.Open(name)
+}
+
+func (fs *noSymlinkFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
+	if _, _, err := fs.stat(name); err != nil {
+		return nil, err
+	}
+	return fs.Fs.OpenFile(name, flag, perm)
+}

hugofs/nosymlink_test.go

@@ -0,0 +1,97 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/gohugoio/hugo/htesting"
+
+	"github.com/spf13/afero"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNoSymlinkFs(t *testing.T) {
+	if skipSymlink() {
+		t.Skip("Skip; os.Symlink needs administrator rights on Windows")
+	}
+	assert := require.New(t)
+	workDir, clean, err := htesting.CreateTempDir(Os, "hugo-nosymlink")
+	assert.NoError(err)
+	defer clean()
+	wd, _ := os.Getwd()
+	defer func() {
+		os.Chdir(wd)
+	}()
+
+	blogDir := filepath.Join(workDir, "blog")
+	blogFile := filepath.Join(blogDir, "a.txt")
+	assert.NoError(os.MkdirAll(blogDir, 0777))
+	afero.WriteFile(Os, filepath.Join(blogFile), []byte("content"), 0777)
+	os.Chdir(workDir)
+	assert.NoError(os.Symlink("blog", "symlinkdedir"))
+	os.Chdir(blogDir)
+	assert.NoError(os.Symlink("a.txt", "symlinkdedfile.txt"))
+
+	fs := NewNoSymlinkFs(Os)
+	ls := fs.(afero.Lstater)
+	symlinkedDir := filepath.Join(workDir, "symlinkdedir")
+	symlinkedFile := filepath.Join(blogDir, "symlinkdedfile.txt")
+
+	// Check Stat and Lstat
+	for _, stat := range []func(name string) (os.FileInfo, error){
+		func(name string) (os.FileInfo, error) {
+			return fs.Stat(name)
+		},
+		func(name string) (os.FileInfo, error) {
+			fi, _, err := ls.LstatIfPossible(name)
+			return fi, err
+		},
+	} {
+		_, err = stat(symlinkedDir)
+		assert.Equal(ErrPermissionSymlink, err)
+		_, err = stat(symlinkedFile)
+		assert.Equal(ErrPermissionSymlink, err)
+
+		fi, err := stat(filepath.Join(workDir, "blog"))
+		assert.NoError(err)
+		assert.NotNil(fi)
+
+		fi, err = stat(blogFile)
+		assert.NoError(err)
+		assert.NotNil(fi)
+	}
+
+	// Check Open
+	_, err = fs.Open(symlinkedDir)
+	assert.Equal(ErrPermissionSymlink, err)
+	_, err = fs.OpenFile(symlinkedDir, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
+	assert.Equal(ErrPermissionSymlink, err)
+	_, err = fs.OpenFile(symlinkedFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
+	assert.Equal(ErrPermissionSymlink, err)
+	_, err = fs.Open(symlinkedFile)
+	assert.Equal(ErrPermissionSymlink, err)
+	f, err := fs.Open(blogDir)
+	assert.NoError(err)
+	f.Close()
+	f, err = fs.Open(blogFile)
+	assert.NoError(err)
+	f.Close()
+
+	// os.OpenFile(logFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
+
+}

hugofs/rootmapping_fs.go

@@ -1,4 +1,4 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,10 +14,14 @@
 package hugofs
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
+
+	"github.com/gohugoio/hugo/hugofs/files"
+
+	"github.com/pkg/errors"
 
 	radix "github.com/hashicorp/go-immutable-radix"
 	"github.com/spf13/afero"
@@ -25,90 +29,235 @@ import (
 
 var filepathSeparator = string(filepath.Separator)
 
+// NewRootMappingFs creates a new RootMappingFs on top of the provided with
+// of root mappings with some optional metadata about the root.
+// Note that From represents a virtual root that maps to the actual filename in To.
+func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {
+	rootMapToReal := radix.New().Txn()
+
+	for _, rm := range rms {
+		(&rm).clean()
+
+		fromBase := files.ResolveComponentFolder(rm.From)
+		if fromBase == "" {
+			panic("unrecognised component folder in" + rm.From)
+		}
+
+		if len(rm.To) < 2 {
+			panic(fmt.Sprintf("invalid root mapping; from/to: %s/%s", rm.From, rm.To))
+		}
+
+		_, err := fs.Stat(rm.To)
+		if err != nil {
+			if os.IsNotExist(err) {
+				continue
+			}
+			return nil, err
+		}
+
+		// Extract "blog" from "content/blog"
+		rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator)
+
+		key := []byte(rm.rootKey())
+		var mappings []RootMapping
+		v, found := rootMapToReal.Get(key)
+		if found {
+			// There may be more than one language pointing to the same root.
+			mappings = v.([]RootMapping)
+		}
+		mappings = append(mappings, rm)
+		rootMapToReal.Insert(key, mappings)
+	}
+
+	rfs := &RootMappingFs{Fs: fs,
+		virtualRoots:  rms,
+		rootMapToReal: rootMapToReal.Commit().Root()}
+
+	return rfs, nil
+}
+
+// NewRootMappingFsFromFromTo is a convenicence variant of NewRootMappingFs taking
+// From and To as string pairs.
+func NewRootMappingFsFromFromTo(fs afero.Fs, fromTo ...string) (*RootMappingFs, error) {
+	rms := make([]RootMapping, len(fromTo)/2)
+	for i, j := 0, 0; j < len(fromTo); i, j = i+1, j+2 {
+		rms[i] = RootMapping{
+			From: fromTo[j],
+			To:   fromTo[j+1],
+		}
+	}
+
+	return NewRootMappingFs(fs, rms...)
+}
+
+type RootMapping struct {
+	From string
+	To   string
+
+	path string   // The virtual mount point, e.g. "blog".
+	Meta FileMeta // File metadata (lang etc.)
+}
+
+func (rm *RootMapping) clean() {
+	rm.From = filepath.Clean(rm.From)
+	rm.To = filepath.Clean(rm.To)
+}
+
+func (r RootMapping) filename(name string) string {
+	return filepath.Join(r.To, strings.TrimPrefix(name, r.From))
+}
+
+func (r RootMapping) rootKey() string {
+	return r.From
+}
+
 // A RootMappingFs maps several roots into one. Note that the root of this filesystem
 // is directories only, and they will be returned in Readdir and Readdirnames
 // in the order given.
 type RootMappingFs struct {
 	afero.Fs
 	rootMapToReal *radix.Node
-	virtualRoots  []string
+	virtualRoots  []RootMapping
+	filter        func(r RootMapping) bool
 }
 
-type rootMappingFile struct {
-	afero.File
-	fs   *RootMappingFs
-	name string
-}
+func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) {
+	roots := fs.getRootsWithPrefix(base)
 
-type rootMappingFileInfo struct {
-	name string
-}
-
-func (fi *rootMappingFileInfo) Name() string {
-	return fi.name
-}
-
-func (fi *rootMappingFileInfo) Size() int64 {
-	panic("not implemented")
-}
-
-func (fi *rootMappingFileInfo) Mode() os.FileMode {
-	return os.ModeDir
-}
-
-func (fi *rootMappingFileInfo) ModTime() time.Time {
-	panic("not implemented")
-}
-
-func (fi *rootMappingFileInfo) IsDir() bool {
-	return true
-}
-
-func (fi *rootMappingFileInfo) Sys() interface{} {
-	return nil
-}
-
-func newRootMappingDirFileInfo(name string) *rootMappingFileInfo {
-	return &rootMappingFileInfo{name: name}
-}
-
-// NewRootMappingFs creates a new RootMappingFs on top of the provided with
-// a list of from, to string pairs of root mappings.
-// Note that 'from' represents a virtual root that maps to the actual filename in 'to'.
-func NewRootMappingFs(fs afero.Fs, fromTo ...string) (*RootMappingFs, error) {
-	rootMapToReal := radix.New().Txn()
-	var virtualRoots []string
-
-	for i := 0; i < len(fromTo); i += 2 {
-		vr := filepath.Clean(fromTo[i])
-		rr := filepath.Clean(fromTo[i+1])
-
-		// We need to preserve the original order for Readdir
-		virtualRoots = append(virtualRoots, vr)
-
-		rootMapToReal.Insert([]byte(vr), rr)
+	if roots == nil {
+		return nil, nil
 	}
 
-	return &RootMappingFs{Fs: fs,
-		virtualRoots:  virtualRoots,
-		rootMapToReal: rootMapToReal.Commit().Root()}, nil
+	fss := make([]FileMetaInfo, len(roots))
+	for i, r := range roots {
+		bfs := afero.NewBasePathFs(fs.Fs, r.To)
+		bfs = decoratePath(bfs, func(name string) string {
+			p := strings.TrimPrefix(name, r.To)
+			if r.path != "" {
+				// Make sure it's mounted to a any sub path, e.g. blog
+				p = filepath.Join(r.path, p)
+			}
+			p = strings.TrimLeft(p, filepathSeparator)
+			return p
+		})
+		fs := decorateDirs(bfs, r.Meta)
+		fi, err := fs.Stat("")
+		if err != nil {
+			return nil, errors.Wrap(err, "RootMappingFs.Dirs")
+		}
+		fss[i] = fi.(FileMetaInfo)
+	}
+
+	return fss, nil
+}
+
+// LstatIfPossible returns the os.FileInfo structure describing a given file.
+func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
+	fis, b, err := fs.doLstat(name, false)
+	if err != nil {
+		return nil, b, err
+	}
+	return fis[0], b, nil
+
+}
+
+func (fs *RootMappingFs) virtualDirOpener(name string, isRoot bool) func() (afero.File, error) {
+	return func() (afero.File, error) { return &rootMappingFile{name: name, isRoot: isRoot, fs: fs}, nil }
+}
+
+func (fs *RootMappingFs) doLstat(name string, allowMultiple bool) ([]FileMetaInfo, bool, error) {
+
+	if fs.isRoot(name) {
+		return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, true))}, false, nil
+	}
+
+	roots := fs.getRoots(name)
+
+	if len(roots) == 0 {
+		roots := fs.getRootsWithPrefix(name)
+		if len(roots) != 0 {
+			// We have root mappings below name, let's make it look like
+			// a directory.
+			return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, false))}, false, nil
+		}
+
+		return nil, false, os.ErrNotExist
+	}
+
+	var (
+		fis  []FileMetaInfo
+		b    bool
+		fi   os.FileInfo
+		root RootMapping
+		err  error
+	)
+
+	for _, root = range roots {
+		fi, b, err = fs.statRoot(root, name)
+		if err != nil {
+			if os.IsNotExist(err) {
+				continue
+			}
+			return nil, false, err
+		}
+		fim := fi.(FileMetaInfo)
+		fis = append(fis, fim)
+	}
+
+	if len(fis) == 0 {
+		return nil, false, os.ErrNotExist
+	}
+
+	if allowMultiple || len(fis) == 1 {
+		return fis, b, nil
+	}
+
+	// Open it in this composite filesystem.
+	opener := func() (afero.File, error) {
+		return fs.Open(name)
+	}
+
+	return []FileMetaInfo{decorateFileInfo(fi, fs, opener, "", "", root.Meta)}, b, nil
+
+}
+
+// Open opens the namedrootMappingFile file for reading.
+func (fs *RootMappingFs) Open(name string) (afero.File, error) {
+	if fs.isRoot(name) {
+		return &rootMappingFile{name: name, fs: fs, isRoot: true}, nil
+	}
+
+	fis, _, err := fs.doLstat(name, true)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(fis) == 1 {
+		fi := fis[0]
+		meta := fi.(FileMetaInfo).Meta()
+		f, err := meta.Open()
+		if err != nil {
+			return nil, err
+		}
+		return &rootMappingFile{File: f, fs: fs, name: name, meta: meta}, nil
+	}
+
+	return fs.newUnionFile(fis...)
+
 }
 
 // Stat returns the os.FileInfo structure describing a given file.  If there is
 // an error, it will be of type *os.PathError.
 func (fs *RootMappingFs) Stat(name string) (os.FileInfo, error) {
-	if fs.isRoot(name) {
-		return newRootMappingDirFileInfo(name), nil
-	}
-	realName := fs.realName(name)
+	fi, _, err := fs.LstatIfPossible(name)
+	return fi, err
 
-	fi, err := fs.Fs.Stat(realName)
-	if rfi, ok := fi.(RealFilenameInfo); ok {
-		return rfi, err
-	}
-
-	return &realFilenameInfo{FileInfo: fi, realFilename: realName}, err
+}
 
+// Filter creates a copy of this filesystem with the applied filter.
+func (fs RootMappingFs) Filter(f func(m RootMapping) bool) *RootMappingFs {
+	fs.filter = f
+	return &fs
 }
 
 func (fs *RootMappingFs) isRoot(name string) bool {
@@ -116,60 +265,193 @@ func (fs *RootMappingFs) isRoot(name string) bool {
 
 }
 
-// Open opens the named file for reading.
-func (fs *RootMappingFs) Open(name string) (afero.File, error) {
-	if fs.isRoot(name) {
-		return &rootMappingFile{name: name, fs: fs}, nil
+func (fs *RootMappingFs) getRoots(name string) []RootMapping {
+	nameb := []byte(filepath.Clean(name))
+	_, v, found := fs.rootMapToReal.LongestPrefix(nameb)
+	if !found {
+		return nil
 	}
-	realName := fs.realName(name)
-	f, err := fs.Fs.Open(realName)
+
+	rm := v.([]RootMapping)
+
+	if fs.filter != nil {
+		var filtered []RootMapping
+		for _, m := range rm {
+			if fs.filter(m) {
+				filtered = append(filtered, m)
+			}
+		}
+		return filtered
+	}
+
+	return rm
+}
+
+func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []RootMapping {
+	if fs.isRoot(prefix) {
+		return fs.virtualRoots
+	}
+	prefixb := []byte(filepath.Clean(prefix))
+	var roots []RootMapping
+
+	fs.rootMapToReal.WalkPrefix(prefixb, func(b []byte, v interface{}) bool {
+		roots = append(roots, v.([]RootMapping)...)
+		return false
+	})
+
+	return roots
+}
+
+func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) {
+	meta := fis[0].Meta()
+	f, err := meta.Open()
 	if err != nil {
 		return nil, err
 	}
-	return &rootMappingFile{File: f, name: name, fs: fs}, nil
+	rf := &rootMappingFile{File: f, fs: fs, name: meta.Name(), meta: meta}
+	if len(fis) == 1 {
+		return rf, err
+	}
+
+	next, err := fs.newUnionFile(fis[1:]...)
+	if err != nil {
+		return nil, err
+	}
+
+	uf := &afero.UnionFile{Base: rf, Layer: next}
+
+	uf.Merger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) {
+		// Ignore duplicate directory entries
+		seen := make(map[string]bool)
+		var result []os.FileInfo
+
+		for _, fis := range [][]os.FileInfo{bofi, lofi} {
+			for _, fi := range fis {
+
+				if fi.IsDir() && seen[fi.Name()] {
+					continue
+				}
+
+				if fi.IsDir() {
+					seen[fi.Name()] = true
+				}
+
+				result = append(result, fi)
+			}
+		}
+
+		return result, nil
+	}
+
+	return uf, nil
+
 }
 
-// LstatIfPossible returns the os.FileInfo structure describing a given file.
-// It attempts to use Lstat if supported or defers to the os.  In addition to
-// the FileInfo, a boolean is returned telling whether Lstat was called.
-func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
+func (fs *RootMappingFs) statRoot(root RootMapping, name string) (os.FileInfo, bool, error) {
+	filename := root.filename(name)
 
-	if fs.isRoot(name) {
-		return newRootMappingDirFileInfo(name), false, nil
-	}
-	name = fs.realName(name)
+	var b bool
+	var fi os.FileInfo
+	var err error
 
 	if ls, ok := fs.Fs.(afero.Lstater); ok {
-		fi, b, err := ls.LstatIfPossible(name)
-		return &realFilenameInfo{FileInfo: fi, realFilename: name}, b, err
+		fi, b, err = ls.LstatIfPossible(filename)
+		if err != nil {
+			return nil, b, err
+		}
+
+	} else {
+		fi, err = fs.Fs.Stat(filename)
+		if err != nil {
+			return nil, b, err
+		}
 	}
-	fi, err := fs.Stat(name)
-	return fi, false, err
+
+	// Opens the real directory/file.
+	opener := func() (afero.File, error) {
+		return fs.Fs.Open(filename)
+	}
+
+	if fi.IsDir() {
+		_, name = filepath.Split(name)
+		fi = newDirNameOnlyFileInfo(name, false, opener)
+	}
+
+	return decorateFileInfo(fi, fs.Fs, opener, "", "", root.Meta), b, nil
+
 }
 
-func (fs *RootMappingFs) realName(name string) string {
-	key, val, found := fs.rootMapToReal.LongestPrefix([]byte(filepath.Clean(name)))
-	if !found {
-		return name
-	}
-	keystr := string(key)
+type rootMappingFile struct {
+	afero.File
+	fs     *RootMappingFs
+	name   string
+	meta   FileMeta
+	isRoot bool
+}
 
-	return filepath.Join(val.(string), strings.TrimPrefix(name, keystr))
+func (f *rootMappingFile) Close() error {
+	if f.File == nil {
+		return nil
+	}
+	return f.File.Close()
+}
+
+func (f *rootMappingFile) Name() string {
+	return f.name
 }
 
 func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) {
 	if f.File == nil {
 		dirsn := make([]os.FileInfo, 0)
-		for i := 0; i < len(f.fs.virtualRoots); i++ {
-			if count != -1 && i >= count {
+		roots := f.fs.getRootsWithPrefix(f.name)
+		seen := make(map[string]bool)
+
+		j := 0
+		for _, rm := range roots {
+			if count != -1 && j >= count {
 				break
 			}
-			dirsn = append(dirsn, newRootMappingDirFileInfo(f.fs.virtualRoots[i]))
+
+			opener := func() (afero.File, error) {
+				return f.fs.Open(rm.From)
+			}
+
+			name := rm.From
+			if !f.isRoot {
+				_, name = filepath.Split(rm.From)
+			}
+
+			if seen[name] {
+				continue
+			}
+			seen[name] = true
+
+			j++
+
+			fi := newDirNameOnlyFileInfo(name, false, opener)
+			if rm.Meta != nil {
+				mergeFileMeta(rm.Meta, fi.Meta())
+			}
+
+			dirsn = append(dirsn, fi)
 		}
 		return dirsn, nil
 	}
-	return f.File.Readdir(count)
 
+	if f.File == nil {
+		panic(fmt.Sprintf("no File for %q", f.name))
+	}
+
+	fis, err := f.File.Readdir(count)
+	if err != nil {
+		return nil, err
+	}
+
+	for i, fi := range fis {
+		fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta)
+	}
+
+	return fis, nil
 }
 
 func (f *rootMappingFile) Readdirnames(count int) ([]string, error) {
@@ -183,14 +465,3 @@ func (f *rootMappingFile) Readdirnames(count int) ([]string, error) {
 	}
 	return dirss, nil
 }
-
-func (f *rootMappingFile) Name() string {
-	return f.name
-}
-
-func (f *rootMappingFile) Close() error {
-	if f.File == nil {
-		return nil
-	}
-	return f.File.Close()
-}

hugofs/rootmapping_fs_test.go

@@ -1,4 +1,4 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -19,24 +19,115 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/spf13/viper"
+
+	"github.com/gohugoio/hugo/htesting"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/require"
 )
 
-func TestRootMappingFsRealName(t *testing.T) {
+func TestLanguageRootMapping(t *testing.T) {
 	assert := require.New(t)
-	fs := afero.NewMemMapFs()
+	v := viper.New()
+	v.Set("contentDir", "content")
+
+	fs := NewBaseFileDecorator(afero.NewMemMapFs())
+
+	assert.NoError(afero.WriteFile(fs, filepath.Join("content/sv/svdir", "main.txt"), []byte("main sv"), 0755))
+	assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", "sv-f.txt"), []byte("some sv blog content"), 0755))
+	assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/myenblogcontent", "en-f.txt"), []byte("some en blog content in a"), 0755))
+	assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/myotherenblogcontent", "en-f2.txt"), []byte("some en content"), 0755))
+	assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mysvdocs", "sv-docs.txt"), []byte("some sv docs content"), 0755))
+	assert.NoError(afero.WriteFile(fs, filepath.Join("themes/b/myenblogcontent", "en-b-f.txt"), []byte("some en content"), 0755))
+
+	rfs, err := NewRootMappingFs(fs,
+		RootMapping{
+			From: "content/blog",             // Virtual path, first element is one of content, static, layouts etc.
+			To:   "themes/a/mysvblogcontent", // Real path
+			Meta: FileMeta{"lang": "sv"},
+		},
+		RootMapping{
+			From: "content/blog",
+			To:   "themes/a/myenblogcontent",
+			Meta: FileMeta{"lang": "en"},
+		},
+		RootMapping{
+			From: "content/blog",
+			To:   "content/sv",
+			Meta: FileMeta{"lang": "sv"},
+		},
+		RootMapping{
+			From: "content/blog",
+			To:   "themes/a/myotherenblogcontent",
+			Meta: FileMeta{"lang": "en"},
+		},
+		RootMapping{
+			From: "content/docs",
+			To:   "themes/a/mysvdocs",
+			Meta: FileMeta{"lang": "sv"},
+		},
+	)
 
-	rfs, err := NewRootMappingFs(fs, "f1", "f1t", "f2", "f2t")
 	assert.NoError(err)
 
-	assert.Equal(filepath.FromSlash("f1t/foo/file.txt"), rfs.realName(filepath.Join("f1", "foo", "file.txt")))
+	collected, err := collectFilenames(rfs, "content", "content")
+	assert.NoError(err)
+	assert.Equal([]string{"blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"}, collected)
+
+	bfs := afero.NewBasePathFs(rfs, "content")
+	collected, err = collectFilenames(bfs, "", "")
+	assert.NoError(err)
+	assert.Equal([]string{"blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"}, collected)
+
+	dirs, err := rfs.Dirs(filepath.FromSlash("content/blog"))
+	assert.NoError(err)
+
+	assert.Equal(4, len(dirs))
+
+	getDirnames := func(name string, rfs *RootMappingFs) []string {
+		filename := filepath.FromSlash(name)
+		f, err := rfs.Open(filename)
+		assert.NoError(err)
+		names, err := f.Readdirnames(-1)
+
+		f.Close()
+		assert.NoError(err)
+
+		info, err := rfs.Stat(filename)
+		assert.NoError(err)
+		f2, err := info.(FileMetaInfo).Meta().Open()
+		assert.NoError(err)
+		names2, err := f2.Readdirnames(-1)
+		assert.NoError(err)
+		assert.Equal(names, names2)
+		f2.Close()
+
+		return names
+	}
+
+	rfsEn := rfs.Filter(func(rm RootMapping) bool {
+		return rm.Meta.Lang() == "en"
+	})
+
+	assert.Equal([]string{"en-f.txt", "en-f2.txt"}, getDirnames("content/blog", rfsEn))
+
+	rfsSv := rfs.Filter(func(rm RootMapping) bool {
+		return rm.Meta.Lang() == "sv"
+	})
+
+	assert.Equal([]string{"sv-f.txt", "svdir"}, getDirnames("content/blog", rfsSv))
+
+	// Make sure we have not messed with the original
+	assert.Equal([]string{"sv-f.txt", "en-f.txt", "svdir", "en-f2.txt"}, getDirnames("content/blog", rfs))
+
+	assert.Equal([]string{"blog", "docs"}, getDirnames("content", rfsSv))
+	assert.Equal([]string{"blog", "docs"}, getDirnames("content", rfs))
 
 }
 
 func TestRootMappingFsDirnames(t *testing.T) {
 	assert := require.New(t)
-	fs := afero.NewMemMapFs()
+	fs := NewBaseFileDecorator(afero.NewMemMapFs())
 
 	testfile := "myfile.txt"
 	assert.NoError(fs.Mkdir("f1t", 0755))
@@ -44,13 +135,14 @@ func TestRootMappingFsDirnames(t *testing.T) {
 	assert.NoError(fs.Mkdir("f3t", 0755))
 	assert.NoError(afero.WriteFile(fs, filepath.Join("f2t", testfile), []byte("some content"), 0755))
 
-	rfs, err := NewRootMappingFs(fs, "bf1", "f1t", "cf2", "f2t", "af3", "f3t")
+	rfs, err := NewRootMappingFsFromFromTo(fs, "static/bf1", "f1t", "static/cf2", "f2t", "static/af3", "f3t")
 	assert.NoError(err)
 
-	fif, err := rfs.Stat(filepath.Join("cf2", testfile))
+	fif, err := rfs.Stat(filepath.Join("static/cf2", testfile))
 	assert.NoError(err)
 	assert.Equal("myfile.txt", fif.Name())
-	assert.Equal(filepath.FromSlash("f2t/myfile.txt"), fif.(RealFilenameInfo).RealFilename())
+	fifm := fif.(FileMetaInfo).Meta()
+	assert.Equal(filepath.FromSlash("f2t/myfile.txt"), fifm.Filename())
 
 	root, err := rfs.Open(filepathSeparator)
 	assert.NoError(err)
@@ -61,6 +153,91 @@ func TestRootMappingFsDirnames(t *testing.T) {
 
 }
 
+func TestRootMappingFsFilename(t *testing.T) {
+	assert := require.New(t)
+	workDir, clean, err := htesting.CreateTempDir(Os, "hugo-root-filename")
+	assert.NoError(err)
+	defer clean()
+	fs := NewBaseFileDecorator(Os)
+
+	testfilename := filepath.Join(workDir, "f1t/foo/file.txt")
+
+	assert.NoError(fs.MkdirAll(filepath.Join(workDir, "f1t/foo"), 0777))
+	assert.NoError(afero.WriteFile(fs, testfilename, []byte("content"), 0666))
+
+	rfs, err := NewRootMappingFsFromFromTo(fs, "static/f1", filepath.Join(workDir, "f1t"), "static/f2", filepath.Join(workDir, "f2t"))
+	assert.NoError(err)
+
+	fi, err := rfs.Stat(filepath.FromSlash("static/f1/foo/file.txt"))
+	assert.NoError(err)
+	fim := fi.(FileMetaInfo)
+	assert.Equal(testfilename, fim.Meta().Filename())
+	_, err = rfs.Stat(filepath.FromSlash("static/f1"))
+	assert.NoError(err)
+}
+
+func TestRootMappingFsMount(t *testing.T) {
+	assert := require.New(t)
+	fs := NewBaseFileDecorator(afero.NewMemMapFs())
+
+	testfile := "test.txt"
+
+	assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mynoblogcontent", testfile), []byte("some no content"), 0755))
+	assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/myenblogcontent", testfile), []byte("some en content"), 0755))
+	assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", testfile), []byte("some sv content"), 0755))
+	assert.NoError(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", "other.txt"), []byte("some sv content"), 0755))
+
+	bfs := afero.NewBasePathFs(fs, "themes/a").(*afero.BasePathFs)
+	rm := []RootMapping{
+		RootMapping{From: "content/blog",
+			To:   "mynoblogcontent",
+			Meta: FileMeta{"lang": "no"},
+		},
+		RootMapping{From: "content/blog",
+			To:   "myenblogcontent",
+			Meta: FileMeta{"lang": "en"},
+		},
+		RootMapping{From: "content/blog",
+			To:   "mysvblogcontent",
+			Meta: FileMeta{"lang": "sv"},
+		},
+	}
+
+	rfs, err := NewRootMappingFs(bfs, rm...)
+	assert.NoError(err)
+
+	blog, err := rfs.Stat(filepath.FromSlash("content/blog"))
+	assert.NoError(err)
+	blogm := blog.(FileMetaInfo).Meta()
+	assert.Equal("sv", blogm.Lang()) // Last match
+
+	f, err := blogm.Open()
+	assert.NoError(err)
+	defer f.Close()
+	dirs1, err := f.Readdirnames(-1)
+	assert.NoError(err)
+	// Union with duplicate dir names filtered.
+	assert.Equal([]string{"test.txt", "test.txt", "other.txt", "test.txt"}, dirs1)
+
+	files, err := afero.ReadDir(rfs, filepath.FromSlash("content/blog"))
+	assert.NoError(err)
+	assert.Equal(4, len(files))
+
+	testfilefi := files[1]
+	assert.Equal(testfile, testfilefi.Name())
+
+	testfilem := testfilefi.(FileMetaInfo).Meta()
+	assert.Equal(filepath.FromSlash("themes/a/mynoblogcontent/test.txt"), testfilem.Filename())
+
+	tf, err := testfilem.Open()
+	assert.NoError(err)
+	defer tf.Close()
+	c, err := ioutil.ReadAll(tf)
+	assert.NoError(err)
+	assert.Equal("some no content", string(c))
+
+}
+
 func TestRootMappingFsOs(t *testing.T) {
 	assert := require.New(t)
 	fs := afero.NewOsFs()
@@ -77,10 +254,10 @@ func TestRootMappingFsOs(t *testing.T) {
 	assert.NoError(fs.Mkdir(filepath.Join(d, "f3t"), 0755))
 	assert.NoError(afero.WriteFile(fs, filepath.Join(d, "f2t", testfile), []byte("some content"), 0755))
 
-	rfs, err := NewRootMappingFs(fs, "bf1", filepath.Join(d, "f1t"), "cf2", filepath.Join(d, "f2t"), "af3", filepath.Join(d, "f3t"))
+	rfs, err := NewRootMappingFsFromFromTo(fs, "static/bf1", filepath.Join(d, "f1t"), "static/cf2", filepath.Join(d, "f2t"), "static/af3", filepath.Join(d, "f3t"))
 	assert.NoError(err)
 
-	fif, err := rfs.Stat(filepath.Join("cf2", testfile))
+	fif, err := rfs.Stat(filepath.Join("static/cf2", testfile))
 	assert.NoError(err)
 	assert.Equal("myfile.txt", fif.Name())
 

hugofs/slice_fs.go

@@ -0,0 +1,293 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"os"
+	"syscall"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/spf13/afero"
+)
+
+var (
+	_ afero.Fs      = (*SliceFs)(nil)
+	_ afero.Lstater = (*SliceFs)(nil)
+	_ afero.File    = (*sliceDir)(nil)
+)
+
+func NewSliceFs(dirs ...FileMetaInfo) (afero.Fs, error) {
+	if len(dirs) == 0 {
+		return NoOpFs, nil
+	}
+
+	for _, dir := range dirs {
+		if !dir.IsDir() {
+			return nil, errors.New("this fs supports directories only")
+		}
+	}
+
+	fs := &SliceFs{
+		dirs: dirs,
+	}
+
+	return fs, nil
+
+}
+
+// SliceFs is an ordered composite filesystem.
+type SliceFs struct {
+	dirs []FileMetaInfo
+}
+
+func (fs *SliceFs) Chmod(n string, m os.FileMode) error {
+	return syscall.EPERM
+}
+
+func (fs *SliceFs) Chtimes(n string, a, m time.Time) error {
+	return syscall.EPERM
+}
+
+func (fs *SliceFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
+	fi, _, err := fs.pickFirst(name)
+
+	if err != nil {
+		return nil, false, err
+	}
+
+	if fi.IsDir() {
+		return decorateFileInfo(fi, fs, fs.getOpener(name), "", "", nil), false, nil
+	}
+
+	return nil, false, errors.Errorf("lstat: files not supported: %q", name)
+
+}
+
+func (fs *SliceFs) Mkdir(n string, p os.FileMode) error {
+	return syscall.EPERM
+}
+
+func (fs *SliceFs) MkdirAll(n string, p os.FileMode) error {
+	return syscall.EPERM
+}
+
+func (fs *SliceFs) Name() string {
+	return "SliceFs"
+}
+
+func (fs *SliceFs) Open(name string) (afero.File, error) {
+	fi, idx, err := fs.pickFirst(name)
+	if err != nil {
+		return nil, err
+	}
+
+	if !fi.IsDir() {
+		panic("currently only dirs in here")
+	}
+
+	return &sliceDir{
+		lfs:     fs,
+		idx:     idx,
+		dirname: name,
+	}, nil
+
+}
+
+func (fs *SliceFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
+	panic("not implemented")
+}
+
+func (fs *SliceFs) ReadDir(name string) ([]os.FileInfo, error) {
+	panic("not implemented")
+}
+
+func (fs *SliceFs) Remove(n string) error {
+	return syscall.EPERM
+}
+
+func (fs *SliceFs) RemoveAll(p string) error {
+	return syscall.EPERM
+}
+
+func (fs *SliceFs) Rename(o, n string) error {
+	return syscall.EPERM
+}
+
+func (fs *SliceFs) Stat(name string) (os.FileInfo, error) {
+	fi, _, err := fs.LstatIfPossible(name)
+	return fi, err
+}
+
+func (fs *SliceFs) Create(n string) (afero.File, error) {
+	return nil, syscall.EPERM
+}
+
+func (fs *SliceFs) getOpener(name string) func() (afero.File, error) {
+	return func() (afero.File, error) {
+		return fs.Open(name)
+	}
+}
+
+func (fs *SliceFs) pickFirst(name string) (os.FileInfo, int, error) {
+	for i, mfs := range fs.dirs {
+		meta := mfs.Meta()
+		fs := meta.Fs()
+		fi, _, err := lstatIfPossible(fs, name)
+		if err == nil {
+			// Gotta match!
+			return fi, i, nil
+		}
+
+		if !os.IsNotExist(err) {
+			// Real error
+			return nil, -1, err
+		}
+	}
+
+	// Not found
+	return nil, -1, os.ErrNotExist
+}
+
+func (fs *SliceFs) readDirs(name string, startIdx, count int) ([]os.FileInfo, error) {
+	collect := func(lfs FileMeta) ([]os.FileInfo, error) {
+		d, err := lfs.Fs().Open(name)
+		if err != nil {
+			if !os.IsNotExist(err) {
+				return nil, err
+			}
+			return nil, nil
+		} else {
+			defer d.Close()
+			dirs, err := d.Readdir(-1)
+			if err != nil {
+				return nil, err
+			}
+			return dirs, nil
+		}
+	}
+
+	var dirs []os.FileInfo
+
+	for i := startIdx; i < len(fs.dirs); i++ {
+		mfs := fs.dirs[i]
+
+		fis, err := collect(mfs.Meta())
+		if err != nil {
+			return nil, err
+		}
+
+		dirs = append(dirs, fis...)
+
+	}
+
+	seen := make(map[string]bool)
+	var duplicates []int
+	for i, fi := range dirs {
+		if !fi.IsDir() {
+			continue
+		}
+
+		if seen[fi.Name()] {
+			duplicates = append(duplicates, i)
+		} else {
+			// Make sure it's opened by this filesystem.
+			dirs[i] = decorateFileInfo(fi, fs, fs.getOpener(fi.(FileMetaInfo).Meta().Filename()), "", "", nil)
+			seen[fi.Name()] = true
+		}
+	}
+
+	// Remove duplicate directories, keep first.
+	if len(duplicates) > 0 {
+		for i := len(duplicates) - 1; i >= 0; i-- {
+			idx := duplicates[i]
+			dirs = append(dirs[:idx], dirs[idx+1:]...)
+		}
+	}
+
+	if count > 0 && len(dirs) >= count {
+		return dirs[:count], nil
+	}
+
+	return dirs, nil
+
+}
+
+type sliceDir struct {
+	lfs     *SliceFs
+	idx     int
+	dirname string
+}
+
+func (f *sliceDir) Close() error {
+	return nil
+}
+
+func (f *sliceDir) Name() string {
+	return f.dirname
+}
+
+func (f *sliceDir) Read(p []byte) (n int, err error) {
+	panic("not implemented")
+}
+
+func (f *sliceDir) ReadAt(p []byte, off int64) (n int, err error) {
+	panic("not implemented")
+}
+
+func (f *sliceDir) Readdir(count int) ([]os.FileInfo, error) {
+	return f.lfs.readDirs(f.dirname, f.idx, count)
+}
+
+func (f *sliceDir) Readdirnames(count int) ([]string, error) {
+	dirsi, err := f.Readdir(count)
+	if err != nil {
+		return nil, err
+	}
+
+	dirs := make([]string, len(dirsi))
+	for i, d := range dirsi {
+		dirs[i] = d.Name()
+	}
+	return dirs, nil
+}
+
+func (f *sliceDir) Seek(offset int64, whence int) (int64, error) {
+	panic("not implemented")
+}
+
+func (f *sliceDir) Stat() (os.FileInfo, error) {
+	panic("not implemented")
+}
+
+func (f *sliceDir) Sync() error {
+	panic("not implemented")
+}
+
+func (f *sliceDir) Truncate(size int64) error {
+	panic("not implemented")
+}
+
+func (f *sliceDir) Write(p []byte) (n int, err error) {
+	panic("not implemented")
+}
+
+func (f *sliceDir) WriteAt(p []byte, off int64) (n int, err error) {
+	panic("not implemented")
+}
+
+func (f *sliceDir) WriteString(s string) (ret int, err error) {
+	panic("not implemented")
+}

hugofs/walk.go

@@ -0,0 +1,308 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/gohugoio/hugo/common/loggers"
+
+	"github.com/pkg/errors"
+
+	"github.com/spf13/afero"
+)
+
+type (
+	WalkFunc func(path string, info FileMetaInfo, err error) error
+	WalkHook func(dir FileMetaInfo, path string, readdir []FileMetaInfo) ([]FileMetaInfo, error)
+)
+
+type Walkway struct {
+	fs       afero.Fs
+	root     string
+	basePath string
+
+	logger *loggers.Logger
+
+	// May be pre-set
+	fi         FileMetaInfo
+	dirEntries []FileMetaInfo
+
+	walkFn WalkFunc
+	walked bool
+
+	// We may traverse symbolic links and bite ourself.
+	seen map[string]bool
+
+	// Optional hooks
+	hookPre  WalkHook
+	hookPost WalkHook
+}
+
+type WalkwayConfig struct {
+	Fs       afero.Fs
+	Root     string
+	BasePath string
+
+	Logger *loggers.Logger
+
+	// One or both of these may be pre-set.
+	Info       FileMetaInfo
+	DirEntries []FileMetaInfo
+
+	WalkFn   WalkFunc
+	HookPre  WalkHook
+	HookPost WalkHook
+}
+
+func NewWalkway(cfg WalkwayConfig) *Walkway {
+	var fs afero.Fs
+	if cfg.Info != nil {
+		fs = cfg.Info.Meta().Fs()
+	} else {
+		fs = cfg.Fs
+	}
+
+	basePath := cfg.BasePath
+	if basePath != "" && !strings.HasSuffix(basePath, filepathSeparator) {
+		basePath += filepathSeparator
+	}
+
+	logger := cfg.Logger
+	if logger == nil {
+		logger = loggers.NewWarningLogger()
+	}
+
+	return &Walkway{
+		fs:         fs,
+		root:       cfg.Root,
+		basePath:   basePath,
+		fi:         cfg.Info,
+		dirEntries: cfg.DirEntries,
+		walkFn:     cfg.WalkFn,
+		hookPre:    cfg.HookPre,
+		hookPost:   cfg.HookPost,
+		logger:     logger,
+		seen:       make(map[string]bool)}
+}
+
+func (w *Walkway) Walk() error {
+	if w.walked {
+		panic("this walkway is already walked")
+	}
+	w.walked = true
+
+	if w.fs == NoOpFs {
+		return nil
+	}
+
+	var fi FileMetaInfo
+	if w.fi != nil {
+		fi = w.fi
+	} else {
+		info, _, err := lstatIfPossible(w.fs, w.root)
+		if err != nil {
+			if os.IsNotExist(err) {
+				return nil
+			}
+
+			if err == ErrPermissionSymlink {
+				w.logger.WARN.Printf("Unsupported symlink found in %q, skipping.", w.root)
+				return nil
+			}
+
+			return w.walkFn(w.root, nil, errors.Wrapf(err, "walk: %q", w.root))
+		}
+		fi = info.(FileMetaInfo)
+	}
+
+	if !fi.IsDir() {
+		return w.walkFn(w.root, nil, errors.New("file to walk must be a directory"))
+	}
+
+	return w.walk(w.root, fi, w.dirEntries, w.walkFn)
+
+}
+
+// if the filesystem supports it, use Lstat, else use fs.Stat
+func lstatIfPossible(fs afero.Fs, path string) (os.FileInfo, bool, error) {
+	if lfs, ok := fs.(afero.Lstater); ok {
+		fi, b, err := lfs.LstatIfPossible(path)
+		return fi, b, err
+	}
+	fi, err := fs.Stat(path)
+	return fi, false, err
+}
+
+// walk recursively descends path, calling walkFn.
+// It follow symlinks if supported by the filesystem, but only the same path once.
+func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo, walkFn WalkFunc) error {
+	err := walkFn(path, info, nil)
+	if err != nil {
+		if info.IsDir() && err == filepath.SkipDir {
+			return nil
+		}
+		return err
+	}
+	if !info.IsDir() {
+		return nil
+	}
+
+	meta := info.Meta()
+	filename := meta.Filename()
+
+	if dirEntries == nil {
+		f, err := w.fs.Open(path)
+
+		if err != nil {
+			return walkFn(path, info, errors.Wrapf(err, "walk: open %q (%q)", path, w.root))
+		}
+
+		fis, err := f.Readdir(-1)
+		f.Close()
+		if err != nil {
+			if err == ErrPermissionSymlink {
+				w.logger.WARN.Printf("Unsupported symlink found in %q, skipping.", filename)
+				return nil
+			}
+			return walkFn(path, info, errors.Wrap(err, "walk: Readdir"))
+		}
+
+		dirEntries = fileInfosToFileMetaInfos(fis)
+
+		if !meta.IsOrdered() {
+			sort.Slice(dirEntries, func(i, j int) bool {
+				fii := dirEntries[i]
+				fij := dirEntries[j]
+
+				fim, fjm := fii.Meta(), fij.Meta()
+
+				// Pull bundle headers to the top.
+				ficlass, fjclass := fim.Classifier(), fjm.Classifier()
+				if ficlass != fjclass {
+					return ficlass < fjclass
+				}
+
+				// With multiple content dirs with different languages,
+				// there can be duplicate files, and a weight will be added
+				// to the closest one.
+				fiw, fjw := fim.Weight(), fjm.Weight()
+				if fiw != fjw {
+					return fiw > fjw
+				}
+
+				// Explicit order set.
+				fio, fjo := fim.Ordinal(), fjm.Ordinal()
+				if fio != fjo {
+					return fio < fjo
+				}
+
+				// When we walk into a symlink, we keep the reference to
+				// the original name.
+				fin, fjn := fim.Name(), fjm.Name()
+				if fin != "" && fjn != "" {
+					return fin < fjn
+				}
+
+				return fii.Name() < fij.Name()
+			})
+		}
+	}
+
+	// First add some metadata to the dir entries
+	for _, fi := range dirEntries {
+		fim := fi.(FileMetaInfo)
+
+		meta := fim.Meta()
+
+		// Note that we use the original Name even if it's a symlink.
+		name := meta.Name()
+		if name == "" {
+			name = fim.Name()
+		}
+
+		if name == "" {
+			panic(fmt.Sprintf("[%s] no name set in %v", path, meta))
+		}
+		pathn := filepath.Join(path, name)
+
+		pathMeta := pathn
+		if w.basePath != "" {
+			pathMeta = strings.TrimPrefix(pathn, w.basePath)
+		}
+
+		meta[metaKeyPath] = normalizeFilename(pathMeta)
+		meta[metaKeyPathWalk] = pathn
+
+		if fim.IsDir() && w.isSeen(meta.Filename()) {
+			// Prevent infinite recursion
+			// Possible cyclic reference
+			meta[metaKeySkipDir] = true
+		}
+	}
+
+	if w.hookPre != nil {
+		dirEntries, err = w.hookPre(info, path, dirEntries)
+		if err != nil {
+			if err == filepath.SkipDir {
+				return nil
+			}
+			return err
+		}
+	}
+
+	for _, fi := range dirEntries {
+		fim := fi.(FileMetaInfo)
+		meta := fim.Meta()
+
+		if meta.SkipDir() {
+			continue
+		}
+
+		err := w.walk(meta.GetString(metaKeyPathWalk), fim, nil, walkFn)
+		if err != nil {
+			if !fi.IsDir() || err != filepath.SkipDir {
+				return err
+			}
+		}
+	}
+
+	if w.hookPost != nil {
+		dirEntries, err = w.hookPost(info, path, dirEntries)
+		if err != nil {
+			if err == filepath.SkipDir {
+				return nil
+			}
+			return err
+		}
+	}
+	return nil
+}
+
+func (w *Walkway) isSeen(filename string) bool {
+	if filename == "" {
+		return false
+	}
+
+	if w.seen[filename] {
+		return true
+	}
+
+	w.seen[filename] = true
+	return false
+}

hugofs/walk_test.go

@@ -0,0 +1,225 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/gohugoio/hugo/common/hugo"
+
+	"github.com/pkg/errors"
+
+	"github.com/gohugoio/hugo/htesting"
+
+	"github.com/spf13/afero"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestWalk(t *testing.T) {
+	assert := require.New(t)
+
+	fs := NewBaseFileDecorator(afero.NewMemMapFs())
+
+	afero.WriteFile(fs, "b.txt", []byte("content"), 0777)
+	afero.WriteFile(fs, "c.txt", []byte("content"), 0777)
+	afero.WriteFile(fs, "a.txt", []byte("content"), 0777)
+
+	names, err := collectFilenames(fs, "", "")
+
+	assert.NoError(err)
+	assert.Equal([]string{"a.txt", "b.txt", "c.txt"}, names)
+}
+
+func TestWalkRootMappingFs(t *testing.T) {
+	assert := require.New(t)
+	fs := NewBaseFileDecorator(afero.NewMemMapFs())
+
+	testfile := "test.txt"
+
+	assert.NoError(afero.WriteFile(fs, filepath.Join("a/b", testfile), []byte("some content"), 0755))
+	assert.NoError(afero.WriteFile(fs, filepath.Join("c/d", testfile), []byte("some content"), 0755))
+	assert.NoError(afero.WriteFile(fs, filepath.Join("e/f", testfile), []byte("some content"), 0755))
+
+	rm := []RootMapping{
+		RootMapping{
+			From: "static/b",
+			To:   "e/f",
+		},
+		RootMapping{
+			From: "static/a",
+			To:   "c/d",
+		},
+
+		RootMapping{
+			From: "static/c",
+			To:   "a/b",
+		},
+	}
+
+	rfs, err := NewRootMappingFs(fs, rm...)
+	assert.NoError(err)
+	bfs := afero.NewBasePathFs(rfs, "static")
+
+	names, err := collectFilenames(bfs, "", "")
+
+	assert.NoError(err)
+	assert.Equal([]string{"a/test.txt", "b/test.txt", "c/test.txt"}, names)
+
+}
+
+func skipSymlink() bool {
+	return runtime.GOOS == "windows" && os.Getenv("CI") == ""
+}
+
+func TestWalkSymbolicLink(t *testing.T) {
+	if skipSymlink() {
+		t.Skip("Skip; os.Symlink needs administrator rights on Windows")
+	}
+	assert := require.New(t)
+	workDir, clean, err := htesting.CreateTempDir(Os, "hugo-walk-sym")
+	assert.NoError(err)
+	defer clean()
+	wd, _ := os.Getwd()
+	defer func() {
+		os.Chdir(wd)
+	}()
+
+	fs := NewBaseFileDecorator(Os)
+
+	blogDir := filepath.Join(workDir, "blog")
+	docsDir := filepath.Join(workDir, "docs")
+	blogReal := filepath.Join(blogDir, "real")
+	blogRealSub := filepath.Join(blogReal, "sub")
+	assert.NoError(os.MkdirAll(blogRealSub, 0777))
+	assert.NoError(os.MkdirAll(docsDir, 0777))
+	afero.WriteFile(fs, filepath.Join(blogRealSub, "a.txt"), []byte("content"), 0777)
+	afero.WriteFile(fs, filepath.Join(docsDir, "b.txt"), []byte("content"), 0777)
+
+	os.Chdir(blogDir)
+	assert.NoError(os.Symlink("real", "symlinked"))
+	os.Chdir(blogReal)
+	assert.NoError(os.Symlink("../real", "cyclic"))
+	os.Chdir(docsDir)
+	assert.NoError(os.Symlink("../blog/real/cyclic", "docsreal"))
+
+	t.Run("OS Fs", func(t *testing.T) {
+		assert := require.New(t)
+
+		names, err := collectFilenames(fs, workDir, workDir)
+		assert.NoError(err)
+
+		assert.Equal([]string{"blog/real/sub/a.txt", "docs/b.txt"}, names)
+	})
+
+	t.Run("BasePath Fs", func(t *testing.T) {
+		if hugo.GoMinorVersion() < 12 {
+			// https://github.com/golang/go/issues/30520
+			// This is fixed in Go 1.13 and in the latest Go 1.12
+			t.Skip("skip this for Go <= 1.11 due to a bug in Go's stdlib")
+
+		}
+		assert := require.New(t)
+
+		docsFs := afero.NewBasePathFs(fs, docsDir)
+
+		names, err := collectFilenames(docsFs, "", "")
+		assert.NoError(err)
+
+		// Note: the docsreal folder is considered cyclic when walking from the root, but this works.
+		assert.Equal([]string{"b.txt", "docsreal/sub/a.txt"}, names)
+	})
+
+}
+
+func collectFilenames(fs afero.Fs, base, root string) ([]string, error) {
+	var names []string
+
+	walkFn := func(path string, info FileMetaInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		filename := info.Meta().Path()
+		filename = filepath.ToSlash(filename)
+
+		names = append(names, filename)
+
+		return nil
+	}
+
+	w := NewWalkway(WalkwayConfig{Fs: fs, BasePath: base, Root: root, WalkFn: walkFn})
+
+	err := w.Walk()
+
+	return names, err
+
+}
+
+func BenchmarkWalk(b *testing.B) {
+	assert := require.New(b)
+	fs := NewBaseFileDecorator(afero.NewMemMapFs())
+
+	writeFiles := func(dir string, numfiles int) {
+		for i := 0; i < numfiles; i++ {
+			filename := filepath.Join(dir, fmt.Sprintf("file%d.txt", i))
+			assert.NoError(afero.WriteFile(fs, filename, []byte("content"), 0777))
+		}
+	}
+
+	const numFilesPerDir = 20
+
+	writeFiles("root", numFilesPerDir)
+	writeFiles("root/l1_1", numFilesPerDir)
+	writeFiles("root/l1_1/l2_1", numFilesPerDir)
+	writeFiles("root/l1_1/l2_2", numFilesPerDir)
+	writeFiles("root/l1_2", numFilesPerDir)
+	writeFiles("root/l1_2/l2_1", numFilesPerDir)
+	writeFiles("root/l1_3", numFilesPerDir)
+
+	walkFn := func(path string, info FileMetaInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+
+		filename := info.Meta().Filename()
+		if !strings.HasPrefix(filename, "root") {
+			return errors.New(filename)
+		}
+
+		return nil
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		w := NewWalkway(WalkwayConfig{Fs: fs, Root: "root", WalkFn: walkFn})
+
+		if err := w.Walk(); err != nil {
+			b.Fatal(err)
+		}
+	}
+
+}