Add support for a content dir set per language

A sample config:

```toml
defaultContentLanguage = "en"
defaultContentLanguageInSubdir = true

[Languages]
[Languages.en]
weight = 10
title = "In English"
languageName = "English"
contentDir = "content/english"

[Languages.nn]
weight = 20
title = "På Norsk"
languageName = "Norsk"
contentDir = "content/norwegian"
```

The value of `contentDir` can be any valid path, even absolute path references. The only restriction is that the content dirs cannot overlap.

The content files will be assigned a language by

1. The placement: `content/norwegian/post/my-post.md` will be read as Norwegian content.
2. The filename: `content/english/post/my-post.nn.md` will be read as Norwegian even if it lives in the English content folder.

The content directories will be merged into a big virtual filesystem with one simple rule: The most specific language file will win.
This means that if both `content/norwegian/post/my-post.md` and `content/english/post/my-post.nn.md` exists, they will be considered duplicates and the version inside `content/norwegian` will win.

Note that translations will be automatically assigned by Hugo by the content file's relative placement, so `content/norwegian/post/my-post.md` will be a translation of `content/english/post/my-post.md`.

If this does not work for you, you can connect the translations together by setting a `translationKey` in the content files' front matter.

Fixes #4523
Fixes #4552
Fixes #4553

hugofs/base_fs.go

@@ -0,0 +1,35 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"github.com/spf13/afero"
+)
+
+// BaseFs contains the core base filesystems used by Hugo. The name "base" is used
+// to underline that even if they can be composites, they all have a base path set to a specific
+// resource folder, e.g "/my-project/content". So, no absolute filenames needed.
+type BaseFs struct {
+	// The filesystem used to capture content. This can be a composite and
+	// language aware file system.
+	ContentFs afero.Fs
+
+	// The filesystem used to store resources (processed images etc.).
+	// This usually maps to /my-project/resources.
+	ResourcesFs afero.Fs
+
+	// The filesystem used to publish the rendered site.
+	// This usually maps to /my-project/public.
+	PublishFs afero.Fs
+}

hugofs/language_composite_fs.go

@@ -0,0 +1,51 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"github.com/spf13/afero"
+)
+
+var (
+	_ afero.Fs      = (*languageCompositeFs)(nil)
+	_ afero.Lstater = (*languageCompositeFs)(nil)
+)
+
+type languageCompositeFs struct {
+	*afero.CopyOnWriteFs
+}
+
+// NewLanguageCompositeFs creates a composite and language aware filesystem.
+// This is a hybrid filesystem. To get a specific file in Open, Stat etc., use the full filename
+// to the target filesystem. This information is available in Readdir, Stat etc. via the
+// special LanguageFileInfo FileInfo implementation.
+func NewLanguageCompositeFs(base afero.Fs, overlay *LanguageFs) afero.Fs {
+	return afero.NewReadOnlyFs(&languageCompositeFs{afero.NewCopyOnWriteFs(base, overlay).(*afero.CopyOnWriteFs)})
+}
+
+// Open takes the full path to the file in the target filesystem. If it is a directory, it gets merged
+// using the language as a weight.
+func (fs *languageCompositeFs) Open(name string) (afero.File, error) {
+	f, err := fs.CopyOnWriteFs.Open(name)
+	if err != nil {
+		return nil, err
+	}
+
+	fu, ok := f.(*afero.UnionFile)
+	if ok {
+		// This is a directory: Merge it.
+		fu.Merger = LanguageDirsMerger
+	}
+	return f, nil
+}

hugofs/language_composite_fs_test.go

@@ -0,0 +1,106 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"path/filepath"
+
+	"strings"
+
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCompositeLanguagFsTest(t *testing.T) {
+	assert := require.New(t)
+
+	languages := map[string]bool{
+		"sv": true,
+		"en": true,
+		"nn": true,
+	}
+	msv := afero.NewMemMapFs()
+	baseSv := "/content/sv"
+	lfssv := NewLanguageFs("sv", languages, afero.NewBasePathFs(msv, baseSv))
+	mnn := afero.NewMemMapFs()
+	baseNn := "/content/nn"
+	lfsnn := NewLanguageFs("nn", languages, afero.NewBasePathFs(mnn, baseNn))
+	men := afero.NewMemMapFs()
+	baseEn := "/content/en"
+	lfsen := NewLanguageFs("en", languages, afero.NewBasePathFs(men, baseEn))
+
+	// The order will be sv, en, nn
+	composite := NewLanguageCompositeFs(lfsnn, lfsen)
+	composite = NewLanguageCompositeFs(composite, lfssv)
+
+	afero.WriteFile(msv, filepath.Join(baseSv, "f1.txt"), []byte("some sv"), 0755)
+	afero.WriteFile(mnn, filepath.Join(baseNn, "f1.txt"), []byte("some nn"), 0755)
+	afero.WriteFile(men, filepath.Join(baseEn, "f1.txt"), []byte("some en"), 0755)
+
+	// Swedish is the top layer.
+	assertLangFile(t, composite, "f1.txt", "sv")
+
+	afero.WriteFile(msv, filepath.Join(baseSv, "f2.en.txt"), []byte("some sv"), 0755)
+	afero.WriteFile(mnn, filepath.Join(baseNn, "f2.en.txt"), []byte("some nn"), 0755)
+	afero.WriteFile(men, filepath.Join(baseEn, "f2.en.txt"), []byte("some en"), 0755)
+
+	// English is in the middle, but the most specific language match wins.
+	//assertLangFile(t, composite, "f2.en.txt", "en")
+
+	// Fetch some specific language versions
+	assertLangFile(t, composite, filepath.Join(baseNn, "f2.en.txt"), "nn")
+	assertLangFile(t, composite, filepath.Join(baseEn, "f2.en.txt"), "en")
+	assertLangFile(t, composite, filepath.Join(baseSv, "f2.en.txt"), "sv")
+
+	// Read the root
+	f, err := composite.Open("/")
+	assert.NoError(err)
+	defer f.Close()
+	files, err := f.Readdir(-1)
+	assert.Equal(4, len(files))
+	expected := map[string]bool{
+		filepath.FromSlash("/content/en/f1.txt"):    true,
+		filepath.FromSlash("/content/nn/f1.txt"):    true,
+		filepath.FromSlash("/content/sv/f1.txt"):    true,
+		filepath.FromSlash("/content/en/f2.en.txt"): true,
+	}
+	got := make(map[string]bool)
+
+	for _, fi := range files {
+		fil, ok := fi.(*LanguageFileInfo)
+		assert.True(ok)
+		got[fil.Filename()] = true
+	}
+	assert.Equal(expected, got)
+}
+
+func assertLangFile(t testing.TB, fs afero.Fs, filename, match string) {
+	f, err := fs.Open(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	b, err := afero.ReadAll(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s := string(b)
+	if !strings.Contains(s, match) {
+		t.Fatalf("got %q expected it to contain %q", s, match)
+
+	}
+}

hugofs/language_fs.go

@@ -0,0 +1,328 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/afero"
+)
+
+const hugoFsMarker = "__hugofs"
+
+var (
+	_ LanguageAnnouncer = (*LanguageFileInfo)(nil)
+	_ FilePather        = (*LanguageFileInfo)(nil)
+	_ afero.Lstater     = (*LanguageFs)(nil)
+)
+
+// LanguageAnnouncer is aware of its language.
+type LanguageAnnouncer interface {
+	Lang() string
+	TranslationBaseName() string
+}
+
+// FilePather is aware of its file's location.
+type FilePather interface {
+	// Filename gets the full path and filename to the file.
+	Filename() string
+
+	// Path gets the content relative path including file name and extension.
+	// The directory is relative to the content root where "content" is a broad term.
+	Path() string
+
+	// RealName is FileInfo.Name in its original form.
+	RealName() string
+
+	BaseDir() string
+}
+
+var LanguageDirsMerger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) {
+	m := make(map[string]*LanguageFileInfo)
+
+	for _, fi := range lofi {
+		fil, ok := fi.(*LanguageFileInfo)
+		if !ok {
+			return nil, fmt.Errorf("received %T, expected *LanguageFileInfo", fi)
+		}
+		m[fil.virtualName] = fil
+	}
+
+	for _, fi := range bofi {
+		fil, ok := fi.(*LanguageFileInfo)
+		if !ok {
+			return nil, fmt.Errorf("received %T, expected *LanguageFileInfo", fi)
+		}
+		existing, found := m[fil.virtualName]
+
+		if !found || existing.weight < fil.weight {
+			m[fil.virtualName] = fil
+		}
+	}
+
+	merged := make([]os.FileInfo, len(m))
+	i := 0
+	for _, v := range m {
+		merged[i] = v
+		i++
+	}
+
+	return merged, nil
+}
+
+type LanguageFileInfo struct {
+	os.FileInfo
+	lang                string
+	baseDir             string
+	realFilename        string
+	relFilename         string
+	name                string
+	realName            string
+	virtualName         string
+	translationBaseName string
+
+	// We add some weight to the files in their own language's content directory.
+	weight int
+}
+
+func (fi *LanguageFileInfo) Filename() string {
+	return fi.realFilename
+}
+
+func (fi *LanguageFileInfo) Path() string {
+	return fi.relFilename
+}
+
+func (fi *LanguageFileInfo) RealName() string {
+	return fi.realName
+}
+
+func (fi *LanguageFileInfo) BaseDir() string {
+	return fi.baseDir
+}
+
+func (fi *LanguageFileInfo) Lang() string {
+	return fi.lang
+}
+
+// TranslationBaseName returns the base filename without any extension or language
+// identificator.
+func (fi *LanguageFileInfo) TranslationBaseName() string {
+	return fi.translationBaseName
+}
+
+// Name is the name of the file within this filesystem without any path info.
+// It will be marked with language information so we can identify it as ours.
+func (fi *LanguageFileInfo) Name() string {
+	return fi.name
+}
+
+type languageFile struct {
+	afero.File
+	fs *LanguageFs
+}
+
+// Readdir creates FileInfo entries by calling Lstat if possible.
+func (l *languageFile) Readdir(c int) (ofi []os.FileInfo, err error) {
+	names, err := l.File.Readdirnames(c)
+	if err != nil {
+		return nil, err
+	}
+
+	fis := make([]os.FileInfo, len(names))
+
+	for i, name := range names {
+		fi, _, err := l.fs.LstatIfPossible(filepath.Join(l.Name(), name))
+
+		if err != nil {
+			return nil, err
+		}
+		fis[i] = fi
+	}
+
+	return fis, err
+}
+
+type LanguageFs struct {
+	// This Fs is usually created with a BasePathFs
+	basePath   string
+	lang       string
+	nameMarker string
+	languages  map[string]bool
+	afero.Fs
+}
+
+func NewLanguageFs(lang string, languages map[string]bool, fs afero.Fs) *LanguageFs {
+	if lang == "" {
+		panic("no lang set for the language fs")
+	}
+	var basePath string
+
+	if bfs, ok := fs.(*afero.BasePathFs); ok {
+		basePath, _ = bfs.RealPath("")
+	}
+
+	marker := hugoFsMarker + "_" + lang + "_"
+
+	return &LanguageFs{lang: lang, languages: languages, basePath: basePath, Fs: fs, nameMarker: marker}
+}
+
+func (fs *LanguageFs) Lang() string {
+	return fs.lang
+}
+
+func (fs *LanguageFs) Stat(name string) (os.FileInfo, error) {
+	name, err := fs.realName(name)
+	if err != nil {
+		return nil, err
+	}
+
+	fi, err := fs.Fs.Stat(name)
+	if err != nil {
+		return nil, err
+	}
+
+	return fs.newLanguageFileInfo(name, fi)
+}
+
+func (fs *LanguageFs) Open(name string) (afero.File, error) {
+	name, err := fs.realName(name)
+	if err != nil {
+		return nil, err
+	}
+	f, err := fs.Fs.Open(name)
+
+	if err != nil {
+		return nil, err
+	}
+	return &languageFile{File: f, fs: fs}, nil
+}
+
+func (fs *LanguageFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
+	name, err := fs.realName(name)
+	if err != nil {
+		return nil, false, err
+	}
+
+	var fi os.FileInfo
+	var b bool
+
+	if lif, ok := fs.Fs.(afero.Lstater); ok {
+		fi, b, err = lif.LstatIfPossible(name)
+	} else {
+		fi, err = fs.Fs.Stat(name)
+	}
+
+	if err != nil {
+		return nil, b, err
+	}
+
+	lfi, err := fs.newLanguageFileInfo(name, fi)
+
+	return lfi, b, err
+}
+
+func (fs *LanguageFs) realPath(name string) (string, error) {
+	if baseFs, ok := fs.Fs.(*afero.BasePathFs); ok {
+		return baseFs.RealPath(name)
+	}
+	return name, nil
+}
+
+func (fs *LanguageFs) realName(name string) (string, error) {
+	if strings.Contains(name, hugoFsMarker) {
+		if !strings.Contains(name, fs.nameMarker) {
+			return "", os.ErrNotExist
+		}
+		return strings.Replace(name, fs.nameMarker, "", 1), nil
+	}
+
+	if fs.basePath == "" {
+		return name, nil
+	}
+
+	return strings.TrimPrefix(name, fs.basePath), nil
+}
+
+func (fs *LanguageFs) newLanguageFileInfo(filename string, fi os.FileInfo) (*LanguageFileInfo, error) {
+	filename = filepath.Clean(filename)
+	_, name := filepath.Split(filename)
+
+	realName := name
+	virtualName := name
+
+	realPath, err := fs.realPath(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	lang := fs.Lang()
+
+	baseNameNoExt := ""
+
+	if !fi.IsDir() {
+
+		// Try to extract the language from the file name.
+		// Any valid language identificator in the name will win over the
+		// language set on the file system, e.g. "mypost.en.md".
+		baseName := filepath.Base(name)
+		ext := filepath.Ext(baseName)
+		baseNameNoExt = baseName
+
+		if ext != "" {
+			baseNameNoExt = strings.TrimSuffix(baseNameNoExt, ext)
+		}
+
+		fileLangExt := filepath.Ext(baseNameNoExt)
+		fileLang := strings.TrimPrefix(fileLangExt, ".")
+
+		if fs.languages[fileLang] {
+			lang = fileLang
+		}
+
+		baseNameNoExt = strings.TrimSuffix(baseNameNoExt, fileLangExt)
+
+		// This connects the filename to the filesystem, not the language.
+		virtualName = baseNameNoExt + "." + lang + ext
+
+		name = fs.nameMarker + name
+	}
+
+	weight := 1
+	// If this file's language belongs in this directory, add some weight to it
+	// to make it more important.
+	if lang == fs.Lang() {
+		weight = 2
+	}
+
+	if fi.IsDir() {
+		// For directories we always want to start from the union view.
+		realPath = strings.TrimPrefix(realPath, fs.basePath)
+	}
+
+	return &LanguageFileInfo{
+		lang:                lang,
+		weight:              weight,
+		realFilename:        realPath,
+		realName:            realName,
+		relFilename:         strings.TrimPrefix(strings.TrimPrefix(realPath, fs.basePath), string(os.PathSeparator)),
+		name:                name,
+		virtualName:         virtualName,
+		translationBaseName: baseNameNoExt,
+		baseDir:             fs.basePath,
+		FileInfo:            fi}, nil
+}

hugofs/language_fs_test.go

@@ -0,0 +1,54 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLanguagFs(t *testing.T) {
+	languages := map[string]bool{
+		"sv": true,
+	}
+	base := filepath.FromSlash("/my/base")
+	assert := require.New(t)
+	m := afero.NewMemMapFs()
+	bfs := afero.NewBasePathFs(m, base)
+	lfs := NewLanguageFs("sv", languages, bfs)
+	assert.NotNil(lfs)
+	assert.Equal("sv", lfs.Lang())
+	err := afero.WriteFile(lfs, filepath.FromSlash("sect/page.md"), []byte("abc"), 0777)
+	assert.NoError(err)
+	fi, err := lfs.Stat(filepath.FromSlash("sect/page.md"))
+	assert.NoError(err)
+	assert.Equal("__hugofs_sv_page.md", fi.Name())
+
+	languager, ok := fi.(LanguageAnnouncer)
+	assert.True(ok)
+
+	assert.Equal("sv", languager.Lang())
+
+	lfi, ok := fi.(*LanguageFileInfo)
+	assert.True(ok)
+	assert.Equal(filepath.FromSlash("/my/base/sect/page.md"), lfi.Filename())
+	assert.Equal(filepath.FromSlash("sect/page.md"), lfi.Path())
+	assert.Equal("page.sv.md", lfi.virtualName)
+	assert.Equal("__hugofs_sv_page.md", lfi.Name())
+	assert.Equal("page.md", lfi.RealName())
+
+}