Add support for a content dir set per language

A sample config:

```toml
defaultContentLanguage = "en"
defaultContentLanguageInSubdir = true

[Languages]
[Languages.en]
weight = 10
title = "In English"
languageName = "English"
contentDir = "content/english"

[Languages.nn]
weight = 20
title = "På Norsk"
languageName = "Norsk"
contentDir = "content/norwegian"
```

The value of `contentDir` can be any valid path, even absolute path references. The only restriction is that the content dirs cannot overlap.

The content files will be assigned a language by

1. The placement: `content/norwegian/post/my-post.md` will be read as Norwegian content.
2. The filename: `content/english/post/my-post.nn.md` will be read as Norwegian even if it lives in the English content folder.

The content directories will be merged into a big virtual filesystem with one simple rule: The most specific language file will win.
This means that if both `content/norwegian/post/my-post.md` and `content/english/post/my-post.nn.md` exists, they will be considered duplicates and the version inside `content/norwegian` will win.

Note that translations will be automatically assigned by Hugo by the content file's relative placement, so `content/norwegian/post/my-post.md` will be a translation of `content/english/post/my-post.md`.

If this does not work for you, you can connect the translations together by setting a `translationKey` in the content files' front matter.

Fixes #4523
Fixes #4552
Fixes #4553

source/content_directory_test.go

@@ -17,11 +17,15 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/gohugoio/hugo/helpers"
+
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
 )
 
 func TestIgnoreDotFilesAndDirectories(t *testing.T) {
+	assert := require.New(t)
 
 	tests := []struct {
 		path                string
@@ -35,7 +39,6 @@ func TestIgnoreDotFilesAndDirectories(t *testing.T) {
 		{"foobar/.barfoo.md", true, nil},
 		{".barfoo.md", true, nil},
 		{".md", true, nil},
-		{"", true, nil},
 		{"foobar/barfoo.md~", true, nil},
 		{".foobar/barfoo.md~", true, nil},
 		{"foobar~/barfoo.md", false, nil},
@@ -51,9 +54,13 @@ func TestIgnoreDotFilesAndDirectories(t *testing.T) {
 	for i, test := range tests {
 
 		v := viper.New()
+		v.Set("contentDir", "content")
 		v.Set("ignoreFiles", test.ignoreFilesRegexpes)
+		fs := hugofs.NewMem(v)
+		ps, err := helpers.NewPathSpec(fs, v)
+		assert.NoError(err)
 
-		s := NewSourceSpec(v, hugofs.NewMem(v))
+		s := NewSourceSpec(ps, fs.Source)
 
 		if ignored := s.IgnoreFile(filepath.FromSlash(test.path)); test.ignore != ignored {
 			t.Errorf("[%d] File not ignored", i)

source/dirs_test.go

@@ -101,6 +101,8 @@ func TestStaticDirs(t *testing.T) {
 	for i, test := range tests {
 		msg := fmt.Sprintf("Test %d", i)
 		v := viper.New()
+		v.Set("contentDir", "content")
+
 		fs := hugofs.NewMem(v)
 		cfg := test.setup(v, fs)
 		cfg.Set("workingDir", filepath.FromSlash("/work"))
@@ -134,6 +136,7 @@ func TestStaticDirsFs(t *testing.T) {
 	v.Set("workingDir", filepath.FromSlash("/work"))
 	v.Set("theme", "mytheme")
 	v.Set("themesDir", "themes")
+	v.Set("contentDir", "content")
 	v.Set("staticDir", []string{"s1", "s2"})
 	v.Set("languagesSorted", helpers.Languages{helpers.NewDefaultLanguage(v)})
 

source/fileInfo.go

@@ -14,12 +14,17 @@
 package source
 
 import (
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 	"strings"
 	"sync"
 
+	"github.com/spf13/afero"
+
+	"github.com/gohugoio/hugo/hugofs"
+
 	"github.com/gohugoio/hugo/helpers"
 )
 
@@ -86,7 +91,10 @@ type FileInfo struct {
 
 	// Absolute filename to the file on disk.
 	filename string
-	fi       os.FileInfo
+
+	sp *SourceSpec
+
+	fi os.FileInfo
 
 	// Derived from filename
 	ext  string // Extension without any "."
@@ -104,8 +112,6 @@ type FileInfo struct {
 
 	uniqueID string
 
-	sp *SourceSpec
-
 	lazyInit sync.Once
 }
 
@@ -146,7 +152,6 @@ func (fi *FileInfo) init() {
 	fi.lazyInit.Do(func() {
 		relDir := strings.Trim(fi.relDir, helpers.FilePathSeparator)
 		parts := strings.Split(relDir, helpers.FilePathSeparator)
-
 		var section string
 		if (!fi.isLeafBundle && len(parts) == 1) || len(parts) > 1 {
 			section = parts[0]
@@ -161,6 +166,19 @@ func (fi *FileInfo) init() {
 
 func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, fi os.FileInfo) *FileInfo {
 
+	var lang, translationBaseName, relPath string
+
+	if fp, ok := fi.(hugofs.FilePather); ok {
+		filename = fp.Filename()
+		baseDir = fp.BaseDir()
+		relPath = fp.Path()
+	}
+
+	if fl, ok := fi.(hugofs.LanguageAnnouncer); ok {
+		lang = fl.Lang()
+		translationBaseName = fl.TranslationBaseName()
+	}
+
 	dir, name := filepath.Split(filename)
 	if !strings.HasSuffix(dir, helpers.FilePathSeparator) {
 		dir = dir + helpers.FilePathSeparator
@@ -175,19 +193,20 @@ func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, f
 
 	relDir = strings.TrimPrefix(relDir, helpers.FilePathSeparator)
 
-	relPath := filepath.Join(relDir, name)
+	if relPath == "" {
+		relPath = filepath.Join(relDir, name)
+	}
 
 	ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), "."))
 	baseName := helpers.Filename(name)
 
-	lang := strings.TrimPrefix(filepath.Ext(baseName), ".")
-	var translationBaseName string
-
-	if _, ok := sp.Languages[lang]; lang == "" || !ok {
-		lang = sp.DefaultContentLanguage
-		translationBaseName = baseName
-	} else {
-		translationBaseName = helpers.Filename(baseName)
+	if translationBaseName == "" {
+		// This is usyally provided by the filesystem. But this FileInfo is also
+		// created in a standalone context when doing "hugo new". This is
+		// an approximate implementation, which is "good enough" in that case.
+		translationBaseName = strings.TrimSuffix(baseName, ext)
+		fileLangExt := filepath.Ext(translationBaseName)
+		translationBaseName = strings.TrimSuffix(translationBaseName, fileLangExt)
 	}
 
 	f := &FileInfo{
@@ -211,5 +230,27 @@ func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, f
 
 // Open implements ReadableFile.
 func (fi *FileInfo) Open() (io.ReadCloser, error) {
-	return fi.sp.Fs.Source.Open(fi.Filename())
+	f, err := fi.sp.PathSpec.Fs.Source.Open(fi.Filename())
+	return f, err
+}
+
+func printFs(fs afero.Fs, path string, w io.Writer) {
+	if fs == nil {
+		return
+	}
+	afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {
+
+		if info != nil && !info.IsDir() {
+
+			s := path
+			if lang, ok := info.(hugofs.LanguageAnnouncer); ok {
+				s = s + "\t" + lang.Lang()
+			}
+			if fp, ok := info.(hugofs.FilePather); ok {
+				s = s + "\t" + fp.Filename()
+			}
+			fmt.Fprintln(w, "    ", s)
+		}
+		return nil
+	})
 }

source/fileInfo_test.go

@@ -17,6 +17,12 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/gohugoio/hugo/helpers"
+
+	"github.com/spf13/viper"
+
+	"github.com/gohugoio/hugo/hugofs"
+	"github.com/spf13/afero"
 	"github.com/stretchr/testify/require"
 )
 
@@ -35,6 +41,8 @@ func TestFileInfo(t *testing.T) {
 			assert.Equal(filepath.FromSlash("b/"), f.Dir())
 			assert.Equal(filepath.FromSlash("b/page.md"), f.Path())
 			assert.Equal("b", f.Section())
+			assert.Equal(filepath.FromSlash("page"), f.TranslationBaseName())
+			assert.Equal(filepath.FromSlash("page"), f.BaseFileName())
 
 		}},
 		{filepath.FromSlash("/a/"), filepath.FromSlash("/a/b/c/d/page.md"), func(f *FileInfo) {
@@ -47,3 +55,39 @@ func TestFileInfo(t *testing.T) {
 	}
 
 }
+
+func TestFileInfoLanguage(t *testing.T) {
+	assert := require.New(t)
+	langs := map[string]bool{
+		"sv": true,
+		"en": true,
+	}
+
+	m := afero.NewMemMapFs()
+	lfs := hugofs.NewLanguageFs("sv", langs, m)
+	v := viper.New()
+	v.Set("contentDir", "content")
+
+	fs := hugofs.NewFrom(m, v)
+
+	ps, err := helpers.NewPathSpec(fs, v)
+	assert.NoError(err)
+	s := SourceSpec{Fs: lfs, PathSpec: ps}
+	s.Languages = map[string]interface{}{
+		"en": true,
+	}
+
+	err = afero.WriteFile(lfs, "page.md", []byte("abc"), 0777)
+	assert.NoError(err)
+	err = afero.WriteFile(lfs, "page.en.md", []byte("abc"), 0777)
+	assert.NoError(err)
+
+	sv, _ := lfs.Stat("page.md")
+	en, _ := lfs.Stat("page.en.md")
+
+	fiSv := s.NewFileInfo("", "page.md", false, sv)
+	fiEn := s.NewFileInfo("", "page.en.md", false, en)
+
+	assert.Equal("sv", fiSv.Lang())
+	assert.Equal("en", fiEn.Lang())
+}

source/filesystem.go

@@ -82,11 +82,11 @@ func (f *Filesystem) captureFiles() {
 	if f.Fs == nil {
 		panic("Must have a fs")
 	}
-	err := helpers.SymbolicWalk(f.Fs.Source, f.Base, walker)
+	err := helpers.SymbolicWalk(f.Fs, f.Base, walker)
 
 	if err != nil {
 		jww.ERROR.Println(err)
-		if err == helpers.ErrWalkRootTooShort {
+		if err == helpers.ErrPathTooShort {
 			panic("The root path is too short. If this is a test, make sure to init the content paths.")
 		}
 	}
@@ -100,7 +100,7 @@ func (f *Filesystem) shouldRead(filename string, fi os.FileInfo) (bool, error) {
 			jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filename, err)
 			return false, nil
 		}
-		linkfi, err := f.Fs.Source.Stat(link)
+		linkfi, err := f.Fs.Stat(link)
 		if err != nil {
 			jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err)
 			return false, nil

source/filesystem_test.go

@@ -18,6 +18,8 @@ import (
 	"runtime"
 	"testing"
 
+	"github.com/gohugoio/hugo/helpers"
+
 	"github.com/gohugoio/hugo/hugofs"
 
 	"github.com/spf13/viper"
@@ -69,5 +71,7 @@ func TestUnicodeNorm(t *testing.T) {
 
 func newTestSourceSpec() SourceSpec {
 	v := viper.New()
-	return SourceSpec{Fs: hugofs.NewMem(v), Cfg: v}
+	v.Set("contentDir", "content")
+	ps, _ := helpers.NewPathSpec(hugofs.NewMem(v), v)
+	return SourceSpec{Fs: hugofs.NewMem(v).Source, PathSpec: ps}
 }

source/sourceSpec.go

@@ -18,17 +18,18 @@ import (
 	"path/filepath"
 	"regexp"
 
-	"github.com/gohugoio/hugo/config"
+	"github.com/spf13/afero"
+
 	"github.com/gohugoio/hugo/helpers"
-	"github.com/gohugoio/hugo/hugofs"
 	"github.com/spf13/cast"
 )
 
 // SourceSpec abstracts language-specific file creation.
 // TODO(bep) rename to Spec
 type SourceSpec struct {
-	Cfg config.Provider
-	Fs  *hugofs.Fs
+	*helpers.PathSpec
+
+	Fs afero.Fs
 
 	// This is set if the ignoreFiles config is set.
 	ignoreFilesRe []*regexp.Regexp
@@ -38,8 +39,9 @@ type SourceSpec struct {
 	DisabledLanguages      map[string]bool
 }
 
-// NewSourceSpec initializes SourceSpec using languages from a given configuration.
-func NewSourceSpec(cfg config.Provider, fs *hugofs.Fs) *SourceSpec {
+// NewSourceSpec initializes SourceSpec using languages the given filesystem and PathSpec.
+func NewSourceSpec(ps *helpers.PathSpec, fs afero.Fs) *SourceSpec {
+	cfg := ps.Cfg
 	defaultLang := cfg.GetString("defaultContentLanguage")
 	languages := cfg.GetStringMap("languages")
 
@@ -69,10 +71,17 @@ func NewSourceSpec(cfg config.Provider, fs *hugofs.Fs) *SourceSpec {
 		}
 	}
 
-	return &SourceSpec{ignoreFilesRe: regexps, Cfg: cfg, Fs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet}
+	return &SourceSpec{ignoreFilesRe: regexps, PathSpec: ps, Fs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet}
 }
 
 func (s *SourceSpec) IgnoreFile(filename string) bool {
+	if filename == "" {
+		if _, ok := s.Fs.(*afero.OsFs); ok {
+			return true
+		}
+		return false
+	}
+
 	base := filepath.Base(filename)
 
 	if len(base) > 0 {
@@ -99,7 +108,7 @@ func (s *SourceSpec) IgnoreFile(filename string) bool {
 }
 
 func (s *SourceSpec) IsRegularSourceFile(filename string) (bool, error) {
-	fi, err := helpers.LstatIfOs(s.Fs.Source, filename)
+	fi, err := helpers.LstatIfPossible(s.Fs, filename)
 	if err != nil {
 		return false, err
 	}
@@ -110,7 +119,7 @@ func (s *SourceSpec) IsRegularSourceFile(filename string) (bool, error) {
 
 	if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
 		link, err := filepath.EvalSymlinks(filename)
-		fi, err = helpers.LstatIfOs(s.Fs.Source, link)
+		fi, err = helpers.LstatIfPossible(s.Fs, link)
 		if err != nil {
 			return false, err
 		}