Add support for a content dir set per language

A sample config:

```toml
defaultContentLanguage = "en"
defaultContentLanguageInSubdir = true

[Languages]
[Languages.en]
weight = 10
title = "In English"
languageName = "English"
contentDir = "content/english"

[Languages.nn]
weight = 20
title = "På Norsk"
languageName = "Norsk"
contentDir = "content/norwegian"
```

The value of `contentDir` can be any valid path, even absolute path references. The only restriction is that the content dirs cannot overlap.

The content files will be assigned a language by

1. The placement: `content/norwegian/post/my-post.md` will be read as Norwegian content.
2. The filename: `content/english/post/my-post.nn.md` will be read as Norwegian even if it lives in the English content folder.

The content directories will be merged into a big virtual filesystem with one simple rule: The most specific language file will win.
This means that if both `content/norwegian/post/my-post.md` and `content/english/post/my-post.nn.md` exists, they will be considered duplicates and the version inside `content/norwegian` will win.

Note that translations will be automatically assigned by Hugo by the content file's relative placement, so `content/norwegian/post/my-post.md` will be a translation of `content/english/post/my-post.md`.

If this does not work for you, you can connect the translations together by setting a `translationKey` in the content files' front matter.

Fixes #4523
Fixes #4552
Fixes #4553

helpers/language.go

@@ -41,6 +41,14 @@ type Language struct {
 	Title        string
 	Weight       int
 
+	Disabled bool
+
+	// If set per language, this tells Hugo that all content files without any
+	// language indicator (e.g. my-page.en.md) is in this language.
+	// This is usually a path relative to the working dir, but it can be an
+	// absolute directory referenece. It is what we get.
+	ContentDir string
+
 	Cfg config.Provider
 
 	// These are params declared in the [params] section of the language merged with the
@@ -66,7 +74,13 @@ func NewLanguage(lang string, cfg config.Provider) *Language {
 		params[k] = v
 	}
 	ToLowerMap(params)
-	l := &Language{Lang: lang, Cfg: cfg, params: params, settings: make(map[string]interface{})}
+
+	defaultContentDir := cfg.GetString("contentDir")
+	if defaultContentDir == "" {
+		panic("contentDir not set")
+	}
+
+	l := &Language{Lang: lang, ContentDir: defaultContentDir, Cfg: cfg, params: params, settings: make(map[string]interface{})}
 	return l
 }
 

helpers/language_test.go

@@ -22,11 +22,12 @@ import (
 
 func TestGetGlobalOnlySetting(t *testing.T) {
 	v := viper.New()
+	v.Set("defaultContentLanguageInSubdir", true)
+	v.Set("contentDir", "content")
+	v.Set("paginatePath", "page")
 	lang := NewDefaultLanguage(v)
 	lang.Set("defaultContentLanguageInSubdir", false)
 	lang.Set("paginatePath", "side")
-	v.Set("defaultContentLanguageInSubdir", true)
-	v.Set("paginatePath", "page")
 
 	require.True(t, lang.GetBool("defaultContentLanguageInSubdir"))
 	require.Equal(t, "side", lang.GetString("paginatePath"))
@@ -37,6 +38,7 @@ func TestLanguageParams(t *testing.T) {
 
 	v := viper.New()
 	v.Set("p1", "p1cfg")
+	v.Set("contentDir", "content")
 
 	lang := NewDefaultLanguage(v)
 	lang.SetParam("p1", "p1p")

helpers/path.go

@@ -33,7 +33,7 @@ var (
 	ErrThemeUndefined = errors.New("no theme set")
 
 	// ErrWalkRootTooShort is returned when the root specified for a file walk is shorter than 4 characters.
-	ErrWalkRootTooShort = errors.New("Path too short. Stop walking.")
+	ErrPathTooShort = errors.New("file path is too short")
 )
 
 // filepathPathBridge is a bridge for common functionality in filepath vs path
@@ -446,7 +446,7 @@ func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error {
 
 	// Sanity check
 	if len(root) < 4 {
-		return ErrWalkRootTooShort
+		return ErrPathTooShort
 	}
 
 	// Handle the root first
@@ -481,7 +481,7 @@ func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error {
 }
 
 func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) {
-	fileInfo, err := LstatIfOs(fs, path)
+	fileInfo, err := LstatIfPossible(fs, path)
 	realPath := path
 
 	if err != nil {
@@ -493,7 +493,7 @@ func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) {
 		if err != nil {
 			return nil, "", fmt.Errorf("Cannot read symbolic link '%s', error was: %s", path, err)
 		}
-		fileInfo, err = LstatIfOs(fs, link)
+		fileInfo, err = LstatIfPossible(fs, link)
 		if err != nil {
 			return nil, "", fmt.Errorf("Cannot stat '%s', error was: %s", link, err)
 		}
@@ -514,16 +514,14 @@ func GetRealPath(fs afero.Fs, path string) (string, error) {
 	return realPath, nil
 }
 
-// Code copied from Afero's path.go
-// if the filesystem is OsFs use Lstat, else use fs.Stat
-func LstatIfOs(fs afero.Fs, path string) (info os.FileInfo, err error) {
-	_, ok := fs.(*afero.OsFs)
-	if ok {
-		info, err = os.Lstat(path)
-	} else {
-		info, err = fs.Stat(path)
+// LstatIfPossible can be used to call Lstat if possible, else Stat.
+func LstatIfPossible(fs afero.Fs, path string) (os.FileInfo, error) {
+	if lstater, ok := fs.(afero.Lstater); ok {
+		fi, _, err := lstater.LstatIfPossible(path)
+		return fi, err
 	}
-	return
+
+	return fs.Stat(path)
 }
 
 // SafeWriteToDisk is the same as WriteToDisk

helpers/path_test.go

@@ -57,8 +57,10 @@ func TestMakePath(t *testing.T) {
 
 	for _, test := range tests {
 		v := viper.New()
-		l := NewDefaultLanguage(v)
+		v.Set("contentDir", "content")
 		v.Set("removePathAccents", test.removeAccents)
+
+		l := NewDefaultLanguage(v)
 		p, err := NewPathSpec(hugofs.NewMem(v), l)
 		require.NoError(t, err)
 
@@ -71,6 +73,8 @@ func TestMakePath(t *testing.T) {
 
 func TestMakePathSanitized(t *testing.T) {
 	v := viper.New()
+	v.Set("contentDir", "content")
+
 	l := NewDefaultLanguage(v)
 	p, _ := NewPathSpec(hugofs.NewMem(v), l)
 
@@ -98,6 +102,7 @@ func TestMakePathSanitizedDisablePathToLower(t *testing.T) {
 	v := viper.New()
 
 	v.Set("disablePathToLower", true)
+	v.Set("contentDir", "content")
 
 	l := NewDefaultLanguage(v)
 	p, _ := NewPathSpec(hugofs.NewMem(v), l)

helpers/pathspec.go

@@ -17,6 +17,9 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/spf13/afero"
+
+	"github.com/gohugoio/hugo/common/types"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/spf13/cast"
@@ -44,11 +47,13 @@ type PathSpec struct {
 	theme string
 
 	// Directories
-	contentDir string
-	themesDir  string
-	layoutDir  string
-	workingDir string
-	staticDirs []string
+	contentDir     string
+	themesDir      string
+	layoutDir      string
+	workingDir     string
+	staticDirs     []string
+	absContentDirs []types.KeyValueStr
+
 	PublishDir string
 
 	// The PathSpec looks up its config settings in both the current language
@@ -65,6 +70,9 @@ type PathSpec struct {
 	// The file systems to use
 	Fs *hugofs.Fs
 
+	// The fine grained filesystems in play (resources, content etc.).
+	BaseFs *hugofs.BaseFs
+
 	// The config provider to use
 	Cfg config.Provider
 }
@@ -105,8 +113,65 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) {
 		languages = l
 	}
 
+	defaultContentLanguage := cfg.GetString("defaultContentLanguage")
+
+	// We will eventually pull out this badly placed path logic.
+	contentDir := cfg.GetString("contentDir")
+	workingDir := cfg.GetString("workingDir")
+	resourceDir := cfg.GetString("resourceDir")
+	publishDir := cfg.GetString("publishDir")
+
+	if len(languages) == 0 {
+		// We have some old tests that does not test the entire chain, hence
+		// they have no languages. So create one so we get the proper filesystem.
+		languages = Languages{&Language{Lang: "en", ContentDir: contentDir}}
+	}
+
+	absPuslishDir := AbsPathify(workingDir, publishDir)
+	if !strings.HasSuffix(absPuslishDir, FilePathSeparator) {
+		absPuslishDir += FilePathSeparator
+	}
+	// If root, remove the second '/'
+	if absPuslishDir == "//" {
+		absPuslishDir = FilePathSeparator
+	}
+	absResourcesDir := AbsPathify(workingDir, resourceDir)
+	if !strings.HasSuffix(absResourcesDir, FilePathSeparator) {
+		absResourcesDir += FilePathSeparator
+	}
+	if absResourcesDir == "//" {
+		absResourcesDir = FilePathSeparator
+	}
+
+	contentFs, absContentDirs, err := createContentFs(fs.Source, workingDir, defaultContentLanguage, languages)
+	if err != nil {
+		return nil, err
+	}
+
+	// Make sure we don't have any overlapping content dirs. That will never work.
+	for i, d1 := range absContentDirs {
+		for j, d2 := range absContentDirs {
+			if i == j {
+				continue
+			}
+			if strings.HasPrefix(d1.Value, d2.Value) || strings.HasPrefix(d2.Value, d1.Value) {
+				return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2)
+			}
+		}
+	}
+
+	resourcesFs := afero.NewBasePathFs(fs.Source, absResourcesDir)
+	publishFs := afero.NewBasePathFs(fs.Destination, absPuslishDir)
+
+	baseFs := &hugofs.BaseFs{
+		ContentFs:   contentFs,
+		ResourcesFs: resourcesFs,
+		PublishFs:   publishFs,
+	}
+
 	ps := &PathSpec{
 		Fs:                             fs,
+		BaseFs:                         baseFs,
 		Cfg:                            cfg,
 		disablePathToLower:             cfg.GetBool("disablePathToLower"),
 		removePathAccents:              cfg.GetBool("removePathAccents"),
@@ -116,14 +181,15 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) {
 		Language:                       language,
 		Languages:                      languages,
 		defaultContentLanguageInSubdir: cfg.GetBool("defaultContentLanguageInSubdir"),
-		defaultContentLanguage:         cfg.GetString("defaultContentLanguage"),
+		defaultContentLanguage:         defaultContentLanguage,
 		paginatePath:                   cfg.GetString("paginatePath"),
 		BaseURL:                        baseURL,
-		contentDir:                     cfg.GetString("contentDir"),
+		contentDir:                     contentDir,
 		themesDir:                      cfg.GetString("themesDir"),
 		layoutDir:                      cfg.GetString("layoutDir"),
-		workingDir:                     cfg.GetString("workingDir"),
+		workingDir:                     workingDir,
 		staticDirs:                     staticDirs,
+		absContentDirs:                 absContentDirs,
 		theme:                          cfg.GetString("theme"),
 		ProcessingStats:                NewProcessingStats(lang),
 	}
@@ -135,13 +201,8 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) {
 		}
 	}
 
-	publishDir := ps.AbsPathify(cfg.GetString("publishDir")) + FilePathSeparator
-	// If root, remove the second '/'
-	if publishDir == "//" {
-		publishDir = FilePathSeparator
-	}
-
-	ps.PublishDir = publishDir
+	// TODO(bep) remove this, eventually
+	ps.PublishDir = absPuslishDir
 
 	return ps, nil
 }
@@ -165,6 +226,107 @@ func getStringOrStringSlice(cfg config.Provider, key string, id int) []string {
 	return out
 }
 
+func createContentFs(fs afero.Fs,
+	workingDir,
+	defaultContentLanguage string,
+	languages Languages) (afero.Fs, []types.KeyValueStr, error) {
+
+	var contentLanguages Languages
+	var contentDirSeen = make(map[string]bool)
+	languageSet := make(map[string]bool)
+
+	// The default content language needs to be first.
+	for _, language := range languages {
+		if language.Lang == defaultContentLanguage {
+			contentLanguages = append(contentLanguages, language)
+			contentDirSeen[language.ContentDir] = true
+		}
+		languageSet[language.Lang] = true
+	}
+
+	for _, language := range languages {
+		if contentDirSeen[language.ContentDir] {
+			continue
+		}
+		if language.ContentDir == "" {
+			language.ContentDir = defaultContentLanguage
+		}
+		contentDirSeen[language.ContentDir] = true
+		contentLanguages = append(contentLanguages, language)
+
+	}
+
+	var absContentDirs []types.KeyValueStr
+
+	fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs)
+	return fs, absContentDirs, err
+
+}
+
+func createContentOverlayFs(source afero.Fs,
+	workingDir string,
+	languages Languages,
+	languageSet map[string]bool,
+	absContentDirs *[]types.KeyValueStr) (afero.Fs, error) {
+	if len(languages) == 0 {
+		return source, nil
+	}
+
+	language := languages[0]
+
+	contentDir := language.ContentDir
+	if contentDir == "" {
+		panic("missing contentDir")
+	}
+
+	absContentDir := AbsPathify(workingDir, language.ContentDir)
+	if !strings.HasSuffix(absContentDir, FilePathSeparator) {
+		absContentDir += FilePathSeparator
+	}
+
+	// If root, remove the second '/'
+	if absContentDir == "//" {
+		absContentDir = FilePathSeparator
+	}
+
+	if len(absContentDir) < 6 {
+		return nil, fmt.Errorf("invalid content dir %q: %s", absContentDir, ErrPathTooShort)
+	}
+
+	*absContentDirs = append(*absContentDirs, types.KeyValueStr{Key: language.Lang, Value: absContentDir})
+
+	overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir))
+	if len(languages) == 1 {
+		return overlay, nil
+	}
+
+	base, err := createContentOverlayFs(source, workingDir, languages[1:], languageSet, absContentDirs)
+	if err != nil {
+		return nil, err
+	}
+
+	return hugofs.NewLanguageCompositeFs(base, overlay), nil
+
+}
+
+// RelContentDir tries to create a path relative to the content root from
+// the given filename. The return value is the path and language code.
+func (p *PathSpec) RelContentDir(filename string) (string, string) {
+	for _, dir := range p.absContentDirs {
+		if strings.HasPrefix(filename, dir.Value) {
+			rel := strings.TrimPrefix(filename, dir.Value)
+			return strings.TrimPrefix(rel, FilePathSeparator), dir.Key
+		}
+	}
+	// Either not a content dir or already relative.
+	return filename, ""
+}
+
+// ContentDirs returns all the content dirs (absolute paths).
+func (p *PathSpec) ContentDirs() []types.KeyValueStr {
+	return p.absContentDirs
+}
+
 // PaginatePath returns the configured root path used for paginator pages.
 func (p *PathSpec) PaginatePath() string {
 	return p.paginatePath

helpers/pathspec_test.go

@@ -24,6 +24,7 @@ import (
 
 func TestNewPathSpecFromConfig(t *testing.T) {
 	v := viper.New()
+	v.Set("contentDir", "content")
 	l := NewLanguage("no", v)
 	v.Set("disablePathToLower", true)
 	v.Set("removePathAccents", true)

helpers/testhelpers_test.go

@@ -25,6 +25,7 @@ func newTestDefaultPathSpec(configKeyValues ...interface{}) *PathSpec {
 
 func newTestCfg(fs *hugofs.Fs) *viper.Viper {
 	v := viper.New()
+	v.Set("contentDir", "content")
 
 	v.SetFs(fs.Source)
 

helpers/url_test.go

@@ -27,6 +27,7 @@ import (
 func TestURLize(t *testing.T) {
 
 	v := viper.New()
+	v.Set("contentDir", "content")
 	l := NewDefaultLanguage(v)
 	p, _ := NewPathSpec(hugofs.NewMem(v), l)
 
@@ -88,6 +89,7 @@ func doTestAbsURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool,
 
 	for _, test := range tests {
 		v.Set("baseURL", test.baseURL)
+		v.Set("contentDir", "content")
 		l := NewLanguage(lang, v)
 		p, _ := NewPathSpec(hugofs.NewMem(v), l)
 
@@ -166,6 +168,7 @@ func doTestRelURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool,
 	for i, test := range tests {
 		v.Set("baseURL", test.baseURL)
 		v.Set("canonifyURLs", test.canonify)
+		v.Set("contentDir", "content")
 		l := NewLanguage(lang, v)
 		p, _ := NewPathSpec(hugofs.NewMem(v), l)
 
@@ -254,6 +257,7 @@ func TestURLPrep(t *testing.T) {
 	for i, d := range data {
 		v := viper.New()
 		v.Set("uglyURLs", d.ugly)
+		v.Set("contentDir", "content")
 		l := NewDefaultLanguage(v)
 		p, _ := NewPathSpec(hugofs.NewMem(v), l)