Add support for multiple staticDirs

This commit adds support for multiple statDirs both on the global and language level.

A simple `config.toml` example:

```bash
staticDir = ["static1", "static2"]
[languages]
[languages.no]
staticDir = ["staticDir_override", "static_no"]
baseURL = "https://example.no"
languageName = "Norsk"
weight = 1
title = "På norsk"

[languages.en]
staticDir2 = "static_en"
baseURL = "https://example.com"
languageName = "English"
weight = 2
title = "In English"
```

In the above, with no theme used:

the English site will get its static files as a union of "static1", "static2" and "static_en". On file duplicates, the right-most version will win.
the Norwegian site will get its static files as a union of "staticDir_override" and "static_no".

This commit also concludes the Multihost support in #4027.

Fixes #36
Closes #4027

source/dirs.go

@@ -0,0 +1,191 @@
+// Copyright 2017 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package source
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/afero"
+
+	"github.com/gohugoio/hugo/config"
+	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/hugofs"
+	jww "github.com/spf13/jwalterweatherman"
+)
+
+// Dirs holds the source directories for a given build.
+// In case where there are more than one of a kind, the order matters:
+// It will be used to construct a union filesystem, so the right-most directory
+// will "win" on duplicates. Typically, the theme version will be the first.
+type Dirs struct {
+	logger   *jww.Notepad
+	pathSpec *helpers.PathSpec
+
+	staticDirs    []string
+	AbsStaticDirs []string
+
+	publishDir string
+}
+
+// NewDirs creates a new dirs with the given configuration and filesystem.
+func NewDirs(fs *hugofs.Fs, cfg config.Provider, logger *jww.Notepad) (*Dirs, error) {
+	ps, err := helpers.NewPathSpec(fs, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	d := &Dirs{pathSpec: ps, logger: logger}
+
+	return d, d.init(cfg)
+
+}
+
+func (d *Dirs) init(cfg config.Provider) error {
+
+	var (
+		statics []string
+	)
+
+	if d.pathSpec.Theme() != "" {
+		statics = append(statics, filepath.Join(d.pathSpec.ThemesDir(), d.pathSpec.Theme(), "static"))
+	}
+
+	_, isLanguage := cfg.(*helpers.Language)
+	languages, hasLanguages := cfg.Get("languagesSorted").(helpers.Languages)
+
+	if !isLanguage && !hasLanguages {
+		return errors.New("missing languagesSorted in config")
+	}
+
+	if !isLanguage {
+		// Merge all the static dirs.
+		for _, l := range languages {
+			addend, err := d.staticDirsFor(l)
+			if err != nil {
+				return err
+			}
+
+			statics = append(statics, addend...)
+		}
+	} else {
+		addend, err := d.staticDirsFor(cfg)
+		if err != nil {
+			return err
+		}
+
+		statics = append(statics, addend...)
+	}
+
+	d.staticDirs = removeDuplicatesKeepRight(statics)
+	d.AbsStaticDirs = make([]string, len(d.staticDirs))
+	for i, di := range d.staticDirs {
+		d.AbsStaticDirs[i] = d.pathSpec.AbsPathify(di) + helpers.FilePathSeparator
+	}
+
+	d.publishDir = d.pathSpec.AbsPathify(cfg.GetString("publishDir")) + helpers.FilePathSeparator
+
+	return nil
+}
+
+func (d *Dirs) staticDirsFor(cfg config.Provider) ([]string, error) {
+	var statics []string
+	ps, err := helpers.NewPathSpec(d.pathSpec.Fs, cfg)
+	if err != nil {
+		return statics, err
+	}
+
+	statics = append(statics, ps.StaticDirs()...)
+
+	return statics, nil
+}
+
+// CreateStaticFs will create a union filesystem with the static paths configured.
+// Any missing directories will be logged as warnings.
+func (d *Dirs) CreateStaticFs() (afero.Fs, error) {
+	var (
+		source   = d.pathSpec.Fs.Source
+		absPaths []string
+	)
+
+	for _, staticDir := range d.AbsStaticDirs {
+		if _, err := source.Stat(staticDir); os.IsNotExist(err) {
+			d.logger.WARN.Printf("Unable to find Static Directory: %s", staticDir)
+		} else {
+			absPaths = append(absPaths, staticDir)
+		}
+
+	}
+
+	if len(absPaths) == 0 {
+		return nil, nil
+	}
+
+	return d.createOverlayFs(absPaths), nil
+
+}
+
+// IsStatic returns whether the given filename is located in one of the static
+// source dirs.
+func (d *Dirs) IsStatic(filename string) bool {
+	for _, absPath := range d.AbsStaticDirs {
+		if strings.HasPrefix(filename, absPath) {
+			return true
+		}
+	}
+	return false
+}
+
+// MakeStaticPathRelative creates a relative path from the given filename.
+// It will return an empty string if the filename is not a member of dirs.
+func (d *Dirs) MakeStaticPathRelative(filename string) string {
+	for _, currentPath := range d.AbsStaticDirs {
+		if strings.HasPrefix(filename, currentPath) {
+			return strings.TrimPrefix(filename, currentPath)
+		}
+	}
+
+	return ""
+
+}
+
+func (d *Dirs) createOverlayFs(absPaths []string) afero.Fs {
+	source := d.pathSpec.Fs.Source
+
+	if len(absPaths) == 1 {
+		return afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0]))
+	}
+
+	base := afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0]))
+	overlay := d.createOverlayFs(absPaths[1:])
+
+	return afero.NewCopyOnWriteFs(base, overlay)
+}
+
+func removeDuplicatesKeepRight(in []string) []string {
+	seen := make(map[string]bool)
+	var out []string
+	for i := len(in) - 1; i >= 0; i-- {
+		v := in[i]
+		if seen[v] {
+			continue
+		}
+		out = append([]string{v}, out...)
+		seen[v] = true
+	}
+
+	return out
+}

source/dirs_test.go

@@ -0,0 +1,177 @@
+// Copyright 2017 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package source
+
+import (
+	"testing"
+
+	"github.com/gohugoio/hugo/helpers"
+
+	"fmt"
+
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/gohugoio/hugo/config"
+	"github.com/spf13/afero"
+
+	jww "github.com/spf13/jwalterweatherman"
+
+	"github.com/gohugoio/hugo/hugofs"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+)
+
+var logger = jww.NewNotepad(jww.LevelInfo, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime)
+
+func TestStaticDirs(t *testing.T) {
+	assert := require.New(t)
+
+	tests := []struct {
+		setup    func(cfg config.Provider, fs *hugofs.Fs) config.Provider
+		expected []string
+	}{
+
+		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
+			cfg.Set("staticDir", "s1")
+			return cfg
+		}, []string{"s1"}},
+		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
+			cfg.Set("staticDir", []string{"s2", "s1", "s2"})
+			return cfg
+		}, []string{"s1", "s2"}},
+		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
+			cfg.Set("theme", "mytheme")
+			cfg.Set("themesDir", "themes")
+			cfg.Set("staticDir", []string{"s1", "s2"})
+			return cfg
+		}, []string{filepath.FromSlash("themes/mytheme/static"), "s1", "s2"}},
+		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
+			cfg.Set("staticDir", "s1")
+
+			l1 := helpers.NewLanguage("en", cfg)
+			l1.Set("staticDir", []string{"l1s1", "l1s2"})
+			return l1
+
+		}, []string{"l1s1", "l1s2"}},
+		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
+			cfg.Set("staticDir", "s1")
+
+			l1 := helpers.NewLanguage("en", cfg)
+			l1.Set("staticDir2", []string{"l1s1", "l1s2"})
+			return l1
+
+		}, []string{"s1", "l1s1", "l1s2"}},
+		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
+			cfg.Set("staticDir", "s1")
+
+			l1 := helpers.NewLanguage("en", cfg)
+			l1.Set("staticDir2", []string{"l1s1", "l1s2"})
+			l2 := helpers.NewLanguage("nn", cfg)
+			l2.Set("staticDir3", []string{"l2s1", "l2s2"})
+			l2.Set("staticDir", []string{"l2"})
+
+			cfg.Set("languagesSorted", helpers.Languages{l1, l2})
+			return cfg
+
+		}, []string{"s1", "l1s1", "l1s2", "l2", "l2s1", "l2s2"}},
+	}
+
+	for i, test := range tests {
+		if i != 0 {
+			break
+		}
+		msg := fmt.Sprintf("Test %d", i)
+		v := viper.New()
+		fs := hugofs.NewMem(v)
+		cfg := test.setup(v, fs)
+		cfg.Set("workingDir", filepath.FromSlash("/work"))
+		_, isLanguage := cfg.(*helpers.Language)
+		if !isLanguage && !cfg.IsSet("languagesSorted") {
+			cfg.Set("languagesSorted", helpers.Languages{helpers.NewDefaultLanguage(cfg)})
+		}
+		dirs, err := NewDirs(fs, cfg, logger)
+		assert.NoError(err)
+		assert.Equal(test.expected, dirs.staticDirs, msg)
+		assert.Len(dirs.AbsStaticDirs, len(dirs.staticDirs))
+
+		for i, d := range dirs.staticDirs {
+			abs := dirs.AbsStaticDirs[i]
+			assert.Equal(filepath.Join("/work", d)+helpers.FilePathSeparator, abs)
+			assert.True(dirs.IsStatic(filepath.Join(abs, "logo.png")))
+			rel := dirs.MakeStaticPathRelative(filepath.Join(abs, "logo.png"))
+			assert.Equal("logo.png", rel)
+		}
+
+		assert.False(dirs.IsStatic(filepath.FromSlash("/some/other/dir/logo.png")))
+
+	}
+
+}
+
+func TestStaticDirsFs(t *testing.T) {
+	assert := require.New(t)
+	v := viper.New()
+	fs := hugofs.NewMem(v)
+	v.Set("workingDir", filepath.FromSlash("/work"))
+	v.Set("theme", "mytheme")
+	v.Set("themesDir", "themes")
+	v.Set("staticDir", []string{"s1", "s2"})
+	v.Set("languagesSorted", helpers.Languages{helpers.NewDefaultLanguage(v)})
+
+	writeToFs(t, fs.Source, "/work/s1/f1.txt", "s1-f1")
+	writeToFs(t, fs.Source, "/work/s2/f2.txt", "s2-f2")
+	writeToFs(t, fs.Source, "/work/s1/f2.txt", "s1-f2")
+	writeToFs(t, fs.Source, "/work/themes/mytheme/static/f1.txt", "theme-f1")
+	writeToFs(t, fs.Source, "/work/themes/mytheme/static/f3.txt", "theme-f3")
+
+	dirs, err := NewDirs(fs, v, logger)
+	assert.NoError(err)
+
+	sfs, err := dirs.CreateStaticFs()
+	assert.NoError(err)
+
+	assert.Equal("s1-f1", readFileFromFs(t, sfs, "f1.txt"))
+	assert.Equal("s2-f2", readFileFromFs(t, sfs, "f2.txt"))
+	assert.Equal("theme-f3", readFileFromFs(t, sfs, "f3.txt"))
+
+}
+
+func TestRemoveDuplicatesKeepRight(t *testing.T) {
+	in := []string{"a", "b", "c", "a"}
+	out := removeDuplicatesKeepRight(in)
+
+	require.Equal(t, []string{"b", "c", "a"}, out)
+}
+
+func writeToFs(t testing.TB, fs afero.Fs, filename, content string) {
+	if err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte(content), 0755); err != nil {
+		t.Fatalf("Failed to write file: %s", err)
+	}
+}
+
+func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string {
+	filename = filepath.FromSlash(filename)
+	b, err := afero.ReadFile(fs, filename)
+	if err != nil {
+		afero.Walk(fs, "", func(path string, info os.FileInfo, err error) error {
+			fmt.Println("    ", path, " ", info)
+			return nil
+		})
+		t.Fatalf("Failed to read file: %s", err)
+	}
+	return string(b)
+}