Add Hugo Modules

This commit implements Hugo Modules.

This is a broad subject, but some keywords include:

* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`,  `hugo mod get`,  `hugo mod graph`,  `hugo mod tidy`, and  `hugo mod vendor`.

All of the above is backed by Go Modules.

Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092

tpl/cast/docshelper.go

@@ -17,7 +17,7 @@ import (
 	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/docshelper"
-	"github.com/gohugoio/hugo/htesting"
+	"github.com/gohugoio/hugo/resources/page"
 	"github.com/gohugoio/hugo/tpl/internal"
 	"github.com/spf13/viper"
 )
@@ -30,7 +30,7 @@ func init() {
 			Cfg:                 viper.New(),
 			Log:                 loggers.NewErrorLogger(),
 			BuildStartListeners: &deps.Listeners{},
-			Site:                htesting.NewTestHugoSite(),
+			Site:                page.NewDummyHugoSite(newTestConfig()),
 		}
 
 		var namespaces internal.TemplateFuncsNamespaces
@@ -47,3 +47,9 @@ func init() {
 
 	docshelper.AddDocProvider("tpl", docsProvider)
 }
+
+func newTestConfig() *viper.Viper {
+	v := viper.New()
+	v.Set("contentDir", "content")
+	return v
+}

tpl/data/init_test.go

@@ -16,6 +16,7 @@ package data
 import (
 	"testing"
 
+	"github.com/gohugoio/hugo/langs"
 	"github.com/gohugoio/hugo/tpl/internal"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
@@ -27,6 +28,7 @@ func TestInit(t *testing.T) {
 
 	v := viper.New()
 	v.Set("contentDir", "content")
+	langs.LoadLanguageSettings(v, nil)
 
 	for _, nsf := range internal.TemplateFuncsNamespaceRegistry {
 		ns = nsf(newDeps(v))

tpl/data/resources_test.go

@@ -23,6 +23,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/gohugoio/hugo/modules"
+
 	"github.com/gohugoio/hugo/helpers"
 
 	"github.com/gohugoio/hugo/cache/filecache"
@@ -87,7 +89,7 @@ func getTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*httpt
 func TestScpGetRemote(t *testing.T) {
 	t.Parallel()
 	fs := new(afero.MemMapFs)
-	cache := filecache.NewCache(fs, 100)
+	cache := filecache.NewCache(fs, 100, "")
 
 	tests := []struct {
 		path    string
@@ -186,14 +188,19 @@ func newDeps(cfg config.Provider) *deps.Deps {
 	cfg.Set("layoutDir", "layouts")
 	cfg.Set("archetypeDir", "archetypes")
 
-	l := langs.NewLanguage("en", cfg)
-	l.Set("i18nDir", "i18n")
-	cs, err := helpers.NewContentSpec(l)
+	langs.LoadLanguageSettings(cfg, nil)
+	mod, err := modules.CreateProjectModule(cfg)
+	if err != nil {
+		panic(err)
+	}
+	cfg.Set("allModules", modules.Modules{mod})
+
+	cs, err := helpers.NewContentSpec(cfg)
 	if err != nil {
 		panic(err)
 	}
 
-	fs := hugofs.NewMem(l)
+	fs := hugofs.NewMem(cfg)
 	logger := loggers.NewErrorLogger()
 
 	p, err := helpers.NewPathSpec(fs, cfg)

tpl/hugo/init_test.go

@@ -16,17 +16,19 @@ package hugo
 import (
 	"testing"
 
-	"github.com/gohugoio/hugo/htesting"
-
 	"github.com/gohugoio/hugo/deps"
+	"github.com/gohugoio/hugo/resources/page"
 	"github.com/gohugoio/hugo/tpl/internal"
+	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
 )
 
 func TestInit(t *testing.T) {
 	var found bool
 	var ns *internal.TemplateFuncsNamespace
-	s := htesting.NewTestHugoSite()
+	v := viper.New()
+	v.Set("contentDir", "content")
+	s := page.NewDummyHugoSite(v)
 
 	for _, nsf := range internal.TemplateFuncsNamespaceRegistry {
 		ns = nsf(&deps.Deps{Site: s})

tpl/os/os.go

@@ -18,6 +18,7 @@ package os
 import (
 	"errors"
 	"fmt"
+	"os"
 	_os "os"
 
 	"github.com/gohugoio/hugo/deps"
@@ -26,23 +27,20 @@ import (
 )
 
 // New returns a new instance of the os-namespaced template functions.
-func New(deps *deps.Deps) *Namespace {
+func New(d *deps.Deps) *Namespace {
 
-	// Since Hugo 0.38 we can have multiple content dirs. This can make it hard to
-	// reason about where the file is placed relative to the project root.
-	// To make the {{ readFile .Filename }} variant just work, we create a composite
-	// filesystem that first checks the work dir fs and then the content fs.
 	var rfs afero.Fs
-	if deps.Fs != nil {
-		rfs = deps.Fs.WorkingDir
-		if deps.PathSpec != nil && deps.PathSpec.BaseFs != nil {
-			rfs = afero.NewReadOnlyFs(afero.NewCopyOnWriteFs(deps.PathSpec.BaseFs.Content.Fs, deps.Fs.WorkingDir))
+	if d.Fs != nil {
+		rfs = d.Fs.WorkingDir
+		if d.PathSpec != nil && d.PathSpec.BaseFs != nil {
+			rfs = afero.NewReadOnlyFs(afero.NewCopyOnWriteFs(d.PathSpec.BaseFs.Content.Fs, d.Fs.WorkingDir))
 		}
+
 	}
 
 	return &Namespace{
 		readFileFs: rfs,
-		deps:       deps,
+		deps:       d,
 	}
 }
 
@@ -76,6 +74,9 @@ func readFile(fs afero.Fs, filename string) (string, error) {
 			return "", fmt.Errorf("file %q is too big", filename)
 		}
 	} else {
+		if os.IsNotExist(err) {
+			return "", fmt.Errorf("file %q does not exist", filename)
+		}
 		return "", err
 	}
 	b, err := afero.ReadFile(fs, filename)
@@ -96,6 +97,10 @@ func (ns *Namespace) ReadFile(i interface{}) (string, error) {
 		return "", err
 	}
 
+	if ns.deps.PathSpec != nil {
+		s = ns.deps.PathSpec.RelPathify(s)
+	}
+
 	return readFile(ns.readFileFs, s)
 }
 

tpl/site/init_test.go

@@ -16,8 +16,10 @@ package site
 import (
 	"testing"
 
+	"github.com/spf13/viper"
+
 	"github.com/gohugoio/hugo/deps"
-	"github.com/gohugoio/hugo/htesting"
+	"github.com/gohugoio/hugo/resources/page"
 	"github.com/gohugoio/hugo/tpl/internal"
 	"github.com/stretchr/testify/require"
 )
@@ -25,7 +27,9 @@ import (
 func TestInit(t *testing.T) {
 	var found bool
 	var ns *internal.TemplateFuncsNamespace
-	s := htesting.NewTestHugoSite()
+	v := viper.New()
+	v.Set("contentDir", "content")
+	s := page.NewDummyHugoSite(v)
 
 	for _, nsf := range internal.TemplateFuncsNamespaceRegistry {
 		ns = nsf(&deps.Deps{Site: s})

tpl/template.go

@@ -252,12 +252,15 @@ func (t *TemplateAdapter) fileAndFilename(name string) (afero.File, string, erro
 	if err != nil {
 		return nil, "", err
 	}
-	f, err := fs.Open(filename)
+	fim := fi.(hugofs.FileMetaInfo)
+	meta := fim.Meta()
+
+	f, err := meta.Open()
 	if err != nil {
 		return nil, "", errors.Wrapf(err, "failed to open template file %q:", filename)
 	}
 
-	return f, fi.(hugofs.RealFilenameInfo).RealFilename(), nil
+	return f, meta.Filename(), nil
 }
 
 // ExecuteToString executes the current template and returns the result as a

tpl/tplimpl/embedded/templates.autogen.go

@@ -268,8 +268,7 @@ if (!doNotTrack) {
     </li>
     {{ end }}
 </ul>
-{{ end }}
-`},
+{{ end }}`},
 	{`schema.html`, `<meta itemprop="name" content="{{ .Title }}">
 <meta itemprop="description" content="{{ with .Description }}{{ . }}{{ else }}{{if .IsPage}}{{ .Summary }}{{ else }}{{ with .Site.Params.description }}{{ . }}{{ end }}{{ end }}{{ end }}">
 

tpl/tplimpl/template.go

@@ -711,7 +711,7 @@ func (t *templateHandler) RebuildClone() {
 
 func (t *templateHandler) loadTemplates(prefix string) error {
 
-	walker := func(path string, fi os.FileInfo, err error) error {
+	walker := func(path string, fi hugofs.FileMetaInfo, err error) error {
 		if err != nil || fi.IsDir() {
 			return err
 		}
@@ -928,8 +928,8 @@ func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) e
 
 		realFilename := filename
 		if fi, err := fs.Stat(filename); err == nil {
-			if fir, ok := fi.(hugofs.RealFilenameInfo); ok {
-				realFilename = fir.RealFilename()
+			if fim, ok := fi.(hugofs.FileMetaInfo); ok {
+				realFilename = fim.Meta().Filename()
 			}
 		}
 

tpl/tplimpl/template_funcs_test.go

@@ -21,7 +21,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/gohugoio/hugo/htesting"
+	"github.com/gohugoio/hugo/modules"
+
+	"github.com/gohugoio/hugo/resources/page"
 
 	"github.com/gohugoio/hugo/common/hugo"
 	"github.com/gohugoio/hugo/common/loggers"
@@ -52,6 +54,14 @@ func newTestConfig() config.Provider {
 	v.Set("assetDir", "assets")
 	v.Set("resourceDir", "resources")
 	v.Set("publishDir", "public")
+
+	langs.LoadLanguageSettings(v, nil)
+	mod, err := modules.CreateProjectModule(v)
+	if err != nil {
+		panic(err)
+	}
+	v.Set("allModules", modules.Modules{mod})
+
 	return v
 }
 
@@ -59,7 +69,7 @@ func newDepsConfig(cfg config.Provider) deps.DepsCfg {
 	l := langs.NewLanguage("en", cfg)
 	return deps.DepsCfg{
 		Language:            l,
-		Site:                htesting.NewTestHugoSite(),
+		Site:                page.NewDummyHugoSite(cfg),
 		Cfg:                 cfg,
 		Fs:                  hugofs.NewMem(l),
 		Logger:              logger,