Add Hugo Modules

This commit implements Hugo Modules.

This is a broad subject, but some keywords include:

* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`,  `hugo mod get`,  `hugo mod graph`,  `hugo mod tidy`, and  `hugo mod vendor`.

All of the above is backed by Go Modules.

Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092

resources/image_cache.go

@@ -145,7 +145,7 @@ func (c *imageCache) getOrCreate(
 	}
 
 	// The file is now stored in this cache.
-	img.overriddenSourceFs = c.fileCache.Fs
+	img.sourceFs = c.fileCache.Fs
 
 	c.mu.Lock()
 	if img2, found := c.store[key]; found {

resources/page/page_nop.go

@@ -17,9 +17,10 @@ package page
 
 import (
 	"html/template"
-	"os"
 	"time"
 
+	"github.com/gohugoio/hugo/hugofs"
+
 	"github.com/bep/gitmap"
 	"github.com/gohugoio/hugo/navigation"
 
@@ -147,7 +148,7 @@ func (p *nopPage) File() source.File {
 	return nilFile
 }
 
-func (p *nopPage) FileInfo() os.FileInfo {
+func (p *nopPage) FileInfo() hugofs.FileMetaInfo {
 	return nil
 }
 

resources/page/page_wrappers.autogen.go

@@ -18,8 +18,8 @@ package page
 import (
 	"github.com/gohugoio/hugo/common/hugo"
 	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/hugofs"
 	"html/template"
-	"os"
 )
 
 // NewDeprecatedWarningPage adds deprecation warnings to the given implementation.
@@ -91,7 +91,7 @@ func (p *pageDeprecated) UniqueID() string {
 	helpers.Deprecated("Page", ".UniqueID", "Use .File.UniqueID", false)
 	return p.p.UniqueID()
 }
-func (p *pageDeprecated) FileInfo() os.FileInfo {
+func (p *pageDeprecated) FileInfo() hugofs.FileMetaInfo {
 	helpers.Deprecated("Page", ".FileInfo", "Use .File.FileInfo", false)
 	return p.p.FileInfo()
 }

resources/page/pagemeta/page_frontmatter.go

@@ -236,7 +236,7 @@ func addDateFieldAliases(values []string) []string {
 			complete = append(complete, aliases...)
 		}
 	}
-	return helpers.UniqueStrings(complete)
+	return helpers.UniqueStringsReuse(complete)
 }
 
 func expandDefaultValues(values []string, defaults []string) []string {

resources/page/permalinks.go

@@ -15,6 +15,7 @@ package page
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -90,7 +91,11 @@ func (l PermalinkExpander) parse(patterns map[string]string) (map[string]func(Pa
 
 	expanders := make(map[string]func(Page) (string, error))
 
+	// Allow " " and / to represent the root section.
+	const sectionCutSet = " /" + string(os.PathSeparator)
+
 	for k, pattern := range patterns {
+		k = strings.Trim(k, sectionCutSet)
 		if !l.validate(pattern) {
 			return nil, &permalinkExpandError{pattern: pattern, err: errPermalinkIllFormed}
 		}

resources/page/site.go

@@ -17,6 +17,8 @@ import (
 	"html/template"
 	"time"
 
+	"github.com/gohugoio/hugo/config"
+
 	"github.com/gohugoio/hugo/common/hugo"
 	"github.com/gohugoio/hugo/langs"
 	"github.com/gohugoio/hugo/navigation"
@@ -51,3 +53,72 @@ func (s Sites) First() Site {
 	}
 	return s[0]
 }
+
+type testSite struct {
+	h hugo.Info
+	l *langs.Language
+}
+
+func (t testSite) Hugo() hugo.Info {
+	return t.h
+}
+
+func (t testSite) ServerPort() int {
+	return 1313
+}
+
+func (testSite) LastChange() (t time.Time) {
+	return
+}
+
+func (t testSite) Title() string {
+	return "foo"
+}
+
+func (t testSite) Sites() Sites {
+	return nil
+}
+
+func (t testSite) IsServer() bool {
+	return false
+}
+
+func (t testSite) Language() *langs.Language {
+	return t.l
+}
+
+func (t testSite) Pages() Pages {
+	return nil
+}
+
+func (t testSite) RegularPages() Pages {
+	return nil
+}
+
+func (t testSite) Menus() navigation.Menus {
+	return nil
+}
+
+func (t testSite) Taxonomies() interface{} {
+	return nil
+}
+
+func (t testSite) BaseURL() template.URL {
+	return ""
+}
+
+func (t testSite) Params() map[string]interface{} {
+	return nil
+}
+
+func (t testSite) Data() map[string]interface{} {
+	return nil
+}
+
+// NewDummyHugoSite creates a new minimal test site.
+func NewDummyHugoSite(cfg config.Provider) Site {
+	return testSite{
+		h: hugo.NewInfo(hugo.EnvironmentProduction),
+		l: langs.NewLanguage("en", cfg),
+	}
+}

resources/page/testhelpers_test.go

@@ -16,10 +16,11 @@ package page
 import (
 	"fmt"
 	"html/template"
-	"os"
 	"path/filepath"
 	"time"
 
+	"github.com/gohugoio/hugo/modules"
+
 	"github.com/bep/gitmap"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/resources/resource"
@@ -65,6 +66,12 @@ func newTestPathSpec() *helpers.PathSpec {
 
 func newTestPathSpecFor(cfg config.Provider) *helpers.PathSpec {
 	config.SetBaseTestDefaults(cfg)
+	langs.LoadLanguageSettings(cfg, nil)
+	mod, err := modules.CreateProjectModule(cfg)
+	if err != nil {
+		panic(err)
+	}
+	cfg.Set("allModules", modules.Modules{mod})
 	fs := hugofs.NewMem(cfg)
 	s, err := helpers.NewPathSpec(fs, cfg)
 	if err != nil {
@@ -189,7 +196,7 @@ func (p *testPage) File() source.File {
 	return p.file
 }
 
-func (p *testPage) FileInfo() os.FileInfo {
+func (p *testPage) FileInfo() hugofs.FileMetaInfo {
 	panic("not implemented")
 }
 

resources/page/zero_file.autogen.go

@@ -17,8 +17,8 @@ package page
 
 import (
 	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/source"
-	"os"
 )
 
 // ZeroFile represents a zero value of source.File with warnings if invoked.
@@ -82,7 +82,7 @@ func (z zeroFile) UniqueID() (o0 string) {
 	z.log.Println(".File.UniqueID on zero object. Wrap it in if or with: {{ with .File }}{{ .UniqueID }}{{ end }}")
 	return
 }
-func (z zeroFile) FileInfo() (o0 os.FileInfo) {
+func (z zeroFile) FileInfo() (o0 hugofs.FileMetaInfo) {
 	z.log.Println(".File.FileInfo on zero object. Wrap it in if or with: {{ with .File }}{{ .FileInfo }}{{ end }}")
 	return
 }

resources/resource.go

@@ -133,9 +133,13 @@ type ResourceSourceDescriptor struct {
 	SourceFile         source.File
 	OpenReadSeekCloser resource.OpenReadSeekCloser
 
+	FileInfo os.FileInfo
+
 	// If OpenReadSeekerCloser is not set, we use this to open the file.
 	SourceFilename string
 
+	Fs afero.Fs
+
 	// The relative target filename without any language code.
 	RelTargetFilename string
 
@@ -157,19 +161,11 @@ func (r ResourceSourceDescriptor) Filename() string {
 	return r.SourceFilename
 }
 
-func (r *Spec) sourceFs() afero.Fs {
-	return r.PathSpec.BaseFs.Content.Fs
-}
-
 func (r *Spec) New(fd ResourceSourceDescriptor) (resource.Resource, error) {
-	return r.newResourceForFs(r.sourceFs(), fd)
+	return r.newResourceFor(fd)
 }
 
-func (r *Spec) NewForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) {
-	return r.newResourceForFs(sourceFs, fd)
-}
-
-func (r *Spec) newResourceForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) {
+func (r *Spec) newResourceFor(fd ResourceSourceDescriptor) (resource.Resource, error) {
 	if fd.OpenReadSeekCloser == nil {
 		if fd.SourceFile != nil && fd.SourceFilename != "" {
 			return nil, errors.New("both SourceFile and AbsSourceFilename provided")
@@ -187,15 +183,14 @@ func (r *Spec) newResourceForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor)
 		fd.TargetBasePaths = r.MultihostTargetBasePaths
 	}
 
-	return r.newResource(sourceFs, fd)
+	return r.newResource(fd.Fs, fd)
 }
 
 func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) {
-	var fi os.FileInfo
+	fi := fd.FileInfo
 	var sourceFilename string
 
 	if fd.OpenReadSeekCloser != nil {
-
 	} else if fd.SourceFilename != "" {
 		var err error
 		fi, err = sourceFs.Stat(fd.SourceFilename)
@@ -207,7 +202,6 @@ func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (reso
 		}
 		sourceFilename = fd.SourceFilename
 	} else {
-		fi = fd.SourceFile.FileInfo()
 		sourceFilename = fd.SourceFile.Filename()
 	}
 
@@ -245,8 +239,6 @@ func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (reso
 		mimeType)
 
 	if mimeType.MainType == "image" {
-		ext := strings.ToLower(helpers.Ext(sourceFilename))
-
 		imgFormat, ok := imageFormats[ext]
 		if !ok {
 			// This allows SVG etc. to be used as resources. They will not have the methods of the Image, but
@@ -376,7 +368,7 @@ type genericResource struct {
 
 	// This may be set to tell us to look in another filesystem for this resource.
 	// We, by default, use the sourceFs filesystem in the spec below.
-	overriddenSourceFs afero.Fs
+	sourceFs afero.Fs
 
 	spec *Spec
 
@@ -411,7 +403,8 @@ func (l *genericResource) ReadSeekCloser() (hugio.ReadSeekCloser, error) {
 	if l.openReadSeekerCloser != nil {
 		return l.openReadSeekerCloser()
 	}
-	f, err := l.sourceFs().Open(l.sourceFilename)
+
+	f, err := l.getSourceFs().Open(l.sourceFilename)
 	if err != nil {
 		return nil, err
 	}
@@ -497,11 +490,8 @@ func (l *genericResource) initContent() error {
 	return err
 }
 
-func (l *genericResource) sourceFs() afero.Fs {
-	if l.overriddenSourceFs != nil {
-		return l.overriddenSourceFs
-	}
-	return l.spec.sourceFs()
+func (l *genericResource) getSourceFs() afero.Fs {
+	return l.sourceFs
 }
 
 func (l *genericResource) publishIfNeeded() {
@@ -711,6 +701,10 @@ func (r *Spec) newGenericResourceWithBase(
 	baseFilename string,
 	mediaType media.Type) *genericResource {
 
+	if osFileInfo != nil && osFileInfo.IsDir() {
+		panic(fmt.Sprintf("dirs nto supported resource types: %v", osFileInfo))
+	}
+
 	// This value is used both to construct URLs and file paths, but start
 	// with a Unix-styled path.
 	baseFilename = helpers.ToSlashTrimLeading(baseFilename)
@@ -724,7 +718,7 @@ func (r *Spec) newGenericResourceWithBase(
 	}
 
 	pathDescriptor := resourcePathDescriptor{
-		baseTargetPathDirs: helpers.UniqueStrings(targetPathBaseDirs),
+		baseTargetPathDirs: helpers.UniqueStringsReuse(targetPathBaseDirs),
 		targetPathBuilder:  targetPathBuilder,
 		relTargetDirFile:   dirFile{dir: fpath, file: fname},
 	}
@@ -738,7 +732,7 @@ func (r *Spec) newGenericResourceWithBase(
 		openReadSeekerCloser:   openReadSeekerCloser,
 		publishOnce:            po,
 		resourcePathDescriptor: pathDescriptor,
-		overriddenSourceFs:     sourceFs,
+		sourceFs:               sourceFs,
 		osFileInfo:             osFileInfo,
 		sourceFilename:         sourceFilename,
 		mediaType:              mediaType,

resources/resource/params.go

@@ -14,7 +14,7 @@
 package resource
 
 import (
-	"strings"
+	"github.com/gohugoio/hugo/common/maps"
 
 	"github.com/spf13/cast"
 )
@@ -25,65 +25,6 @@ func Param(r ResourceParamsProvider, fallback map[string]interface{}, key interf
 		return nil, err
 	}
 
-	keyStr = strings.ToLower(keyStr)
-	result, _ := traverseDirectParams(r, fallback, keyStr)
-	if result != nil {
-		return result, nil
-	}
+	return maps.GetNestedParam(keyStr, ".", r.Params(), fallback)
 
-	keySegments := strings.Split(keyStr, ".")
-	if len(keySegments) == 1 {
-		return nil, nil
-	}
-
-	return traverseNestedParams(r, fallback, keySegments)
-}
-
-func traverseDirectParams(r ResourceParamsProvider, fallback map[string]interface{}, key string) (interface{}, error) {
-	keyStr := strings.ToLower(key)
-	if val, ok := r.Params()[keyStr]; ok {
-		return val, nil
-	}
-
-	if fallback == nil {
-		return nil, nil
-	}
-
-	return fallback[keyStr], nil
-}
-
-func traverseNestedParams(r ResourceParamsProvider, fallback map[string]interface{}, keySegments []string) (interface{}, error) {
-	result := traverseParams(keySegments, r.Params())
-	if result != nil {
-		return result, nil
-	}
-
-	if fallback != nil {
-		result = traverseParams(keySegments, fallback)
-		if result != nil {
-			return result, nil
-		}
-	}
-
-	// Didn't find anything, but also no problems.
-	return nil, nil
-}
-
-func traverseParams(keys []string, m map[string]interface{}) interface{} {
-	// Shift first element off.
-	firstKey, rest := keys[0], keys[1:]
-	result := m[firstKey]
-
-	// No point in continuing here.
-	if result == nil {
-		return result
-	}
-
-	if len(rest) == 0 {
-		// That was the last key.
-		return result
-	}
-
-	// That was not the last key.
-	return traverseParams(rest, cast.ToStringMap(result))
 }

resources/resource_factories/bundler/bundler.go

@@ -124,9 +124,9 @@ func (c *Client) Concat(targetPath string, r resource.Resources) (resource.Resou
 			return &multiReadSeekCloser{mr: mr, sources: rcsources}, nil
 		}
 
-		composite, err := c.rs.NewForFs(
-			c.rs.FileCaches.AssetsCache().Fs,
+		composite, err := c.rs.New(
 			resources.ResourceSourceDescriptor{
+				Fs:                 c.rs.FileCaches.AssetsCache().Fs,
 				LazyPublish:        true,
 				OpenReadSeekCloser: concatr,
 				RelTargetFilename:  filepath.Clean(targetPath)})

resources/resource_factories/create/create.go

@@ -40,10 +40,10 @@ func New(rs *resources.Spec) *Client {
 func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) {
 	filename = filepath.Clean(filename)
 	return c.rs.ResourceCache.GetOrCreate(resources.ResourceKeyPartition(filename), filename, func() (resource.Resource, error) {
-		return c.rs.NewForFs(fs,
-			resources.ResourceSourceDescriptor{
-				LazyPublish:    true,
-				SourceFilename: filename})
+		return c.rs.New(resources.ResourceSourceDescriptor{
+			Fs:             fs,
+			LazyPublish:    true,
+			SourceFilename: filename})
 	})
 
 }
@@ -51,9 +51,9 @@ func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) {
 // FromString creates a new Resource from a string with the given relative target path.
 func (c *Client) FromString(targetPath, content string) (resource.Resource, error) {
 	return c.rs.ResourceCache.GetOrCreate(resources.CACHE_OTHER, targetPath, func() (resource.Resource, error) {
-		return c.rs.NewForFs(
-			c.rs.FileCaches.AssetsCache().Fs,
+		return c.rs.New(
 			resources.ResourceSourceDescriptor{
+				Fs:          c.rs.FileCaches.AssetsCache().Fs,
 				LazyPublish: true,
 				OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
 					return hugio.NewReadSeekerNoOpCloserFromString(content), nil

resources/resource_test.go

@@ -21,6 +21,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/spf13/afero"
+
 	"github.com/gohugoio/hugo/resources/resource"
 
 	"github.com/gohugoio/hugo/media"
@@ -61,7 +63,9 @@ func TestNewResourceFromFilename(t *testing.T) {
 	writeSource(t, spec.Fs, "content/a/b/logo.png", "image")
 	writeSource(t, spec.Fs, "content/a/b/data.json", "json")
 
-	r, err := spec.New(ResourceSourceDescriptor{SourceFilename: "a/b/logo.png"})
+	bfs := afero.NewBasePathFs(spec.Fs.Source, "content")
+
+	r, err := spec.New(ResourceSourceDescriptor{Fs: bfs, SourceFilename: "a/b/logo.png"})
 
 	assert.NoError(err)
 	assert.NotNil(r)
@@ -69,7 +73,7 @@ func TestNewResourceFromFilename(t *testing.T) {
 	assert.Equal("/a/b/logo.png", r.RelPermalink())
 	assert.Equal("https://example.com/a/b/logo.png", r.Permalink())
 
-	r, err = spec.New(ResourceSourceDescriptor{SourceFilename: "a/b/data.json"})
+	r, err = spec.New(ResourceSourceDescriptor{Fs: bfs, SourceFilename: "a/b/data.json"})
 
 	assert.NoError(err)
 	assert.NotNil(r)
@@ -85,8 +89,10 @@ func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) {
 	spec := newTestResourceSpecForBaseURL(assert, "https://example.com/docs")
 
 	writeSource(t, spec.Fs, "content/a/b/logo.png", "image")
+	bfs := afero.NewBasePathFs(spec.Fs.Source, "content")
 
-	r, err := spec.New(ResourceSourceDescriptor{SourceFilename: filepath.FromSlash("a/b/logo.png")})
+	fmt.Println()
+	r, err := spec.New(ResourceSourceDescriptor{Fs: bfs, SourceFilename: filepath.FromSlash("a/b/logo.png")})
 
 	assert.NoError(err)
 	assert.NotNil(r)

resources/resource_transformers/postcss/postcss.go

@@ -130,7 +130,7 @@ func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationC
 	if !filepath.IsAbs(configFile) {
 		// We resolve this against the virtual Work filesystem, to allow
 		// this config file to live in one of the themes if needed.
-		fi, err := t.rs.BaseFs.Work.Fs.Stat(configFile)
+		fi, err := t.rs.BaseFs.Work.Stat(configFile)
 		if err != nil {
 			if t.options.Config != "" {
 				// Only fail if the user specificed config file is not found.
@@ -138,7 +138,7 @@ func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationC
 			}
 			configFile = ""
 		} else {
-			configFile = fi.(hugofs.RealFilenameInfo).RealFilename()
+			configFile = fi.(hugofs.FileMetaInfo).Meta().Filename()
 		}
 	}
 

resources/resource_transformers/tocss/scss/client.go

@@ -19,6 +19,7 @@ import (
 	"github.com/gohugoio/hugo/hugolib/filesystems"
 	"github.com/gohugoio/hugo/resources"
 	"github.com/gohugoio/hugo/resources/resource"
+	"github.com/spf13/afero"
 
 	"github.com/mitchellh/mapstructure"
 )
@@ -26,7 +27,7 @@ import (
 type Client struct {
 	rs     *resources.Spec
 	sfs    *filesystems.SourceFilesystem
-	workFs *filesystems.SourceFilesystem
+	workFs afero.Fs
 }
 
 func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) (*Client, error) {

resources/resource_transformers/tocss/scss/tocss.go

@@ -55,7 +55,11 @@ func (t *toCSSTransformation) Transform(ctx *resources.ResourceTransformationCtx
 
 	// Append any workDir relative include paths
 	for _, ip := range options.from.IncludePaths {
-		options.to.IncludePaths = append(options.to.IncludePaths, t.c.workFs.RealDirs(filepath.Clean(ip))...)
+		info, err := t.c.workFs.Stat(filepath.Clean(ip))
+		if err == nil {
+			filename := info.(hugofs.FileMetaInfo).Meta().Filename()
+			options.to.IncludePaths = append(options.to.IncludePaths, filename)
+		}
 	}
 
 	// To allow for overrides of SCSS files anywhere in the project/theme hierarchy, we need
@@ -74,6 +78,7 @@ func (t *toCSSTransformation) Transform(ctx *resources.ResourceTransformationCtx
 			prevDir = baseDir
 		} else {
 			prevDir = t.c.sfs.MakePathRelative(filepath.Dir(prev))
+
 			if prevDir == "" {
 				// Not a member of this filesystem. Let LibSASS handle it.
 				return "", "", false
@@ -100,8 +105,8 @@ func (t *toCSSTransformation) Transform(ctx *resources.ResourceTransformationCtx
 			filenameToCheck := filepath.Join(basePath, fmt.Sprintf(namePattern, name))
 			fi, err := t.c.sfs.Fs.Stat(filenameToCheck)
 			if err == nil {
-				if fir, ok := fi.(hugofs.RealFilenameInfo); ok {
-					return fir.RealFilename(), "", true
+				if fim, ok := fi.(hugofs.FileMetaInfo); ok {
+					return fim.Meta().Filename(), "", true
 				}
 			}
 		}

resources/testhelpers_test.go

@@ -4,7 +4,6 @@ import (
 	"path/filepath"
 	"testing"
 
-	"fmt"
 	"image"
 	"io"
 	"io/ioutil"
@@ -12,6 +11,9 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/gohugoio/hugo/langs"
+	"github.com/gohugoio/hugo/modules"
+
 	"github.com/gohugoio/hugo/cache/filecache"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
@@ -28,9 +30,8 @@ func newTestResourceSpec(assert *require.Assertions) *Spec {
 	return newTestResourceSpecForBaseURL(assert, "https://example.com/")
 }
 
-func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) *Spec {
+func createTestCfg() *viper.Viper {
 	cfg := viper.New()
-	cfg.Set("baseURL", baseURL)
 	cfg.Set("resourceDir", "resources")
 	cfg.Set("contentDir", "content")
 	cfg.Set("dataDir", "data")
@@ -40,6 +41,21 @@ func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) *
 	cfg.Set("archetypeDir", "archetypes")
 	cfg.Set("publishDir", "public")
 
+	langs.LoadLanguageSettings(cfg, nil)
+	mod, err := modules.CreateProjectModule(cfg)
+	if err != nil {
+		panic(err)
+	}
+	cfg.Set("allModules", modules.Modules{mod})
+
+	return cfg
+
+}
+
+func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) *Spec {
+	cfg := createTestCfg()
+	cfg.Set("baseURL", baseURL)
+
 	imagingCfg := map[string]interface{}{
 		"resampleFilter": "linear",
 		"quality":        68,
@@ -71,7 +87,7 @@ func newTargetPaths(link string) func() page.TargetPaths {
 }
 
 func newTestResourceOsFs(assert *require.Assertions) *Spec {
-	cfg := viper.New()
+	cfg := createTestCfg()
 	cfg.Set("baseURL", "https://example.com")
 
 	workDir, _ := ioutil.TempDir("", "hugores")
@@ -83,17 +99,10 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec {
 	}
 
 	cfg.Set("workingDir", workDir)
-	cfg.Set("resourceDir", "resources")
-	cfg.Set("contentDir", "content")
-	cfg.Set("dataDir", "data")
-	cfg.Set("i18nDir", "i18n")
-	cfg.Set("layoutDir", "layouts")
-	cfg.Set("assetDir", "assets")
-	cfg.Set("archetypeDir", "archetypes")
-	cfg.Set("publishDir", "public")
 
 	fs := hugofs.NewFrom(hugofs.Os, cfg)
 	fs.Destination = &afero.MemMapFs{}
+	fs.Source = afero.NewBasePathFs(hugofs.Os, workDir)
 
 	s, err := helpers.NewPathSpec(fs, cfg)
 	assert.NoError(err)
@@ -126,7 +135,7 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) r
 	src, err := os.Open(filepath.FromSlash("testdata/" + name))
 	assert.NoError(err)
 
-	out, err := helpers.OpenFileForWriting(spec.BaseFs.Content.Fs, name)
+	out, err := helpers.OpenFileForWriting(spec.Fs.Source, name)
 	assert.NoError(err)
 	_, err = io.Copy(out, src)
 	out.Close()
@@ -135,7 +144,7 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) r
 
 	factory := newTargetPaths("/a")
 
-	r, err := spec.New(ResourceSourceDescriptor{TargetPaths: factory, LazyPublish: true, SourceFilename: name})
+	r, err := spec.New(ResourceSourceDescriptor{Fs: spec.Fs.Source, TargetPaths: factory, LazyPublish: true, SourceFilename: name})
 	assert.NoError(err)
 
 	return r.(resource.ContentResource)
@@ -144,9 +153,6 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) r
 func assertImageFile(assert *require.Assertions, fs afero.Fs, filename string, width, height int) {
 	filename = filepath.Clean(filename)
 	f, err := fs.Open(filename)
-	if err != nil {
-		printFs(fs, "", os.Stdout)
-	}
 	assert.NoError(err)
 	defer f.Close()
 
@@ -170,22 +176,3 @@ func writeToFs(t testing.TB, fs afero.Fs, filename, content string) {
 		t.Fatalf("Failed to write file: %s", err)
 	}
 }
-
-func printFs(fs afero.Fs, path string, w io.Writer) {
-	if fs == nil {
-		return
-	}
-	afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {
-		if info != nil && !info.IsDir() {
-			s := path
-			if lang, ok := info.(hugofs.LanguageAnnouncer); ok {
-				s = s + "\t" + lang.Lang()
-			}
-			if fp, ok := info.(hugofs.FilePather); ok {
-				s += "\tFilename: " + fp.Filename() + "\tBase: " + fp.BaseDir()
-			}
-			fmt.Fprintln(w, "    ", s)
-		}
-		return nil
-	})
-}

resources/transform.go

@@ -406,6 +406,7 @@ func (r *transformedResource) transform(setContent, publish bool) (err error) {
 		}
 
 		if err := tr.transformation.Transform(tctx); err != nil {
+
 			if err == herrors.ErrFeatureNotAvailable {
 				// This transformation is not available in this
 				// Hugo installation (scss not compiled in, PostCSS not available etc.)
@@ -515,6 +516,9 @@ func (r *transformedResource) initTransform(setContent, publish bool) error {
 
 		// Copy the file from cache to /public
 		_, src, err := r.cache.fileCache.Get(r.sourceFilename)
+		if src == nil {
+			panic(fmt.Sprintf("[BUG] resource cache file not found: %q", r.sourceFilename))
+		}
 
 		if err == nil {
 			defer src.Close()