Rework the Destination filesystem to make --renderStaticToDisk work

See #9626

hugofs/createcounting_fs.go

@@ -33,10 +33,18 @@ type DuplicatesReporter interface {
 	ReportDuplicates() string
 }
 
+var (
+	_ FilesystemUnwrapper = (*createCountingFs)(nil)
+)
+
 func NewCreateCountingFs(fs afero.Fs) afero.Fs {
 	return &createCountingFs{Fs: fs, fileCount: make(map[string]int)}
 }
 
+func (fs *createCountingFs) UnwrapFilesystem() afero.Fs {
+	return fs.Fs
+}
+
 // ReportDuplicates reports filenames written more than once.
 func (c *createCountingFs) ReportDuplicates() string {
 	c.mu.Lock()

hugofs/decorators.go

@@ -23,6 +23,10 @@ import (
 	"github.com/spf13/afero"
 )
 
+var (
+	_ FilesystemUnwrapper = (*baseFileDecoratorFs)(nil)
+)
+
 func decorateDirs(fs afero.Fs, meta *FileMeta) afero.Fs {
 	ffs := &baseFileDecoratorFs{Fs: fs}
 
@@ -151,6 +155,10 @@ type baseFileDecoratorFs struct {
 	decorate func(fi os.FileInfo, filename string) (os.FileInfo, error)
 }
 
+func (fs *baseFileDecoratorFs) UnwrapFilesystem() afero.Fs {
+	return fs.Fs
+}
+
 func (fs *baseFileDecoratorFs) Stat(name string) (os.FileInfo, error) {
 	fi, err := fs.Fs.Stat(name)
 	if err != nil {

hugofs/filename_filter_fs.go

@@ -23,6 +23,10 @@ import (
 	"github.com/spf13/afero"
 )
 
+var (
+	_ FilesystemUnwrapper = (*filenameFilterFs)(nil)
+)
+
 func newFilenameFilterFs(fs afero.Fs, base string, filter *glob.FilenameFilter) afero.Fs {
 	return &filenameFilterFs{
 		fs:     fs,
@@ -39,6 +43,10 @@ type filenameFilterFs struct {
 	filter *glob.FilenameFilter
 }
 
+func (fs *filenameFilterFs) UnwrapFilesystem() afero.Fs {
+	return fs.fs
+}
+
 func (fs *filenameFilterFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
 	fi, b, err := fs.fs.(afero.Lstater).LstatIfPossible(name)
 	if err != nil {

hugofs/filter_fs.go

@@ -121,6 +121,10 @@ func NewFilterFs(fs afero.Fs) (afero.Fs, error) {
 	return ffs, nil
 }
 
+var (
+	_ FilesystemUnwrapper = (*FilterFs)(nil)
+)
+
 // FilterFs is an ordered composite filesystem.
 type FilterFs struct {
 	fs afero.Fs
@@ -141,6 +145,10 @@ func (fs *FilterFs) Chown(n string, uid, gid int) error {
 	return syscall.EPERM
 }
 
+func (fs *FilterFs) UnwrapFilesystem() afero.Fs {
+	return fs.fs
+}
+
 func (fs *FilterFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
 	fi, b, err := lstatIfPossible(fs.fs, name)
 	if err != nil {

hugofs/fs.go

@@ -19,6 +19,8 @@ import (
 	"os"
 	"strings"
 
+	"github.com/bep/overlayfs"
+	"github.com/gohugoio/hugo/common/paths"
 	"github.com/gohugoio/hugo/config"
 	"github.com/spf13/afero"
 )
@@ -26,32 +28,43 @@ import (
 // Os points to the (real) Os filesystem.
 var Os = &afero.OsFs{}
 
-// Fs abstracts the file system to separate source and destination file systems
-// and allows both to be mocked for testing.
+// Fs holds the core filesystems used by Hugo.
 type Fs struct {
 	// Source is Hugo's source file system.
+	// Note that this will always be a "plain" Afero filesystem:
+	// * afero.OsFs when running in production
+	// * afero.MemMapFs for many of the tests.
 	Source afero.Fs
 
-	// Destination is Hugo's destination file system.
-	Destination afero.Fs
+	// PublishDir is where Hugo publishes its rendered content.
+	// It's mounted inside publishDir (default /public).
+	PublishDir afero.Fs
 
-	// Destination used for `renderStaticToDisk`
-	DestinationStatic afero.Fs
+	// PublishDirStatic is the file system used for static files  when --renderStaticToDisk is set.
+	// When this is set, PublishDir is set to write to memory.
+	PublishDirStatic afero.Fs
+
+	// PublishDirServer is the file system used for serving the public directory with Hugo's development server.
+	// This will typically be the same as PublishDir, but not if --renderStaticToDisk is set.
+	PublishDirServer afero.Fs
 
 	// Os is an OS file system.
 	// NOTE: Field is currently unused.
 	Os afero.Fs
 
-	// WorkingDir is a read-only file system
+	// WorkingDirReadOnly is a read-only file system
 	// restricted to the project working dir.
-	// TODO(bep) get rid of this (se BaseFs)
-	WorkingDir *afero.BasePathFs
+	WorkingDirReadOnly afero.Fs
+
+	// WorkingDirWritable is a writable file system
+	// restricted to the project working dir.
+	WorkingDirWritable afero.Fs
 }
 
 // NewDefault creates a new Fs with the OS file system
 // as source and destination file systems.
 func NewDefault(cfg config.Provider) *Fs {
-	fs := &afero.OsFs{}
+	fs := Os
 	return newFs(fs, cfg)
 }
 
@@ -71,23 +84,49 @@ func NewFrom(fs afero.Fs, cfg config.Provider) *Fs {
 }
 
 func newFs(base afero.Fs, cfg config.Provider) *Fs {
+	workingDir := cfg.GetString("workingDir")
+	publishDir := cfg.GetString("publishDir")
+	if publishDir == "" {
+		panic("publishDir is empty")
+	}
+
+	// Sanity check
+	if IsOsFs(base) && len(workingDir) < 2 {
+		panic("workingDir is too short")
+	}
+
+	absPublishDir := paths.AbsPathify(workingDir, publishDir)
+
+	// Make sure we always have the /public folder ready to use.
+	if err := base.MkdirAll(absPublishDir, 0777); err != nil && !os.IsExist(err) {
+		panic(err)
+	}
+
+	pubFs := afero.NewBasePathFs(base, absPublishDir)
+
 	return &Fs{
-		Source:            base,
-		Destination:       base,
-		DestinationStatic: base,
-		Os:                &afero.OsFs{},
-		WorkingDir:        getWorkingDirFs(base, cfg),
+		Source:             base,
+		PublishDir:         pubFs,
+		PublishDirServer:   pubFs,
+		PublishDirStatic:   pubFs,
+		Os:                 &afero.OsFs{},
+		WorkingDirReadOnly: getWorkingDirFsReadOnly(base, workingDir),
+		WorkingDirWritable: getWorkingDirFsWritable(base, workingDir),
 	}
 }
 
-func getWorkingDirFs(base afero.Fs, cfg config.Provider) *afero.BasePathFs {
-	workingDir := cfg.GetString("workingDir")
-
-	if workingDir != "" {
-		return afero.NewBasePathFs(afero.NewReadOnlyFs(base), workingDir).(*afero.BasePathFs)
+func getWorkingDirFsReadOnly(base afero.Fs, workingDir string) afero.Fs {
+	if workingDir == "" {
+		return afero.NewReadOnlyFs(base)
 	}
+	return afero.NewBasePathFs(afero.NewReadOnlyFs(base), workingDir)
+}
 
-	return nil
+func getWorkingDirFsWritable(base afero.Fs, workingDir string) afero.Fs {
+	if workingDir == "" {
+		return base
+	}
+	return afero.NewBasePathFs(base, workingDir)
 }
 
 func isWrite(flag int) bool {
@@ -117,3 +156,64 @@ func MakeReadableAndRemoveAllModulePkgDir(fs afero.Fs, dir string) (int, error)
 	})
 	return counter, fs.RemoveAll(dir)
 }
+
+// HasOsFs returns whether fs is an OsFs or if it fs wraps an OsFs.
+// TODO(bep) make this nore robust.
+func IsOsFs(fs afero.Fs) bool {
+	var isOsFs bool
+	WalkFilesystems(fs, func(fs afero.Fs) bool {
+		switch base := fs.(type) {
+		case *afero.MemMapFs:
+			isOsFs = false
+		case *afero.OsFs:
+			isOsFs = true
+		case *afero.BasePathFs:
+			_, supportsLstat, _ := base.LstatIfPossible("asdfasdfasdf")
+			isOsFs = supportsLstat
+		}
+		return isOsFs
+	})
+	return isOsFs
+}
+
+// FilesystemsUnwrapper returns the underlying filesystems.
+type FilesystemsUnwrapper interface {
+	UnwrapFilesystems() []afero.Fs
+}
+
+// FilesystemsProvider returns the underlying filesystem.
+type FilesystemUnwrapper interface {
+	UnwrapFilesystem() afero.Fs
+}
+
+// WalkFn is the walk func for WalkFilesystems.
+type WalkFn func(fs afero.Fs) bool
+
+// WalkFilesystems walks fs recursively and calls fn.
+// If fn returns true, walking is stopped.
+func WalkFilesystems(fs afero.Fs, fn WalkFn) bool {
+	if fn(fs) {
+		return true
+	}
+
+	if afs, ok := fs.(FilesystemUnwrapper); ok {
+		if WalkFilesystems(afs.UnwrapFilesystem(), fn) {
+			return true
+		}
+
+	} else if bfs, ok := fs.(FilesystemsUnwrapper); ok {
+		for _, sf := range bfs.UnwrapFilesystems() {
+			if WalkFilesystems(sf, fn) {
+				return true
+			}
+		}
+	} else if cfs, ok := fs.(overlayfs.FilesystemIterator); ok {
+		for i := 0; i < cfs.NumFilesystems(); i++ {
+			if WalkFilesystems(cfs.Filesystem(i), fn) {
+				return true
+			}
+		}
+	}
+
+	return false
+}

hugofs/fs_test.go

@@ -23,38 +23,46 @@ import (
 	"github.com/spf13/afero"
 )
 
+func TestIsOsFs(t *testing.T) {
+	c := qt.New(t)
+
+	c.Assert(IsOsFs(Os), qt.Equals, true)
+	c.Assert(IsOsFs(&afero.MemMapFs{}), qt.Equals, false)
+	c.Assert(IsOsFs(afero.NewBasePathFs(&afero.MemMapFs{}, "/public")), qt.Equals, false)
+	c.Assert(IsOsFs(afero.NewBasePathFs(Os, t.TempDir())), qt.Equals, true)
+
+}
+
 func TestNewDefault(t *testing.T) {
 	c := qt.New(t)
-	v := config.New()
+	v := config.NewWithTestDefaults()
+	v.Set("workingDir", t.TempDir())
 	f := NewDefault(v)
 
-	c.Assert(f.Source, qt.Not(qt.IsNil))
+	c.Assert(f.Source, qt.IsNotNil)
 	c.Assert(f.Source, hqt.IsSameType, new(afero.OsFs))
-	c.Assert(f.Os, qt.Not(qt.IsNil))
-	c.Assert(f.WorkingDir, qt.IsNil)
+	c.Assert(f.Os, qt.IsNotNil)
+	c.Assert(f.WorkingDirReadOnly, qt.IsNotNil)
+	c.Assert(f.WorkingDirReadOnly, hqt.IsSameType, new(afero.BasePathFs))
+	c.Assert(IsOsFs(f.Source), qt.IsTrue)
+	c.Assert(IsOsFs(f.WorkingDirReadOnly), qt.IsTrue)
+	c.Assert(IsOsFs(f.PublishDir), qt.IsTrue)
+	c.Assert(IsOsFs(f.Os), qt.IsTrue)
 }
 
 func TestNewMem(t *testing.T) {
 	c := qt.New(t)
-	v := config.New()
+	v := config.NewWithTestDefaults()
 	f := NewMem(v)
 
 	c.Assert(f.Source, qt.Not(qt.IsNil))
 	c.Assert(f.Source, hqt.IsSameType, new(afero.MemMapFs))
-	c.Assert(f.Destination, qt.Not(qt.IsNil))
-	c.Assert(f.Destination, hqt.IsSameType, new(afero.MemMapFs))
+	c.Assert(f.PublishDir, qt.Not(qt.IsNil))
+	c.Assert(f.PublishDir, hqt.IsSameType, new(afero.BasePathFs))
 	c.Assert(f.Os, hqt.IsSameType, new(afero.OsFs))
-	c.Assert(f.WorkingDir, qt.IsNil)
-}
-
-func TestWorkingDir(t *testing.T) {
-	c := qt.New(t)
-	v := config.New()
-
-	v.Set("workingDir", "/a/b/")
-
-	f := NewMem(v)
-
-	c.Assert(f.WorkingDir, qt.Not(qt.IsNil))
-	c.Assert(f.WorkingDir, hqt.IsSameType, new(afero.BasePathFs))
+	c.Assert(f.WorkingDirReadOnly, qt.IsNotNil)
+	c.Assert(IsOsFs(f.Source), qt.IsFalse)
+	c.Assert(IsOsFs(f.WorkingDirReadOnly), qt.IsFalse)
+	c.Assert(IsOsFs(f.PublishDir), qt.IsFalse)
+	c.Assert(IsOsFs(f.Os), qt.IsTrue)
 }

hugofs/hashing_fs.go

@@ -22,7 +22,10 @@ import (
 	"github.com/spf13/afero"
 )
 
-var _ afero.Fs = (*md5HashingFs)(nil)
+var (
+	_ afero.Fs            = (*md5HashingFs)(nil)
+	_ FilesystemUnwrapper = (*md5HashingFs)(nil)
+)
 
 // FileHashReceiver will receive the filename an the content's MD5 sum on file close.
 type FileHashReceiver interface {
@@ -45,6 +48,10 @@ func NewHashingFs(delegate afero.Fs, hashReceiver FileHashReceiver) afero.Fs {
 	return &md5HashingFs{Fs: delegate, hashReceiver: hashReceiver}
 }
 
+func (fs *md5HashingFs) UnwrapFilesystem() afero.Fs {
+	return fs.Fs
+}
+
 func (fs *md5HashingFs) Create(name string) (afero.File, error) {
 	f, err := fs.Fs.Create(name)
 	if err == nil {

hugofs/language_composite_fs.go

@@ -20,11 +20,14 @@ import (
 )
 
 var (
-	_ afero.Fs      = (*languageCompositeFs)(nil)
-	_ afero.Lstater = (*languageCompositeFs)(nil)
+	_ afero.Fs             = (*languageCompositeFs)(nil)
+	_ afero.Lstater        = (*languageCompositeFs)(nil)
+	_ FilesystemsUnwrapper = (*languageCompositeFs)(nil)
 )
 
 type languageCompositeFs struct {
+	base    afero.Fs
+	overlay afero.Fs
 	*afero.CopyOnWriteFs
 }
 
@@ -33,7 +36,11 @@ type languageCompositeFs struct {
 // to the target filesystem. This information is available in Readdir, Stat etc. via the
 // special LanguageFileInfo FileInfo implementation.
 func NewLanguageCompositeFs(base, overlay afero.Fs) afero.Fs {
-	return &languageCompositeFs{afero.NewCopyOnWriteFs(base, overlay).(*afero.CopyOnWriteFs)}
+	return &languageCompositeFs{base, overlay, afero.NewCopyOnWriteFs(base, overlay).(*afero.CopyOnWriteFs)}
+}
+
+func (fs *languageCompositeFs) UnwrapFilesystems() []afero.Fs {
+	return []afero.Fs{fs.base, fs.overlay}
 }
 
 // Open takes the full path to the file in the target filesystem. If it is a directory, it gets merged

hugofs/nosymlink_fs.go

@@ -30,6 +30,10 @@ func NewNoSymlinkFs(fs afero.Fs, logger loggers.Logger, allowFiles bool) afero.F
 	return &noSymlinkFs{Fs: fs, logger: logger, allowFiles: allowFiles}
 }
 
+var (
+	_ FilesystemUnwrapper = (*noSymlinkFs)(nil)
+)
+
 // noSymlinkFs is a filesystem that prevents symlinking.
 type noSymlinkFs struct {
 	allowFiles bool // block dirs only
@@ -67,6 +71,10 @@ func (f *noSymlinkFile) Readdirnames(count int) ([]string, error) {
 	return fileInfosToNames(dirs), nil
 }
 
+func (fs *noSymlinkFs) UnwrapFilesystem() afero.Fs {
+	return fs.Fs
+}
+
 func (fs *noSymlinkFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
 	return fs.stat(name)
 }

hugofs/rootmapping_fs.go

@@ -151,6 +151,10 @@ func (r RootMapping) trimFrom(name string) string {
 	return strings.TrimPrefix(name, r.From)
 }
 
+var (
+	_ FilesystemUnwrapper = (*RootMappingFs)(nil)
+)
+
 // A RootMappingFs maps several roots into one. Note that the root of this filesystem
 // is directories only, and they will be returned in Readdir and Readdirnames
 // in the order given.
@@ -200,6 +204,10 @@ func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) {
 	return fss, nil
 }
 
+func (fs *RootMappingFs) UnwrapFilesystem() afero.Fs {
+	return fs.Fs
+}
+
 // Filter creates a copy of this filesystem with only mappings matching a filter.
 func (fs RootMappingFs) Filter(f func(m RootMapping) bool) *RootMappingFs {
 	rootMapToReal := radix.New()

hugofs/rootmapping_fs_test.go

@@ -20,9 +20,8 @@ import (
 	"sort"
 	"testing"
 
-	"github.com/gohugoio/hugo/hugofs/glob"
-
 	"github.com/gohugoio/hugo/config"
+	"github.com/gohugoio/hugo/hugofs/glob"
 
 	qt "github.com/frankban/quicktest"
 	"github.com/gohugoio/hugo/htesting"
@@ -31,7 +30,7 @@ import (
 
 func TestLanguageRootMapping(t *testing.T) {
 	c := qt.New(t)
-	v := config.New()
+	v := config.NewWithTestDefaults()
 	v.Set("contentDir", "content")
 
 	fs := NewBaseFileDecorator(afero.NewMemMapFs())

hugofs/slice_fs.go

@@ -24,9 +24,10 @@ import (
 )
 
 var (
-	_ afero.Fs      = (*SliceFs)(nil)
-	_ afero.Lstater = (*SliceFs)(nil)
-	_ afero.File    = (*sliceDir)(nil)
+	_ afero.Fs             = (*SliceFs)(nil)
+	_ afero.Lstater        = (*SliceFs)(nil)
+	_ FilesystemsUnwrapper = (*SliceFs)(nil)
+	_ afero.File           = (*sliceDir)(nil)
 )
 
 func NewSliceFs(dirs ...FileMetaInfo) (afero.Fs, error) {
@@ -52,6 +53,14 @@ type SliceFs struct {
 	dirs []FileMetaInfo
 }
 
+func (fs *SliceFs) UnwrapFilesystems() []afero.Fs {
+	var fss []afero.Fs
+	for _, dir := range fs.dirs {
+		fss = append(fss, dir.Meta().Fs)
+	}
+	return fss
+}
+
 func (fs *SliceFs) Chmod(n string, m os.FileMode) error {
 	return syscall.EPERM
 }

hugofs/stacktracer_fs.go

@@ -24,8 +24,11 @@ import (
 	"github.com/spf13/afero"
 )
 
-// Make sure we don't accidentally use this in the real Hugo.
-var _ types.DevMarker = (*stacktracerFs)(nil)
+var (
+	//  Make sure we don't accidentally use this in the real Hugo.
+	_ types.DevMarker     = (*stacktracerFs)(nil)
+	_ FilesystemUnwrapper = (*stacktracerFs)(nil)
+)
 
 // NewStacktracerFs wraps the given fs printing stack traces for file creates
 // matching the given regexp pattern.
@@ -45,6 +48,10 @@ type stacktracerFs struct {
 func (fs *stacktracerFs) DevOnly() {
 }
 
+func (fs *stacktracerFs) UnwrapFilesystem() afero.Fs {
+	return fs.Fs
+}
+
 func (fs *stacktracerFs) onCreate(filename string) {
 	if fs.re.MatchString(filename) {
 		trace := make([]byte, 1500)