Add Hugo Piper with SCSS support and much more

Before this commit, you would have to use page bundles to do image processing etc. in Hugo. This commit adds * A new `/assets` top-level project or theme dir (configurable via `assetDir`) * A new template func, `resources.Get` which can be used to "get a resource" that can be further processed. This means that you can now do this in your templates (or shortcodes): ```bash {{ $sunset := (resources.Get "images/sunset.jpg").Fill "300x200" }} ``` This also adds a new `extended` build tag that enables powerful SCSS/SASS support with source maps. To compile this from source, you will also need a C compiler installed: ``` HUGO_BUILD_TAGS=extended mage install ``` Note that you can use output of the SCSS processing later in a non-SCSSS-enabled Hugo. The `SCSS` processor is a _Resource transformation step_ and it can be chained with the many others in a pipeline: ```bash {{ $css := resources.Get "styles.scss" | resources.ToCSS | resources.PostCSS | resources.Minify | resources.Fingerprint }} <link rel="stylesheet" href="{{ $styles.RelPermalink }}" integrity="{{ $styles.Data.Digest }}" media="screen"> ``` The transformation funcs above have aliases, so it can be shortened to: ```bash {{ $css := resources.Get "styles.scss" | toCSS | postCSS | minify | fingerprint }} <link rel="stylesheet" href="{{ $styles.RelPermalink }}" integrity="{{ $styles.Data.Digest }}" media="screen"> ``` A quick tip would be to avoid the fingerprinting part, and possibly also the not-superfast `postCSS` when you're doing development, as it allows Hugo to be smarter about the rebuilding. Documentation will follow, but have a look at the demo repo in https://github.com/bep/hugo-sass-test New functions to create `Resource` objects: * `resources.Get` (see above) * `resources.FromString`: Create a Resource from a string. New `Resource` transformation funcs: * `resources.ToCSS`: Compile `SCSS` or `SASS` into `CSS`. * `resources.PostCSS`: Process your CSS with PostCSS. Config file support (project or theme or passed as an option). * `resources.Minify`: Currently supports `css`, `js`, `json`, `html`, `svg`, `xml`. * `resources.Fingerprint`: Creates a fingerprinted version of the given Resource with Subresource Integrity.. * `resources.Concat`: Concatenates a list of Resource objects. Think of this as a poor man's bundler. * `resources.ExecuteAsTemplate`: Parses and executes the given Resource and data context (e.g. .Site) as a Go template. Fixes #4381 Fixes #4903 Fixes #4858
2025-08-27 22:09:53 +02:00 · 2018-02-20 10:02:14 +01:00
parent a5d0a57e6b
commit dea71670c0
90 changed files with 4685 additions and 1125 deletions
--- a/hugolib/filesystems/basefs.go
+++ b/hugolib/filesystems/basefs.go
@@ -28,7 +28,6 @@ import (

 	"fmt"

-	"github.com/gohugoio/hugo/common/types"
 	"github.com/gohugoio/hugo/hugolib/paths"
 	"github.com/gohugoio/hugo/langs"
 	"github.com/spf13/afero"
@@ -45,20 +44,10 @@ var filePathSeparator = string(filepath.Separator)
 // to underline that even if they can be composites, they all have a base path set to a specific
 // resource folder, e.g "/my-project/content". So, no absolute filenames needed.
 type BaseFs struct {
-	// TODO(bep) make this go away
-	AbsContentDirs []types.KeyValueStr
-
-	// The filesystem used to capture content. This can be a composite and
-	// language aware file system.
-	ContentFs afero.Fs

 	// SourceFilesystems contains the different source file systems.
 	*SourceFilesystems

-	// The filesystem used to store resources (processed images etc.).
-	// This usually maps to /my-project/resources.
-	ResourcesFs afero.Fs
-
 	// The filesystem used to publish the rendered site.
 	// This usually maps to /my-project/public.
 	PublishFs afero.Fs
@@ -71,35 +60,31 @@ type BaseFs struct {

 // RelContentDir tries to create a path relative to the content root from
 // the given filename. The return value is the path and language code.
-func (b *BaseFs) RelContentDir(filename string) (string, string) {
-	for _, dir := range b.AbsContentDirs {
-		if strings.HasPrefix(filename, dir.Value) {
-			rel := strings.TrimPrefix(filename, dir.Value)
-			return strings.TrimPrefix(rel, filePathSeparator), dir.Key
+func (b *BaseFs) RelContentDir(filename string) string {
+	for _, dirname := range b.SourceFilesystems.Content.Dirnames {
+		if strings.HasPrefix(filename, dirname) {
+			rel := strings.TrimPrefix(filename, dirname)
+			return strings.TrimPrefix(rel, filePathSeparator)
 		}
 	}
 	// Either not a content dir or already relative.
-	return filename, ""
-}
-
-// IsContent returns whether the given filename is in the content filesystem.
-func (b *BaseFs) IsContent(filename string) bool {
-	for _, dir := range b.AbsContentDirs {
-		if strings.HasPrefix(filename, dir.Value) {
-			return true
-		}
-	}
-	return false
+	return filename
 }

 // SourceFilesystems contains the different source file systems. These can be
 // composite file systems (theme and project etc.), and they have all root
 // set to the source type the provides: data, i18n, static, layouts.
 type SourceFilesystems struct {
+	Content    *SourceFilesystem
 	Data       *SourceFilesystem
 	I18n       *SourceFilesystem
 	Layouts    *SourceFilesystem
 	Archetypes *SourceFilesystem
+	Assets     *SourceFilesystem
+	Resources  *SourceFilesystem
+
+	// This is a unified read-only view of the project's and themes' workdir.
+	Work *SourceFilesystem

 	// When in multihost we have one static filesystem per language. The sync
 	// static files is currently done outside of the Hugo build (where there is
@@ -112,8 +97,14 @@ type SourceFilesystems struct {
 // i18n, layouts, static) and additional metadata to be able to use that filesystem
 // in server mode.
 type SourceFilesystem struct {
+	// This is a virtual composite filesystem. It expects path relative to a context.
 	Fs afero.Fs

+	// This is the base source filesystem. In real Hugo, this will be the OS filesystem.
+	// Use this if you need to resolve items in Dirnames below.
+	SourceFs afero.Fs
+
+	// Dirnames is absolute filenames to the directories in this filesystem.
 	Dirnames []string

 	// When syncing a source folder to the target (e.g. /public), this may
@@ -122,6 +113,50 @@ type SourceFilesystem struct {
 	PublishFolder string
 }

+// ContentStaticAssetFs will create a new composite filesystem from the content,
+// static, and asset filesystems. The site language is needed to pick the correct static filesystem.
+// The order is content, static and then assets.
+// TODO(bep) check usage
+func (s SourceFilesystems) ContentStaticAssetFs(lang string) afero.Fs {
+	staticFs := s.StaticFs(lang)
+
+	base := afero.NewCopyOnWriteFs(s.Assets.Fs, staticFs)
+	return afero.NewCopyOnWriteFs(base, s.Content.Fs)
+
+}
+
+// StaticFs returns the static filesystem for the given language.
+// This can be a composite filesystem.
+func (s SourceFilesystems) StaticFs(lang string) afero.Fs {
+	var staticFs afero.Fs = hugofs.NoOpFs
+
+	if fs, ok := s.Static[lang]; ok {
+		staticFs = fs.Fs
+	} else if fs, ok := s.Static[""]; ok {
+		staticFs = fs.Fs
+	}
+
+	return staticFs
+}
+
+// StatResource looks for a resource in these filesystems in order: static, assets and finally content.
+// If found in any of them, it returns FileInfo and the relevant filesystem.
+// Any non os.IsNotExist error will be returned.
+// An os.IsNotExist error wil be returned only if all filesystems return such an error.
+// Note that if we only wanted to find the file, we could create a composite Afero fs,
+// but we also need to know which filesystem root it lives in.
+func (s SourceFilesystems) StatResource(lang, filename string) (fi os.FileInfo, fs afero.Fs, err error) {
+	for _, fsToCheck := range []afero.Fs{s.StaticFs(lang), s.Assets.Fs, s.Content.Fs} {
+		fs = fsToCheck
+		fi, err = fs.Stat(filename)
+		if err == nil || !os.IsNotExist(err) {
+			return
+		}
+	}
+	// Not found.
+	return
+}
+
 // IsStatic returns true if the given filename is a member of one of the static
 // filesystems.
 func (s SourceFilesystems) IsStatic(filename string) bool {
@@ -133,6 +168,11 @@ func (s SourceFilesystems) IsStatic(filename string) bool {
 	return false
 }

+// IsContent returns true if the given filename is a member of the content filesystem.
+func (s SourceFilesystems) IsContent(filename string) bool {
+	return s.Content.Contains(filename)
+}
+
 // IsLayout returns true if the given filename is a member of the layouts filesystem.
 func (s SourceFilesystems) IsLayout(filename string) bool {
 	return s.Layouts.Contains(filename)
@@ -143,6 +183,11 @@ func (s SourceFilesystems) IsData(filename string) bool {
 	return s.Data.Contains(filename)
 }

+// IsAsset returns true if the given filename is a member of the data filesystem.
+func (s SourceFilesystems) IsAsset(filename string) bool {
+	return s.Assets.Contains(filename)
+}
+
 // IsI18n returns true if the given filename is a member of the i18n filesystem.
 func (s SourceFilesystems) IsI18n(filename string) bool {
 	return s.I18n.Contains(filename)
@@ -171,6 +216,18 @@ func (d *SourceFilesystem) MakePathRelative(filename string) string {
 	return ""
 }

+func (d *SourceFilesystem) RealFilename(rel string) string {
+	fi, err := d.Fs.Stat(rel)
+	if err != nil {
+		return rel
+	}
+	if realfi, ok := fi.(hugofs.RealFilenameInfo); ok {
+		return realfi.RealFilename()
+	}
+
+	return rel
+}
+
 // Contains returns whether the given filename is a member of the current filesystem.
 func (d *SourceFilesystem) Contains(filename string) bool {
 	for _, dir := range d.Dirnames {
@@ -181,6 +238,20 @@ func (d *SourceFilesystem) Contains(filename string) bool {
 	return false
 }

+// RealDirs gets a list of absolute paths to directorys starting from the given
+// path.
+func (d *SourceFilesystem) RealDirs(from string) []string {
+	var dirnames []string
+	for _, dir := range d.Dirnames {
+		dirname := filepath.Join(dir, from)
+
+		if _, err := hugofs.Os.Stat(dirname); err == nil {
+			dirnames = append(dirnames, dirname)
+		}
+	}
+	return dirnames
+}
+
 // WithBaseFs allows reuse of some potentially expensive to create parts that remain
 // the same across sites/languages.
 func WithBaseFs(b *BaseFs) func(*BaseFs) error {
@@ -191,11 +262,15 @@ func WithBaseFs(b *BaseFs) func(*BaseFs) error {
 	}
 }

+func newRealBase(base afero.Fs) afero.Fs {
+	return hugofs.NewBasePathRealFilenameFs(base.(*afero.BasePathFs))
+
+}
+
 // NewBase builds the filesystems used by Hugo given the paths and options provided.NewBase
 func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) {
 	fs := p.Fs

-	resourcesFs := afero.NewBasePathFs(fs.Source, p.AbsResourcesDir)
 	publishFs := afero.NewBasePathFs(fs.Destination, p.AbsPublishDir)

 	contentFs, absContentDirs, err := createContentFs(fs.Source, p.WorkingDir, p.DefaultContentLanguage, p.Languages)
@@ -209,17 +284,14 @@ func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) {
 			if i == j {
 				continue
 			}
-			if strings.HasPrefix(d1.Value, d2.Value) || strings.HasPrefix(d2.Value, d1.Value) {
+			if strings.HasPrefix(d1, d2) || strings.HasPrefix(d2, d1) {
 				return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2)
 			}
 		}
 	}

 	b := &BaseFs{
-		AbsContentDirs: absContentDirs,
-		ContentFs:      contentFs,
-		ResourcesFs:    resourcesFs,
-		PublishFs:      publishFs,
+		PublishFs: publishFs,
 	}

 	for _, opt := range options {
@@ -234,6 +306,12 @@ func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) {
 		return nil, err
 	}

+	sourceFilesystems.Content = &SourceFilesystem{
+		SourceFs: fs.Source,
+		Fs:       contentFs,
+		Dirnames: absContentDirs,
+	}
+
 	b.SourceFilesystems = sourceFilesystems
 	b.themeFs = builder.themeFs
 	b.AbsThemeDirs = builder.absThemeDirs
@@ -281,18 +359,39 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) {
 	}
 	b.result.I18n = sfs

-	sfs, err = b.createFs("layoutDir", "layouts")
+	sfs, err = b.createFs(false, true, "layoutDir", "layouts")
 	if err != nil {
 		return nil, err
 	}
 	b.result.Layouts = sfs

-	sfs, err = b.createFs("archetypeDir", "archetypes")
+	sfs, err = b.createFs(false, true, "archetypeDir", "archetypes")
 	if err != nil {
 		return nil, err
 	}
 	b.result.Archetypes = sfs

+	sfs, err = b.createFs(false, true, "assetDir", "assets")
+	if err != nil {
+		return nil, err
+	}
+	b.result.Assets = sfs
+
+	sfs, err = b.createFs(true, false, "resourceDir", "resources")
+	if err != nil {
+		return nil, err
+	}
+
+	b.result.Resources = sfs
+
+	err = b.createStaticFs()
+
+	sfs, err = b.createFs(false, true, "", "")
+	if err != nil {
+		return nil, err
+	}
+	b.result.Work = sfs
+
 	err = b.createStaticFs()
 	if err != nil {
 		return nil, err
@@ -301,23 +400,38 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) {
 	return b.result, nil
 }

-func (b *sourceFilesystemsBuilder) createFs(dirKey, themeFolder string) (*SourceFilesystem, error) {
-	s := &SourceFilesystem{}
-	dir := b.p.Cfg.GetString(dirKey)
-	if dir == "" {
-		return s, fmt.Errorf("config %q not set", dirKey)
+func (b *sourceFilesystemsBuilder) createFs(
+	mkdir bool,
+	readOnly bool,
+	dirKey, themeFolder string) (*SourceFilesystem, error) {
+	s := &SourceFilesystem{
+		SourceFs: b.p.Fs.Source,
+	}
+	var dir string
+	if dirKey != "" {
+		dir = b.p.Cfg.GetString(dirKey)
+		if dir == "" {
+			return s, fmt.Errorf("config %q not set", dirKey)
+		}
 	}

 	var fs afero.Fs

 	absDir := b.p.AbsPathify(dir)
-	if b.existsInSource(absDir) {
-		fs = afero.NewBasePathFs(b.p.Fs.Source, absDir)
+	existsInSource := b.existsInSource(absDir)
+	if !existsInSource && mkdir {
+		// We really need this directory. Make it.
+		if err := b.p.Fs.Source.MkdirAll(absDir, 0777); err == nil {
+			existsInSource = true
+		}
+	}
+	if existsInSource {
+		fs = newRealBase(afero.NewBasePathFs(b.p.Fs.Source, absDir))
 		s.Dirnames = []string{absDir}
 	}

 	if b.hasTheme {
-		themeFolderFs := afero.NewBasePathFs(b.themeFs, themeFolder)
+		themeFolderFs := newRealBase(afero.NewBasePathFs(b.themeFs, themeFolder))
 		if fs == nil {
 			fs = themeFolderFs
 		} else {
@@ -334,8 +448,10 @@ func (b *sourceFilesystemsBuilder) createFs(dirKey, themeFolder string) (*Source

 	if fs == nil {
 		s.Fs = hugofs.NoOpFs
-	} else {
+	} else if readOnly {
 		s.Fs = afero.NewReadOnlyFs(fs)
+	} else {
+		s.Fs = fs
 	}

 	return s, nil
@@ -344,7 +460,9 @@ func (b *sourceFilesystemsBuilder) createFs(dirKey, themeFolder string) (*Source
 // Used for data, i18n -- we cannot use overlay filsesystems for those, but we need
 // to keep a strict order.
 func (b *sourceFilesystemsBuilder) createRootMappingFs(dirKey, themeFolder string) (*SourceFilesystem, error) {
-	s := &SourceFilesystem{}
+	s := &SourceFilesystem{
+		SourceFs: b.p.Fs.Source,
+	}

 	projectDir := b.p.Cfg.GetString(dirKey)
 	if projectDir == "" {
@@ -396,7 +514,9 @@ func (b *sourceFilesystemsBuilder) createStaticFs() error {

 	if isMultihost {
 		for _, l := range b.p.Languages {
-			s := &SourceFilesystem{PublishFolder: l.Lang}
+			s := &SourceFilesystem{
+				SourceFs:      b.p.Fs.Source,
+				PublishFolder: l.Lang}
 			staticDirs := removeDuplicatesKeepRight(getStaticDirs(l))
 			if len(staticDirs) == 0 {
 				continue
@@ -424,7 +544,10 @@ func (b *sourceFilesystemsBuilder) createStaticFs() error {
 		return nil
 	}

-	s := &SourceFilesystem{}
+	s := &SourceFilesystem{
+		SourceFs: b.p.Fs.Source,
+	}
+
 	var staticDirs []string

 	for _, l := range b.p.Languages {
@@ -451,7 +574,7 @@ func (b *sourceFilesystemsBuilder) createStaticFs() error {

 	if b.hasTheme {
 		themeFolder := "static"
-		fs = afero.NewCopyOnWriteFs(afero.NewBasePathFs(b.themeFs, themeFolder), fs)
+		fs = afero.NewCopyOnWriteFs(newRealBase(afero.NewBasePathFs(b.themeFs, themeFolder)), fs)
 		for _, absThemeDir := range b.absThemeDirs {
 			s.Dirnames = append(s.Dirnames, filepath.Join(absThemeDir, themeFolder))
 		}
@@ -484,7 +607,7 @@ func getStringOrStringSlice(cfg config.Provider, key string, id int) []string {
 func createContentFs(fs afero.Fs,
 	workingDir,
 	defaultContentLanguage string,
-	languages langs.Languages) (afero.Fs, []types.KeyValueStr, error) {
+	languages langs.Languages) (afero.Fs, []string, error) {

 	var contentLanguages langs.Languages
 	var contentDirSeen = make(map[string]bool)
@@ -511,7 +634,7 @@ func createContentFs(fs afero.Fs,

 	}

-	var absContentDirs []types.KeyValueStr
+	var absContentDirs []string

 	fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs)
 	return fs, absContentDirs, err
@@ -522,7 +645,7 @@ func createContentOverlayFs(source afero.Fs,
 	workingDir string,
 	languages langs.Languages,
 	languageSet map[string]bool,
-	absContentDirs *[]types.KeyValueStr) (afero.Fs, error) {
+	absContentDirs *[]string) (afero.Fs, error) {
 	if len(languages) == 0 {
 		return source, nil
 	}
@@ -548,7 +671,7 @@ func createContentOverlayFs(source afero.Fs,
 		return nil, fmt.Errorf("invalid content dir %q: Path is too short", absContentDir)
 	}

-	*absContentDirs = append(*absContentDirs, types.KeyValueStr{Key: language.Lang, Value: absContentDir})
+	*absContentDirs = append(*absContentDirs, absContentDir)

 	overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir))
 	if len(languages) == 1 {
@@ -597,10 +720,10 @@ func createOverlayFs(source afero.Fs, absPaths []string) (afero.Fs, error) {
 	}

 	if len(absPaths) == 1 {
-		return afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0])), nil
+		return afero.NewReadOnlyFs(newRealBase(afero.NewBasePathFs(source, absPaths[0]))), nil
 	}

-	base := afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0]))
+	base := afero.NewReadOnlyFs(newRealBase(afero.NewBasePathFs(source, absPaths[0])))
 	overlay, err := createOverlayFs(source, absPaths[1:])
 	if err != nil {
 		return nil, err