Add Hugo Piper with SCSS support and much more

Before this commit, you would have to use page bundles to do image processing etc. in Hugo.

This commit adds

* A new `/assets` top-level project or theme dir (configurable via `assetDir`)
* A new template func, `resources.Get` which can be used to "get a resource" that can be further processed.

This means that you can now do this in your templates (or shortcodes):

```bash
{{ $sunset := (resources.Get "images/sunset.jpg").Fill "300x200" }}
```

This also adds a new `extended` build tag that enables powerful SCSS/SASS support with source maps. To compile this from source, you will also need a C compiler installed:

```
HUGO_BUILD_TAGS=extended mage install
```

Note that you can use output of the SCSS processing later in a non-SCSSS-enabled Hugo.

The `SCSS` processor is a _Resource transformation step_ and it can be chained with the many others in a pipeline:

```bash
{{ $css := resources.Get "styles.scss" | resources.ToCSS | resources.PostCSS | resources.Minify | resources.Fingerprint }}
<link rel="stylesheet" href="{{ $styles.RelPermalink }}" integrity="{{ $styles.Data.Digest }}" media="screen">
```

The transformation funcs above have aliases, so it can be shortened to:

```bash
{{ $css := resources.Get "styles.scss" | toCSS | postCSS | minify | fingerprint }}
<link rel="stylesheet" href="{{ $styles.RelPermalink }}" integrity="{{ $styles.Data.Digest }}" media="screen">
```

A quick tip would be to avoid the fingerprinting part, and possibly also the not-superfast `postCSS` when you're doing development, as it allows Hugo to be smarter about the rebuilding.

Documentation will follow, but have a look at the demo repo in https://github.com/bep/hugo-sass-test

New functions to create `Resource` objects:

* `resources.Get` (see above)
* `resources.FromString`: Create a Resource from a string.

New `Resource` transformation funcs:

* `resources.ToCSS`: Compile `SCSS` or `SASS` into `CSS`.
* `resources.PostCSS`: Process your CSS with PostCSS. Config file support (project or theme or passed as an option).
* `resources.Minify`: Currently supports `css`, `js`, `json`, `html`, `svg`, `xml`.
* `resources.Fingerprint`: Creates a fingerprinted version of the given Resource with Subresource Integrity..
* `resources.Concat`: Concatenates a list of Resource objects. Think of this as a poor man's bundler.
* `resources.ExecuteAsTemplate`: Parses and executes the given Resource and data context (e.g. .Site) as a Go template.

Fixes #4381
Fixes #4903
Fixes #4858

commands/commandeer.go

@@ -16,6 +16,7 @@ package commands
 import (
 	"os"
 	"path/filepath"
+	"regexp"
 	"sync"
 	"time"
 
@@ -46,6 +47,10 @@ type commandeerHugoState struct {
 type commandeer struct {
 	*commandeerHugoState
 
+	// Currently only set when in "fast render mode". But it seems to
+	// be fast enough that we could maybe just add it for all server modes.
+	changeDetector *fileChangeDetector
+
 	// We need to reuse this on server rebuilds.
 	destinationFs afero.Fs
 
@@ -105,6 +110,68 @@ func newCommandeer(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f fla
 	return c, c.loadConfig(mustHaveConfigFile, running)
 }
 
+type fileChangeDetector struct {
+	sync.Mutex
+	current map[string]string
+	prev    map[string]string
+
+	irrelevantRe *regexp.Regexp
+}
+
+func (f *fileChangeDetector) OnFileClose(name, md5sum string) {
+	f.Lock()
+	defer f.Unlock()
+	f.current[name] = md5sum
+}
+
+func (f *fileChangeDetector) changed() []string {
+	if f == nil {
+		return nil
+	}
+	f.Lock()
+	defer f.Unlock()
+	var c []string
+	for k, v := range f.current {
+		vv, found := f.prev[k]
+		if !found || v != vv {
+			c = append(c, k)
+		}
+	}
+
+	return f.filterIrrelevant(c)
+}
+
+func (f *fileChangeDetector) filterIrrelevant(in []string) []string {
+	var filtered []string
+	for _, v := range in {
+		if !f.irrelevantRe.MatchString(v) {
+			filtered = append(filtered, v)
+		}
+	}
+	return filtered
+}
+
+func (f *fileChangeDetector) PrepareNew() {
+	if f == nil {
+		return
+	}
+
+	f.Lock()
+	defer f.Unlock()
+
+	if f.current == nil {
+		f.current = make(map[string]string)
+		f.prev = make(map[string]string)
+		return
+	}
+
+	f.prev = make(map[string]string)
+	for k, v := range f.current {
+		f.prev[k] = v
+	}
+	f.current = make(map[string]string)
+}
+
 func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {
 
 	if c.DepsCfg == nil {
@@ -202,6 +269,23 @@ func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {
 			fs.Destination = new(afero.MemMapFs)
 		}
 
+		doLiveReload := !c.h.buildWatch && !config.GetBool("disableLiveReload")
+		fastRenderMode := doLiveReload && !config.GetBool("disableFastRender")
+
+		if fastRenderMode {
+			// For now, fast render mode only. It should, however, be fast enough
+			// for the full variant, too.
+			changeDetector := &fileChangeDetector{
+				// We use this detector to decide to do a Hot reload of a single path or not.
+				// We need to filter out source maps and possibly some other to be able
+				// to make that decision.
+				irrelevantRe: regexp.MustCompile(`\.map$`),
+			}
+			changeDetector.PrepareNew()
+			fs.Destination = hugofs.NewHashingFs(fs.Destination, changeDetector)
+			c.changeDetector = changeDetector
+		}
+
 		err = c.initFs(fs)
 		if err != nil {
 			return

commands/hugo.go

@@ -474,6 +474,10 @@ func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint6
 	return numFiles, err
 }
 
+func (c *commandeer) firstPathSpec() *helpers.PathSpec {
+	return c.hugo.Sites[0].PathSpec
+}
+
 func (c *commandeer) timeTrack(start time.Time, name string) {
 	if c.h.quiet {
 		return
@@ -552,8 +556,8 @@ func (c *commandeer) getDirList() ([]string, error) {
 	// SymbolicWalk will log anny ERRORs
 	// Also note that the Dirnames fetched below will contain any relevant theme
 	// directories.
-	for _, contentDir := range c.hugo.PathSpec.BaseFs.AbsContentDirs {
-		_ = helpers.SymbolicWalk(c.Fs.Source, contentDir.Value, symLinkWalker)
+	for _, contentDir := range c.hugo.PathSpec.BaseFs.Content.Dirnames {
+		_ = helpers.SymbolicWalk(c.Fs.Source, contentDir, symLinkWalker)
 	}
 
 	for _, staticDir := range c.hugo.PathSpec.BaseFs.Data.Dirnames {
@@ -574,6 +578,10 @@ func (c *commandeer) getDirList() ([]string, error) {
 		}
 	}
 
+	for _, assetDir := range c.hugo.PathSpec.BaseFs.Assets.Dirnames {
+		_ = helpers.SymbolicWalk(c.Fs.Source, assetDir, regularWalker)
+	}
+
 	if len(nested) > 0 {
 		for {
 
@@ -818,13 +826,11 @@ func (c *commandeer) newWatcher(dirList ...string) (*watcher.Batcher, error) {
 						// Will block forever trying to write to a channel that nobody is reading if livereload isn't initialized
 
 						// force refresh when more than one file
-						if len(staticEvents) > 0 {
-							for _, ev := range staticEvents {
-
-								path := c.hugo.BaseFs.SourceFilesystems.MakeStaticPathRelative(ev.Name)
-								livereload.RefreshPath(path)
-							}
-
+						if len(staticEvents) == 1 {
+							ev := staticEvents[0]
+							path := c.hugo.BaseFs.SourceFilesystems.MakeStaticPathRelative(ev.Name)
+							path = c.firstPathSpec().RelURL(helpers.ToSlashTrimLeading(path), false)
+							livereload.RefreshPath(path)
 						} else {
 							livereload.ForceRefresh()
 						}
@@ -832,34 +838,54 @@ func (c *commandeer) newWatcher(dirList ...string) (*watcher.Batcher, error) {
 				}
 
 				if len(dynamicEvents) > 0 {
+					partitionedEvents := partitionDynamicEvents(
+						c.firstPathSpec().BaseFs.SourceFilesystems,
+						dynamicEvents)
+
 					doLiveReload := !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload")
-					onePageName := pickOneWriteOrCreatePath(dynamicEvents)
+					onePageName := pickOneWriteOrCreatePath(partitionedEvents.ContentEvents)
 
 					c.Logger.FEEDBACK.Println("\nChange detected, rebuilding site")
 					const layout = "2006-01-02 15:04:05.000 -0700"
 					c.Logger.FEEDBACK.Println(time.Now().Format(layout))
 
+					c.changeDetector.PrepareNew()
 					if err := c.rebuildSites(dynamicEvents); err != nil {
 						c.Logger.ERROR.Println("Failed to rebuild site:", err)
 					}
 
 					if doLiveReload {
-						navigate := c.Cfg.GetBool("navigateToChanged")
-						// We have fetched the same page above, but it may have
-						// changed.
-						var p *hugolib.Page
-
-						if navigate {
-							if onePageName != "" {
-								p = c.hugo.GetContentPage(onePageName)
+						if len(partitionedEvents.ContentEvents) == 0 && len(partitionedEvents.AssetEvents) > 0 {
+							changed := c.changeDetector.changed()
+							if c.changeDetector != nil && len(changed) == 0 {
+								// Nothing has changed.
+								continue
+							} else if len(changed) == 1 {
+								pathToRefresh := c.firstPathSpec().RelURL(helpers.ToSlashTrimLeading(changed[0]), false)
+								livereload.RefreshPath(pathToRefresh)
+							} else {
+								livereload.ForceRefresh()
 							}
-
 						}
 
-						if p != nil {
-							livereload.NavigateToPathForPort(p.RelPermalink(), p.Site.ServerPort())
-						} else {
-							livereload.ForceRefresh()
+						if len(partitionedEvents.ContentEvents) > 0 {
+
+							navigate := c.Cfg.GetBool("navigateToChanged")
+							// We have fetched the same page above, but it may have
+							// changed.
+							var p *hugolib.Page
+
+							if navigate {
+								if onePageName != "" {
+									p = c.hugo.GetContentPage(onePageName)
+								}
+							}
+
+							if p != nil {
+								livereload.NavigateToPathForPort(p.RelPermalink(), p.Site.ServerPort())
+							} else {
+								livereload.ForceRefresh()
+							}
 						}
 					}
 				}
@@ -874,6 +900,26 @@ func (c *commandeer) newWatcher(dirList ...string) (*watcher.Batcher, error) {
 	return watcher, nil
 }
 
+// dynamicEvents contains events that is considered dynamic, as in "not static".
+// Both of these categories will trigger a new build, but the asset events
+// does not fit into the "navigate to changed" logic.
+type dynamicEvents struct {
+	ContentEvents []fsnotify.Event
+	AssetEvents   []fsnotify.Event
+}
+
+func partitionDynamicEvents(sourceFs *filesystems.SourceFilesystems, events []fsnotify.Event) (de dynamicEvents) {
+	for _, e := range events {
+		if sourceFs.IsAsset(e.Name) {
+			de.AssetEvents = append(de.AssetEvents, e)
+		} else {
+			de.ContentEvents = append(de.ContentEvents, e)
+		}
+	}
+	return
+
+}
+
 func pickOneWriteOrCreatePath(events []fsnotify.Event) string {
 	name := ""