Add a HTTP cache for remote resources.

Fixes #12502
Closes #11891

hugolib/content_map_page.go

@@ -975,7 +975,7 @@ type contentTreeReverseIndexMap struct {
 
 type sitePagesAssembler struct {
 	*Site
-	assembleChanges *whatChanged
+	assembleChanges *WhatChanged
 	ctx             context.Context
 }
 

hugolib/hugo_sites.go

@@ -405,8 +405,9 @@ func (h *HugoSites) withPage(fn func(s string, p *pageState) bool) {
 type BuildCfg struct {
 	// Skip rendering. Useful for testing.
 	SkipRender bool
+
 	// Use this to indicate what changed (for rebuilds).
-	whatChanged *whatChanged
+	WhatChanged *WhatChanged
 
 	// This is a partial re-render of some selected pages.
 	PartialReRender bool

hugolib/hugo_sites_build.go

@@ -114,9 +114,9 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
 
 	// Need a pointer as this may be modified.
 	conf := &config
-	if conf.whatChanged == nil {
+	if conf.WhatChanged == nil {
 		// Assume everything has changed
-		conf.whatChanged = &whatChanged{needsPagesAssembly: true}
+		conf.WhatChanged = &WhatChanged{needsPagesAssembly: true}
 	}
 
 	var prepareErr error
@@ -128,7 +128,7 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
 					s.Deps.BuildStartListeners.Notify()
 				}
 
-				if len(events) > 0 {
+				if len(events) > 0 || len(conf.WhatChanged.Changes()) > 0 {
 					// Rebuild
 					if err := h.initRebuild(conf); err != nil {
 						return fmt.Errorf("initRebuild: %w", err)
@@ -224,7 +224,7 @@ func (h *HugoSites) initRebuild(config *BuildCfg) error {
 	})
 
 	for _, s := range h.Sites {
-		s.resetBuildState(config.whatChanged.needsPagesAssembly)
+		s.resetBuildState(config.WhatChanged.needsPagesAssembly)
 	}
 
 	h.reset(config)
@@ -245,7 +245,9 @@ func (h *HugoSites) process(ctx context.Context, l logg.LevelLogger, config *Bui
 
 	if len(events) > 0 {
 		// This is a rebuild
-		return h.processPartial(ctx, l, config, init, events)
+		return h.processPartialFileEvents(ctx, l, config, init, events)
+	} else if len(config.WhatChanged.Changes()) > 0 {
+		return h.processPartialRebuildChanges(ctx, l, config)
 	}
 	return h.processFull(ctx, l, config)
 }
@@ -256,8 +258,8 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil
 	l = l.WithField("step", "assemble")
 	defer loggers.TimeTrackf(l, time.Now(), nil, "")
 
-	if !bcfg.whatChanged.needsPagesAssembly {
-		changes := bcfg.whatChanged.Drain()
+	if !bcfg.WhatChanged.needsPagesAssembly {
+		changes := bcfg.WhatChanged.Drain()
 		if len(changes) > 0 {
 			if err := h.resolveAndClearStateForIdentities(ctx, l, nil, changes); err != nil {
 				return err
@@ -273,7 +275,7 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil
 	for i, s := range h.Sites {
 		assemblers[i] = &sitePagesAssembler{
 			Site:            s,
-			assembleChanges: bcfg.whatChanged,
+			assembleChanges: bcfg.WhatChanged,
 			ctx:             ctx,
 		}
 	}
@@ -289,7 +291,7 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil
 		return err
 	}
 
-	changes := bcfg.whatChanged.Drain()
+	changes := bcfg.WhatChanged.Drain()
 
 	// Changes from the assemble step (e.g. lastMod, cascade) needs a re-calculation
 	// of what needs to be re-built.
@@ -612,8 +614,19 @@ func (p pathChange) isStructuralChange() bool {
 	return p.delete || p.isDir
 }
 
-// processPartial prepares the Sites' sources for a partial rebuild.
-func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, config *BuildCfg, init func(config *BuildCfg) error, events []fsnotify.Event) error {
+func (h *HugoSites) processPartialRebuildChanges(ctx context.Context, l logg.LevelLogger, config *BuildCfg) error {
+	if err := h.resolveAndClearStateForIdentities(ctx, l, nil, config.WhatChanged.Drain()); err != nil {
+		return err
+	}
+
+	if err := h.processContentAdaptersOnRebuild(ctx, config); err != nil {
+		return err
+	}
+	return nil
+}
+
+// processPartialFileEvents prepares the Sites' sources for a partial rebuild.
+func (h *HugoSites) processPartialFileEvents(ctx context.Context, l logg.LevelLogger, config *BuildCfg, init func(config *BuildCfg) error, events []fsnotify.Event) error {
 	h.Log.Trace(logg.StringFunc(func() string {
 		var sb strings.Builder
 		sb.WriteString("File events:\n")
@@ -887,13 +900,13 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf
 
 	resourceFiles := h.fileEventsContentPaths(addedOrChangedContent)
 
-	changed := &whatChanged{
+	changed := &WhatChanged{
 		needsPagesAssembly: needsPagesAssemble,
 		identitySet:        make(identity.Identities),
 	}
 	changed.Add(changes...)
 
-	config.whatChanged = changed
+	config.WhatChanged = changed
 
 	if err := init(config); err != nil {
 		return err
@@ -977,14 +990,14 @@ func (s *Site) handleContentAdapterChanges(bi pagesfromdata.BuildInfo, buildConf
 	}
 
 	if len(bi.ChangedIdentities) > 0 {
-		buildConfig.whatChanged.Add(bi.ChangedIdentities...)
-		buildConfig.whatChanged.needsPagesAssembly = true
+		buildConfig.WhatChanged.Add(bi.ChangedIdentities...)
+		buildConfig.WhatChanged.needsPagesAssembly = true
 	}
 
 	for _, p := range bi.DeletedPaths {
 		pp := path.Join(bi.Path.Base(), p)
 		if v, ok := s.pageMap.treePages.Delete(pp); ok {
-			buildConfig.whatChanged.Add(v.GetIdentity())
+			buildConfig.WhatChanged.Add(v.GetIdentity())
 		}
 	}
 }

hugolib/site.go

@@ -371,14 +371,14 @@ func (s *Site) watching() bool {
 	return s.h != nil && s.h.Configs.Base.Internal.Watch
 }
 
-type whatChanged struct {
+type WhatChanged struct {
 	mu sync.Mutex
 
 	needsPagesAssembly bool
 	identitySet        identity.Identities
 }
 
-func (w *whatChanged) Add(ids ...identity.Identity) {
+func (w *WhatChanged) Add(ids ...identity.Identity) {
 	w.mu.Lock()
 	defer w.mu.Unlock()
 
@@ -391,24 +391,24 @@ func (w *whatChanged) Add(ids ...identity.Identity) {
 	}
 }
 
-func (w *whatChanged) Clear() {
+func (w *WhatChanged) Clear() {
 	w.mu.Lock()
 	defer w.mu.Unlock()
 	w.clear()
 }
 
-func (w *whatChanged) clear() {
+func (w *WhatChanged) clear() {
 	w.identitySet = identity.Identities{}
 }
 
-func (w *whatChanged) Changes() []identity.Identity {
+func (w *WhatChanged) Changes() []identity.Identity {
 	if w == nil || w.identitySet == nil {
 		return nil
 	}
 	return w.identitySet.AsSlice()
 }
 
-func (w *whatChanged) Drain() []identity.Identity {
+func (w *WhatChanged) Drain() []identity.Identity {
 	w.mu.Lock()
 	defer w.mu.Unlock()
 	ids := w.identitySet.AsSlice()

hugolib/site_new.go

@@ -141,10 +141,23 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
 
 	memCache := dynacache.New(dynacache.Options{Watching: conf.Watching(), Log: logger})
 
+	var h *HugoSites
+	onSignalRebuild := func(ids ...identity.Identity) {
+		// This channel is buffered, but make sure we do this in a non-blocking way.
+		if cfg.ChangesFromBuild != nil {
+			go func() {
+				cfg.ChangesFromBuild <- ids
+			}()
+		}
+	}
+
 	firstSiteDeps := &deps.Deps{
-		Fs:                  cfg.Fs,
-		Log:                 logger,
-		Conf:                conf,
+		Fs:   cfg.Fs,
+		Log:  logger,
+		Conf: conf,
+		BuildState: &deps.BuildState{
+			OnSignalRebuild: onSignalRebuild,
+		},
 		MemCache:            memCache,
 		TemplateProvider:    tplimpl.DefaultTemplateProvider,
 		TranslationProvider: i18n.NewTranslationProvider(),
@@ -261,7 +274,8 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
 		return li.Lang < lj.Lang
 	})
 
-	h, err := newHugoSites(cfg, firstSiteDeps, pageTrees, sites)
+	var err error
+	h, err = newHugoSites(cfg, firstSiteDeps, pageTrees, sites)
 	if err == nil && h == nil {
 		panic("hugo: newHugoSitesNew returned nil error and nil HugoSites")
 	}