Introduce a tree map for all content

This commit introduces a new data structure to store pages and their resources. This data structure is backed by radix trees. This simplies tree operations, makes all pages a bundle, and paves the way for #6310. It also solves a set of annoying issues (see list below). Not a motivation behind this, but this commit also makes Hugo in general a little bit faster and more memory effective (see benchmarks). Especially for partial rebuilds on content edits, but also when taxonomies is in use. ``` name old time/op new time/op delta SiteNew/Bundle_with_image/Edit-16 1.32ms ± 8% 1.00ms ± 9% -24.42% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file/Edit-16 1.28ms ± 0% 0.94ms ± 0% -26.26% (p=0.029 n=4+4) SiteNew/Tags_and_categories/Edit-16 33.9ms ± 2% 21.8ms ± 1% -35.67% (p=0.029 n=4+4) SiteNew/Canonify_URLs/Edit-16 40.6ms ± 1% 37.7ms ± 3% -7.20% (p=0.029 n=4+4) SiteNew/Deep_content_tree/Edit-16 56.7ms ± 0% 51.7ms ± 1% -8.82% (p=0.029 n=4+4) SiteNew/Many_HTML_templates/Edit-16 19.9ms ± 2% 18.3ms ± 3% -7.64% (p=0.029 n=4+4) SiteNew/Page_collections/Edit-16 37.9ms ± 4% 34.0ms ± 2% -10.28% (p=0.029 n=4+4) SiteNew/Bundle_with_image-16 10.7ms ± 0% 10.6ms ± 0% -1.15% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 10.8ms ± 0% 10.7ms ± 0% -1.05% (p=0.029 n=4+4) SiteNew/Tags_and_categories-16 43.2ms ± 1% 39.6ms ± 1% -8.35% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 47.6ms ± 1% 47.3ms ± 0% ~ (p=0.057 n=4+4) SiteNew/Deep_content_tree-16 73.0ms ± 1% 74.2ms ± 1% ~ (p=0.114 n=4+4) SiteNew/Many_HTML_templates-16 37.9ms ± 0% 38.1ms ± 1% ~ (p=0.114 n=4+4) SiteNew/Page_collections-16 53.6ms ± 1% 54.7ms ± 1% +2.09% (p=0.029 n=4+4) name old alloc/op new alloc/op delta SiteNew/Bundle_with_image/Edit-16 486kB ± 0% 430kB ± 0% -11.47% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file/Edit-16 265kB ± 0% 209kB ± 0% -21.06% (p=0.029 n=4+4) SiteNew/Tags_and_categories/Edit-16 13.6MB ± 0% 8.8MB ± 0% -34.93% (p=0.029 n=4+4) SiteNew/Canonify_URLs/Edit-16 66.5MB ± 0% 63.9MB ± 0% -3.95% (p=0.029 n=4+4) SiteNew/Deep_content_tree/Edit-16 28.8MB ± 0% 25.8MB ± 0% -10.55% (p=0.029 n=4+4) SiteNew/Many_HTML_templates/Edit-16 6.16MB ± 0% 5.56MB ± 0% -9.86% (p=0.029 n=4+4) SiteNew/Page_collections/Edit-16 16.9MB ± 0% 16.0MB ± 0% -5.19% (p=0.029 n=4+4) SiteNew/Bundle_with_image-16 2.28MB ± 0% 2.29MB ± 0% +0.35% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 2.07MB ± 0% 2.07MB ± 0% ~ (p=0.114 n=4+4) SiteNew/Tags_and_categories-16 14.3MB ± 0% 13.2MB ± 0% -7.30% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 69.1MB ± 0% 69.0MB ± 0% ~ (p=0.343 n=4+4) SiteNew/Deep_content_tree-16 31.3MB ± 0% 31.8MB ± 0% +1.49% (p=0.029 n=4+4) SiteNew/Many_HTML_templates-16 10.8MB ± 0% 10.9MB ± 0% +1.11% (p=0.029 n=4+4) SiteNew/Page_collections-16 21.4MB ± 0% 21.6MB ± 0% +1.15% (p=0.029 n=4+4) name old allocs/op new allocs/op delta SiteNew/Bundle_with_image/Edit-16 4.74k ± 0% 3.86k ± 0% -18.57% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file/Edit-16 4.73k ± 0% 3.85k ± 0% -18.58% (p=0.029 n=4+4) SiteNew/Tags_and_categories/Edit-16 301k ± 0% 198k ± 0% -34.14% (p=0.029 n=4+4) SiteNew/Canonify_URLs/Edit-16 389k ± 0% 373k ± 0% -4.07% (p=0.029 n=4+4) SiteNew/Deep_content_tree/Edit-16 338k ± 0% 262k ± 0% -22.63% (p=0.029 n=4+4) SiteNew/Many_HTML_templates/Edit-16 102k ± 0% 88k ± 0% -13.81% (p=0.029 n=4+4) SiteNew/Page_collections/Edit-16 176k ± 0% 152k ± 0% -13.32% (p=0.029 n=4+4) SiteNew/Bundle_with_image-16 26.8k ± 0% 26.8k ± 0% +0.05% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 26.8k ± 0% 26.8k ± 0% +0.05% (p=0.029 n=4+4) SiteNew/Tags_and_categories-16 273k ± 0% 245k ± 0% -10.36% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 396k ± 0% 398k ± 0% +0.39% (p=0.029 n=4+4) SiteNew/Deep_content_tree-16 317k ± 0% 325k ± 0% +2.53% (p=0.029 n=4+4) SiteNew/Many_HTML_templates-16 146k ± 0% 147k ± 0% +0.98% (p=0.029 n=4+4) SiteNew/Page_collections-16 210k ± 0% 215k ± 0% +2.44% (p=0.029 n=4+4) ``` Fixes #6312 Fixes #6087 Fixes #6738 Fixes #6412 Fixes #6743 Fixes #6875 Fixes #6034 Fixes #6902 Fixes #6173 Fixes #6590
2025-08-28 22:19:59 +02:00 · 2019-09-10 11:26:34 +02:00
parent e5329f13c0
commit eada236f87
71 changed files with 4859 additions and 2531 deletions
--- a/hugolib/page__meta.go
+++ b/hugolib/page__meta.go
@@ -61,7 +61,10 @@ type pageMeta struct {
 	// a fixed pageOutput.
 	standalone bool

-	bundleType string
+	draft       bool // Only published when running with -D flag
+	buildConfig pagemeta.BuildConfig
+
+	bundleType files.ContentClass

 	// Params contains configuration defined in the params section of page frontmatter.
 	params map[string]interface{}
@@ -85,8 +88,6 @@ type pageMeta struct {

 	aliases []string

-	draft bool
-
 	description string
 	keywords    []string

@@ -94,13 +95,6 @@ type pageMeta struct {

 	resource.Dates

-	// This is enabled if it is a leaf bundle (the "index.md" type) and it is marked as headless in front matter.
-	// Being headless means that
-	// 1. The page itself is not rendered to disk
-	// 2. It is not available in .Site.Pages etc.
-	// 3. But you can get it via .Site.GetPage
-	headless bool
-
 	// Set if this page is bundled inside another.
 	bundled bool

@@ -160,7 +154,7 @@ func (p *pageMeta) Authors() page.AuthorList {
 	return al
 }

-func (p *pageMeta) BundleType() string {
+func (p *pageMeta) BundleType() files.ContentClass {
 	return p.bundleType
 }

@@ -309,40 +303,53 @@ func (p *pageMeta) Weight() int {
 	return p.weight
 }

-func (pm *pageMeta) setMetadata(bucket *pagesMapBucket, p *pageState, frontmatter map[string]interface{}) error {
-	if frontmatter == nil && bucket.cascade == nil {
-		return errors.New("missing frontmatter data")
+func (pm *pageMeta) mergeBucketCascades(b1, b2 *pagesMapBucket) {
+	if b1.cascade == nil {
+		b1.cascade = make(map[string]interface{})
 	}
+	if b2 != nil && b2.cascade != nil {
+		for k, v := range b2.cascade {
+			if _, found := b1.cascade[k]; !found {
+				b1.cascade[k] = v
+			}
+		}
+	}
+}

+func (pm *pageMeta) setMetadata(parentBucket *pagesMapBucket, p *pageState, frontmatter map[string]interface{}) error {
 	pm.params = make(maps.Params)

+	if frontmatter == nil && (parentBucket == nil || parentBucket.cascade == nil) {
+		return nil
+	}
+
 	if frontmatter != nil {
 		// Needed for case insensitive fetching of params values
 		maps.ToLower(frontmatter)
-		if p.IsNode() {
+		if p.bucket != nil {
 			// Check for any cascade define on itself.
 			if cv, found := frontmatter["cascade"]; found {
-				cvm := maps.ToStringMap(cv)
-				if bucket.cascade == nil {
-					bucket.cascade = cvm
-				} else {
-					for k, v := range cvm {
-						bucket.cascade[k] = v
-					}
-				}
-			}
-		}
-
-		if bucket != nil && bucket.cascade != nil {
-			for k, v := range bucket.cascade {
-				if _, found := frontmatter[k]; !found {
-					frontmatter[k] = v
-				}
+				p.bucket.cascade = maps.ToStringMap(cv)
 			}
 		}
 	} else {
 		frontmatter = make(map[string]interface{})
-		for k, v := range bucket.cascade {
+	}
+
+	var cascade map[string]interface{}
+
+	if p.bucket != nil {
+		if parentBucket != nil {
+			// Merge missing keys from parent into this.
+			pm.mergeBucketCascades(p.bucket, parentBucket)
+		}
+		cascade = p.bucket.cascade
+	} else if parentBucket != nil {
+		cascade = parentBucket.cascade
+	}
+
+	for k, v := range cascade {
+		if _, found := frontmatter[k]; !found {
 			frontmatter[k] = v
 		}
 	}
@@ -379,6 +386,11 @@ func (pm *pageMeta) setMetadata(bucket *pagesMapBucket, p *pageState, frontmatte
 		p.s.Log.ERROR.Printf("Failed to handle dates for page %q: %s", p.pathOrTitle(), err)
 	}

+	pm.buildConfig, err = pagemeta.DecodeBuildConfig(frontmatter["_build"])
+	if err != nil {
+		return err
+	}
+
 	var sitemapSet bool

 	var draft, published, isCJKLanguage *bool
@@ -439,12 +451,15 @@ func (pm *pageMeta) setMetadata(bucket *pagesMapBucket, p *pageState, frontmatte
 			pm.keywords = cast.ToStringSlice(v)
 			pm.params[loki] = pm.keywords
 		case "headless":
-			// For now, only the leaf bundles ("index.md") can be headless (i.e. produce no output).
-			// We may expand on this in the future, but that gets more complex pretty fast.
-			if p.File().TranslationBaseName() == "index" {
-				pm.headless = cast.ToBool(v)
+			// Legacy setting for leaf bundles.
+			// This is since Hugo 0.63 handled in a more general way for all
+			// pages.
+			isHeadless := cast.ToBool(v)
+			pm.params[loki] = isHeadless
+			if p.File().TranslationBaseName() == "index" && isHeadless {
+				pm.buildConfig.List = false
+				pm.buildConfig.Render = false
 			}
-			pm.params[loki] = pm.headless
 		case "outputs":
 			o := cast.ToStringSlice(v)
 			if len(o) > 0 {
@@ -594,7 +609,23 @@ func (pm *pageMeta) setMetadata(bucket *pagesMapBucket, p *pageState, frontmatte
 	return nil
 }

-func (p *pageMeta) applyDefaultValues(ps *pageState) error {
+func (p *pageMeta) noList() bool {
+	return !p.buildConfig.List
+}
+
+func (p *pageMeta) noRender() bool {
+	return !p.buildConfig.Render
+}
+
+func (p *pageMeta) applyDefaultValues(n *contentNode) error {
+	if p.buildConfig.IsZero() {
+		p.buildConfig, _ = pagemeta.DecodeBuildConfig(nil)
+	}
+
+	if !p.s.isEnabled(p.Kind()) {
+		(&p.buildConfig).Disable()
+	}
+
 	if p.markup == "" {
 		if !p.File().IsZero() {
 			// Fall back to file extension
@@ -610,13 +641,21 @@ func (p *pageMeta) applyDefaultValues(ps *pageState) error {
 		case page.KindHome:
 			p.title = p.s.Info.title
 		case page.KindSection:
-			sectionName := helpers.FirstUpper(p.sections[0])
+			var sectionName string
+			if n != nil {
+				sectionName = n.rootSection()
+			} else {
+				sectionName = p.sections[0]
+			}
+
+			sectionName = helpers.FirstUpper(sectionName)
 			if p.s.Cfg.GetBool("pluralizeListTitles") {
 				p.title = inflect.Pluralize(sectionName)
 			} else {
 				p.title = sectionName
 			}
 		case page.KindTaxonomy:
+			// TODO(bep) improve
 			key := p.sections[len(p.sections)-1]
 			p.title = strings.Replace(p.s.titleFunc(key), "-", " ", -1)
 		case page.KindTaxonomyTerm:
@@ -653,7 +692,7 @@ func (p *pageMeta) applyDefaultValues(ps *pageState) error {
 			markup = "markdown"
 		}

-		cp, err := p.newContentConverter(ps, markup, renderingConfigOverrides)
+		cp, err := p.newContentConverter(n.p, markup, renderingConfigOverrides)
 		if err != nil {
 			return err
 		}
@@ -665,6 +704,9 @@ func (p *pageMeta) applyDefaultValues(ps *pageState) error {
 }

 func (p *pageMeta) newContentConverter(ps *pageState, markup string, renderingConfigOverrides map[string]interface{}) (converter.Converter, error) {
+	if ps == nil {
+		panic("no Page provided")
+	}
 	cp := p.s.ContentSpec.Converters.Get(markup)
 	if cp == nil {
 		return nil, errors.Errorf("no content renderer found for markup %q", p.markup)