Implement defer

Closes #8086
Closes #12589
This commit is contained in:
Bjørn Erik Pedersen
2024-06-08 11:52:22 +02:00
parent 8731d88222
commit 6cd0784e44
33 changed files with 1033 additions and 148 deletions

View File

@@ -20,11 +20,13 @@ import (
"reflect"
"regexp"
"strings"
"sync"
"unicode"
bp "github.com/gohugoio/hugo/bufferpool"
"github.com/gohugoio/hugo/common/hcontext"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/langs"
"github.com/gohugoio/hugo/output/layouts"
"github.com/gohugoio/hugo/output"
@@ -160,6 +162,11 @@ type TemplateFuncGetter interface {
GetFunc(name string) (reflect.Value, bool)
}
type RenderingContext struct {
Site site
SiteOutIdx int
}
type contextKey string
// Context manages values passed in the context to templates.
@@ -191,6 +198,15 @@ type page interface {
IsNode() bool
}
type site interface {
Language() *langs.Language
}
const (
HugoDeferredTemplatePrefix = "__hdeferred/"
HugoDeferredTemplateSuffix = "__d="
)
const hugoNewLinePlaceholder = "___hugonl_"
var stripHTMLReplacerPre = strings.NewReplacer("\n", " ", "</p>", hugoNewLinePlaceholder, "<br>", hugoNewLinePlaceholder, "<br />", hugoNewLinePlaceholder)
@@ -228,3 +244,13 @@ func StripHTML(s string) string {
return s
}
type DeferredExecution struct {
Mu sync.Mutex
Ctx context.Context
TemplateName string
Data any
Executed bool
Result string
}

View File

@@ -0,0 +1,202 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package templates_test
import (
"fmt"
"path/filepath"
"strings"
"testing"
qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/hugolib"
)
const deferFilesCommon = `
-- hugo.toml --
disableLiveReload = true
disableKinds = ["taxonomy", "term", "rss", "sitemap", "robotsTXT", "404", "section"]
[languages]
[languages.en]
weight = 1
[languages.nn]
weight = 2
-- i18n/en.toml --
[hello]
other = "Hello"
-- i18n/nn.toml --
[hello]
other = "Hei"
-- content/_index.en.md --
---
title: "Home"
outputs: ["html", "amp"]
---
-- content/_index.nn.md --
---
title: "Heim"
outputs: ["html", "amp"]
---
-- assets/mytext.txt --
Hello.
-- layouts/baseof.html --
HTML|{{ block "main" . }}{{ end }}$
-- layouts/index.html --
{{ define "main" }}
EDIT_COUNTER_OUTSIDE_0
{{ .Store.Set "hello" "Hello" }}
{{ $data := dict "page" . }}
{{ with (templates.Defer (dict "data" $data) ) }}
{{ $mytext := resources.Get "mytext.txt" }}
REPLACE_ME|Title: {{ .page.Title }}|{{ .page.RelPermalink }}|Hello: {{ T "hello" }}|Hello Store: {{ .page.Store.Get "hello" }}|Mytext: {{ $mytext.Content }}|
EDIT_COUNTER_DEFER_0
{{ end }}$
{{ end }}
-- layouts/index.amp.html --
AMP.
{{ $data := dict "page" . }}
{{ with (templates.Defer (dict "data" $data) ) }}Title AMP: {{ .page.Title }}|{{ .page.RelPermalink }}|Hello: {{ T "hello" }}{{ end }}$
`
func TestDeferBasic(t *testing.T) {
t.Parallel()
b := hugolib.Test(t, deferFilesCommon)
b.AssertFileContent("public/index.html", "Title: Home|/|Hello: Hello|Hello Store: Hello|Mytext: Hello.|")
b.AssertFileContent("public/amp/index.html", "Title AMP: Home|/amp/|Hello: Hello")
b.AssertFileContent("public/nn/index.html", "Title: Heim|/nn/|Hello: Hei")
b.AssertFileContent("public/nn/amp/index.html", "Title AMP: Heim|/nn/amp/|Hello: Hei")
}
func TestDeferRepeatedBuildsEditOutside(t *testing.T) {
t.Parallel()
b := hugolib.TestRunning(t, deferFilesCommon)
for i := 0; i < 5; i++ {
old := fmt.Sprintf("EDIT_COUNTER_OUTSIDE_%d", i)
new := fmt.Sprintf("EDIT_COUNTER_OUTSIDE_%d", i+1)
b.EditFileReplaceAll("layouts/index.html", old, new).Build()
b.AssertFileContent("public/index.html", new)
}
}
func TestDeferRepeatedBuildsEditDefer(t *testing.T) {
t.Parallel()
b := hugolib.TestRunning(t, deferFilesCommon)
for i := 0; i < 8; i++ {
old := fmt.Sprintf("EDIT_COUNTER_DEFER_%d", i)
new := fmt.Sprintf("EDIT_COUNTER_DEFER_%d", i+1)
b.EditFileReplaceAll("layouts/index.html", old, new).Build()
b.AssertFileContent("public/index.html", new)
}
}
func TestDeferErrorParse(t *testing.T) {
t.Parallel()
b, err := hugolib.TestE(t, strings.ReplaceAll(deferFilesCommon, "Title AMP: {{ .page.Title }}", "{{ .page.Title }"))
b.Assert(err, qt.Not(qt.IsNil))
b.Assert(err.Error(), qt.Contains, `index.amp.html:3: unexpected "}" in operand`)
}
func TestDeferErrorRuntime(t *testing.T) {
t.Parallel()
b, err := hugolib.TestE(t, strings.ReplaceAll(deferFilesCommon, "Title AMP: {{ .page.Title }}", "{{ .page.Titles }}"))
b.Assert(err, qt.Not(qt.IsNil))
b.Assert(err.Error(), qt.Contains, filepath.FromSlash(`/layouts/index.amp.html:3:57`))
b.Assert(err.Error(), qt.Contains, `execute of template failed: template: index.amp.html:3:57: executing at <.page.Titles>: can't evaluate field Titles`)
}
func TestDeferEditDeferBlock(t *testing.T) {
t.Parallel()
b := hugolib.TestRunning(t, deferFilesCommon)
b.AssertRenderCountPage(4)
b.EditFileReplaceAll("layouts/index.html", "REPLACE_ME", "Edited.").Build()
b.AssertFileContent("public/index.html", "Edited.")
b.AssertRenderCountPage(2)
}
//
func TestDeferEditResourceUsedInDeferBlock(t *testing.T) {
t.Parallel()
b := hugolib.TestRunning(t, deferFilesCommon)
b.AssertRenderCountPage(4)
b.EditFiles("assets/mytext.txt", "Mytext Hello Edited.").Build()
b.AssertFileContent("public/index.html", "Mytext Hello Edited.")
b.AssertRenderCountPage(2)
}
func TestDeferMountPublic(t *testing.T) {
t.Parallel()
files := `
-- hugo.toml --
[module]
[[module.mounts]]
source = "content"
target = "content"
[[module.mounts]]
source = "layouts"
target = "layouts"
[[module.mounts]]
source = 'public'
target = 'assets/public'
disableWatch = true
-- layouts/index.html --
Home.
{{ $mydata := dict "v1" "v1value" }}
{{ $json := resources.FromString "mydata/data.json" ($mydata | jsonify ) }}
{{ $nop := $json.RelPermalink }}
{{ with (templates.Defer (dict "key" "foo")) }}
{{ $jsonFilePublic := resources.Get "public/mydata/data.json" }}
{{ with $jsonFilePublic }}
{{ $m := $jsonFilePublic | transform.Unmarshal }}
v1: {{ $m.v1 }}
{{ end }}
{{ end }}
`
b := hugolib.Test(t, files)
b.AssertFileContent("public/index.html", "v1: v1value")
}
func TestDeferFromContentAdapterShouldFail(t *testing.T) {
t.Parallel()
files := `
-- hugo.toml --
-- content/_content.gotmpl --
{{ with (templates.Defer (dict "key" "foo")) }}
Foo.
{{ end }}
`
b, err := hugolib.TestE(t, files)
b.Assert(err, qt.Not(qt.IsNil))
b.Assert(err.Error(), qt.Contains, "error calling Defer: this method cannot be called before the site is fully initialized")
}

View File

@@ -39,6 +39,16 @@ func init() {
},
)
ns.AddMethodMapping(ctx.Defer,
nil, // No aliases to keep the AST parsing simple.
[][2]string{},
)
ns.AddMethodMapping(ctx.DoDefer,
[]string{"doDefer"},
[][2]string{},
)
return ns
}

View File

@@ -15,14 +15,24 @@
package templates
import (
"context"
"fmt"
"strconv"
"sync/atomic"
"github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/tpl"
"github.com/mitchellh/mapstructure"
)
// New returns a new instance of the templates-namespaced template functions.
func New(deps *deps.Deps) *Namespace {
return &Namespace{
ns := &Namespace{
deps: deps,
}
return ns
}
// Namespace provides template functions for the "templates" namespace.
@@ -36,3 +46,59 @@ type Namespace struct {
func (ns *Namespace) Exists(name string) bool {
return ns.deps.Tmpl().HasTemplate(name)
}
// Defer defers the execution of a template block.
func (ns *Namespace) Defer(args ...any) (bool, error) {
// Prevent defer from being used in content adapters,
// that just doesn't work.
ns.deps.Site.CheckReady()
if len(args) != 0 {
return false, fmt.Errorf("Defer does not take any arguments")
}
return true, nil
}
var defferedIDCounter atomic.Uint64
type DeferOpts struct {
// Optional cache key. If set, the deferred block will be executed
// once per unique key.
Key string
// Optional data context to use when executing the deferred block.
Data any
}
// DoDefer defers the execution of a template block.
// For internal use only.
func (ns *Namespace) DoDefer(ctx context.Context, id string, optsv any) string {
var opts DeferOpts
if optsv != nil {
if err := mapstructure.WeakDecode(optsv, &opts); err != nil {
panic(err)
}
}
templateName := id
var key string
if opts.Key != "" {
key = helpers.MD5String(opts.Key)
} else {
key = strconv.FormatUint(defferedIDCounter.Add(1), 10)
}
id = fmt.Sprintf("%s_%s%s", id, key, tpl.HugoDeferredTemplateSuffix)
_ = ns.deps.BuildState.DeferredExecutions.Executions.GetOrCreate(id,
func() *tpl.DeferredExecution {
return &tpl.DeferredExecution{
TemplateName: templateName,
Ctx: ctx,
Data: opts.Data,
Executed: false,
}
})
return id
}

View File

@@ -42,6 +42,7 @@ import (
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse"
htmltemplate "github.com/gohugoio/hugo/tpl/internal/go_templates/htmltemplate"
texttemplate "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate"
@@ -194,11 +195,12 @@ func newTemplateNamespace(funcs map[string]any) *templateNamespace {
}
}
func newTemplateState(templ tpl.Template, info templateInfo, id identity.Identity) *templateState {
func newTemplateState(owner *templateState, templ tpl.Template, info templateInfo, id identity.Identity) *templateState {
if id == nil {
id = info
}
return &templateState{
owner: owner,
info: info,
typ: info.resolveType(),
Template: templ,
@@ -260,7 +262,11 @@ func (t *templateExec) ExecuteWithContext(ctx context.Context, templ tpl.Templat
execErr := t.executor.ExecuteWithContext(ctx, templ, wr, data)
if execErr != nil {
execErr = t.addFileContext(templ, execErr)
owner := templ
if ts, ok := templ.(*templateState); ok && ts.owner != nil {
owner = ts.owner
}
execErr = t.addFileContext(owner, execErr)
}
return execErr
}
@@ -312,6 +318,9 @@ func (t *templateExec) MarkReady() error {
// We only need the clones if base templates are in use.
if len(t.needsBaseof) > 0 {
err = t.main.createPrototypes()
if err != nil {
return
}
}
})
@@ -369,7 +378,7 @@ type layoutCacheEntry struct {
func (t *templateHandler) AddTemplate(name, tpl string) error {
templ, err := t.addTemplateTo(t.newTemplateInfo(name, tpl), t.main)
if err == nil {
t.applyTemplateTransformers(t.main, templ)
_, err = t.applyTemplateTransformers(t.main, templ)
}
return err
}
@@ -390,6 +399,7 @@ func (t *templateHandler) LookupLayout(d layouts.LayoutDescriptor, f output.Form
t.layoutTemplateCacheMu.RUnlock()
return cacheVal.templ, cacheVal.found, cacheVal.err
}
t.layoutTemplateCacheMu.RUnlock()
t.layoutTemplateCacheMu.Lock()
@@ -497,13 +507,15 @@ func (t *templateHandler) findLayout(d layouts.LayoutDescriptor, f output.Format
return nil, false, err
}
ts := newTemplateState(templ, overlay, identity.Or(base, overlay))
ts := newTemplateState(nil, templ, overlay, identity.Or(base, overlay))
if found {
ts.baseInfo = base
}
t.applyTemplateTransformers(t.main, ts)
if _, err := t.applyTemplateTransformers(t.main, ts); err != nil {
return nil, false, err
}
if err := t.extractPartials(ts.Template); err != nil {
return nil, false, err
@@ -674,7 +686,10 @@ func (t *templateHandler) addTemplateFile(name string, fim hugofs.FileMetaInfo)
if err != nil {
return tinfo.errWithFileContext("parse failed", err)
}
t.applyTemplateTransformers(t.main, templ)
if _, err = t.applyTemplateTransformers(t.main, templ); err != nil {
return tinfo.errWithFileContext("transform failed", err)
}
return nil
}
@@ -745,6 +760,12 @@ func (t *templateHandler) applyTemplateTransformers(ns *templateNamespace, ts *t
t.transformNotFound[k] = ts
}
for k, v := range c.deferNodes {
if err = t.main.addDeferredTemplate(ts, k, v); err != nil {
return nil, err
}
}
return c, err
}
@@ -858,7 +879,7 @@ func (t *templateHandler) extractPartials(templ tpl.Template) error {
continue
}
ts := newTemplateState(templ, templateInfo{name: templ.Name()}, nil)
ts := newTemplateState(nil, templ, templateInfo{name: templ.Name()}, nil)
ts.typ = templatePartial
t.main.mu.RLock()
@@ -954,18 +975,18 @@ type templateNamespace struct {
*templateStateMap
}
func (t templateNamespace) Clone() *templateNamespace {
t.mu.Lock()
defer t.mu.Unlock()
t.templateStateMap = &templateStateMap{
templates: make(map[string]*templateState),
func (t *templateNamespace) getPrototypeText() *texttemplate.Template {
if t.prototypeTextClone != nil {
return t.prototypeTextClone
}
return t.prototypeText
}
t.prototypeText = texttemplate.Must(t.prototypeText.Clone())
t.prototypeHTML = htmltemplate.Must(t.prototypeHTML.Clone())
return &t
func (t *templateNamespace) getPrototypeHTML() *htmltemplate.Template {
if t.prototypeHTMLClone != nil {
return t.prototypeHTMLClone
}
return t.prototypeHTML
}
func (t *templateNamespace) Lookup(name string) (tpl.Template, bool) {
@@ -996,12 +1017,46 @@ func (t *templateNamespace) newTemplateLookup(in *templateState) func(name strin
return templ
}
if templ, found := findTemplateIn(name, in); found {
return newTemplateState(templ, templateInfo{name: templ.Name()}, nil)
return newTemplateState(nil, templ, templateInfo{name: templ.Name()}, nil)
}
return nil
}
}
func (t *templateNamespace) addDeferredTemplate(owner *templateState, name string, n *parse.ListNode) error {
t.mu.Lock()
defer t.mu.Unlock()
if _, found := t.templates[name]; found {
return nil
}
var templ tpl.Template
if owner.isText() {
prototype := t.getPrototypeText()
tt, err := prototype.New(name).Parse("")
if err != nil {
return fmt.Errorf("failed to parse empty text template %q: %w", name, err)
}
tt.Tree.Root = n
templ = tt
} else {
prototype := t.getPrototypeHTML()
tt, err := prototype.New(name).Parse("")
if err != nil {
return fmt.Errorf("failed to parse empty HTML template %q: %w", name, err)
}
tt.Tree.Root = n
templ = tt
}
dts := newTemplateState(owner, templ, templateInfo{name: name}, nil)
t.templates[name] = dts
return nil
}
func (t *templateNamespace) parse(info templateInfo) (*templateState, error) {
t.mu.Lock()
defer t.mu.Unlock()
@@ -1014,7 +1069,7 @@ func (t *templateNamespace) parse(info templateInfo) (*templateState, error) {
return nil, err
}
ts := newTemplateState(templ, info, nil)
ts := newTemplateState(nil, templ, info, nil)
t.templates[info.name] = ts
@@ -1028,7 +1083,7 @@ func (t *templateNamespace) parse(info templateInfo) (*templateState, error) {
return nil, err
}
ts := newTemplateState(templ, info, nil)
ts := newTemplateState(nil, templ, info, nil)
t.templates[info.name] = ts
@@ -1040,6 +1095,9 @@ var _ tpl.IsInternalTemplateProvider = (*templateState)(nil)
type templateState struct {
tpl.Template
// Set for deferred templates.
owner *templateState
typ templateType
parseInfo tpl.ParseInfo
id identity.Identity

View File

@@ -17,6 +17,7 @@ import (
"errors"
"fmt"
"github.com/gohugoio/hugo/helpers"
htmltemplate "github.com/gohugoio/hugo/tpl/internal/go_templates/htmltemplate"
texttemplate "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate"
@@ -38,6 +39,7 @@ const (
type templateContext struct {
visited map[string]bool
templateNotFound map[string]bool
deferNodes map[string]*parse.ListNode
lookupFn func(name string) *templateState
// The last error encountered.
@@ -77,6 +79,7 @@ func newTemplateContext(
lookupFn: lookupFn,
visited: make(map[string]bool),
templateNotFound: make(map[string]bool),
deferNodes: make(map[string]*parse.ListNode),
}
}
@@ -116,9 +119,14 @@ const (
// "range" over a one-element slice so we can shift dot to the
// partial's argument, Arg, while allowing Arg to be falsy.
partialReturnWrapperTempl = `{{ $_hugo_dot := $ }}{{ $ := .Arg }}{{ range (slice .Arg) }}{{ $_hugo_dot.Set ("PLACEHOLDER") }}{{ end }}`
doDeferTempl = `{{ doDefer ("PLACEHOLDER1") ("PLACEHOLDER2") }}`
)
var partialReturnWrapper *parse.ListNode
var (
partialReturnWrapper *parse.ListNode
doDefer *parse.ListNode
)
func init() {
templ, err := texttemplate.New("").Parse(partialReturnWrapperTempl)
@@ -126,6 +134,12 @@ func init() {
panic(err)
}
partialReturnWrapper = templ.Tree.Root
templ, err = texttemplate.New("").Funcs(texttemplate.FuncMap{"doDefer": func(string, string) string { return "" }}).Parse(doDeferTempl)
if err != nil {
panic(err)
}
doDefer = templ.Tree.Root
}
// wrapInPartialReturnWrapper copies and modifies the parsed nodes of a
@@ -158,6 +172,7 @@ func (c *templateContext) applyTransformations(n parse.Node) (bool, error) {
case *parse.IfNode:
c.applyTransformationsToNodes(x.Pipe, x.List, x.ElseList)
case *parse.WithNode:
c.handleDefer(x)
c.applyTransformationsToNodes(x.Pipe, x.List, x.ElseList)
case *parse.RangeNode:
c.applyTransformationsToNodes(x.Pipe, x.List, x.ElseList)
@@ -191,6 +206,58 @@ func (c *templateContext) applyTransformations(n parse.Node) (bool, error) {
return true, c.err
}
func (c *templateContext) handleDefer(withNode *parse.WithNode) {
if len(withNode.Pipe.Cmds) != 1 {
return
}
cmd := withNode.Pipe.Cmds[0]
if len(cmd.Args) != 1 {
return
}
idArg := cmd.Args[0]
p, ok := idArg.(*parse.PipeNode)
if !ok {
return
}
if len(p.Cmds) != 1 {
return
}
cmd = p.Cmds[0]
if len(cmd.Args) != 2 {
return
}
idArg = cmd.Args[0]
id, ok := idArg.(*parse.ChainNode)
if !ok || len(id.Field) != 1 || id.Field[0] != "Defer" {
return
}
if id2, ok := id.Node.(*parse.IdentifierNode); !ok || id2.Ident != "templates" {
return
}
deferArg := cmd.Args[1]
cmd.Args = []parse.Node{idArg}
l := doDefer.CopyList()
n := l.Nodes[0].(*parse.ActionNode)
inner := withNode.List.CopyList()
innerHash := helpers.MD5String(inner.String())
deferredID := tpl.HugoDeferredTemplatePrefix + innerHash
c.deferNodes[deferredID] = inner
withNode.List = l
n.Pipe.Cmds[0].Args[1].(*parse.PipeNode).Cmds[0].Args[0].(*parse.StringNode).Text = deferredID
n.Pipe.Cmds[0].Args[2] = deferArg
}
func (c *templateContext) applyTransformationsToNodes(nodes ...parse.Node) {
for _, node := range nodes {
c.applyTransformations(node)

View File

@@ -47,7 +47,7 @@ func TestTransformRecursiveTemplate(t *testing.T) {
}
func newTestTemplate(templ tpl.Template) *templateState {
return newTemplateState(
return newTemplateState(nil,
templ,
templateInfo{
name: templ.Name(),