mirror/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2025-08-28 22:19:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

tpl/tplimpl: Rework template management to get rid of concurrency issues

Browse Source 
This more or less completes the simplification of the template handling code in Hugo started in v0.62.

The main motivation was to fix a long lasting issue about a crash in HTML content files  without front matter.

But this commit also comes with a big functional improvement.

As we now have moved the base template evaluation to the build stage we now use the same lookup rules for `baseof` as for `list` etc. type of templates.

This means that in this simple example you can have a `baseof` template for the `blog` section without having to duplicate the others:

```
layouts
├── _default
│   ├── baseof.html
│   ├── list.html
│   └── single.html
└── blog
    └── baseof.html
```

Also, when simplifying code, you often get rid of some double work, as shown in the "site building" benchmarks below.

These benchmarks looks suspiciously good, but I have repeated the below with ca. the same result. Compared to master:

```
name                              old time/op    new time/op    delta
SiteNew/Bundle_with_image-16        13.1ms ± 1%    10.5ms ± 1%  -19.34%  (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16    13.0ms ± 0%    10.7ms ± 1%  -18.05%  (p=0.029 n=4+4)
SiteNew/Tags_and_categories-16      46.4ms ± 2%    43.1ms ± 1%   -7.15%  (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16            52.2ms ± 2%    47.8ms ± 1%   -8.30%  (p=0.029 n=4+4)
SiteNew/Deep_content_tree-16        77.9ms ± 1%    70.9ms ± 1%   -9.01%  (p=0.029 n=4+4)
SiteNew/Many_HTML_templates-16      43.0ms ± 0%    37.2ms ± 1%  -13.54%  (p=0.029 n=4+4)
SiteNew/Page_collections-16         58.2ms ± 1%    52.4ms ± 1%   -9.95%  (p=0.029 n=4+4)

name                              old alloc/op   new alloc/op   delta
SiteNew/Bundle_with_image-16        3.81MB ± 0%    2.22MB ± 0%  -41.70%  (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16    3.60MB ± 0%    2.01MB ± 0%  -44.20%  (p=0.029 n=4+4)
SiteNew/Tags_and_categories-16      19.3MB ± 1%    14.1MB ± 0%  -26.91%  (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16            70.7MB ± 0%    69.0MB ± 0%   -2.40%  (p=0.029 n=4+4)
SiteNew/Deep_content_tree-16        37.1MB ± 0%    31.2MB ± 0%  -15.94%  (p=0.029 n=4+4)
SiteNew/Many_HTML_templates-16      17.6MB ± 0%    10.6MB ± 0%  -39.92%  (p=0.029 n=4+4)
SiteNew/Page_collections-16         25.9MB ± 0%    21.2MB ± 0%  -17.99%  (p=0.029 n=4+4)

name                              old allocs/op  new allocs/op  delta
SiteNew/Bundle_with_image-16         52.3k ± 0%     26.1k ± 0%  -50.18%  (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16     52.3k ± 0%     26.1k ± 0%  -50.16%  (p=0.029 n=4+4)
SiteNew/Tags_and_categories-16        336k ± 1%      269k ± 0%  -19.90%  (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16              422k ± 0%      395k ± 0%   -6.43%  (p=0.029 n=4+4)
SiteNew/Deep_content_tree-16          401k ± 0%      313k ± 0%  -21.79%  (p=0.029 n=4+4)
SiteNew/Many_HTML_templates-16        247k ± 0%      143k ± 0%  -42.17%  (p=0.029 n=4+4)
SiteNew/Page_collections-16           282k ± 0%      207k ± 0%  -26.55%  (p=0.029 n=4+4)
```

Fixes #6716
Fixes #6760
Fixes #6768
Fixes #6778
This commit is contained in:
Bjørn Erik Pedersen
2020-01-15 15:59:56 +01:00
parent 8585b388d2
commit c6d650c8c8
46 changed files with 1332 additions and 1446 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tpl/collections/apply.go
View File

@@ -106,7 +106,7 @@ func applyFnToThis(fn, this reflect.Value, args ...interface{}) (reflect.Value,
func (ns *Namespace) lookupFunc(fname string) (reflect.Value, bool) {
if !strings.ContainsRune(fname, '.') {
templ := ns.deps.Tmpl.(tpl.TemplateFuncGetter)
templ := ns.deps.Tmpl().(tpl.TemplateFuncGetter)
return templ.GetFunc(fname)
}

14
tpl/collections/apply_test.go
View File

@@ -22,6 +22,7 @@ import (
qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/output"
"github.com/gohugoio/hugo/tpl"
)
@@ -31,10 +32,18 @@ func (templateFinder) Lookup(name string) (tpl.Template, bool) {
return nil, false
}
func (templateFinder) HasTemplate(name string) bool {
return false
}
func (templateFinder) LookupVariant(name string, variants tpl.TemplateVariants) (tpl.Template, bool, bool) {
return nil, false, false
}
func (templateFinder) LookupLayout(d output.LayoutDescriptor, f output.Format) (tpl.Template, bool, error) {
return nil, false, nil
}
func (templateFinder) Execute(t tpl.Template, wr io.Writer, data interface{}) error {
return nil
}
@@ -51,8 +60,9 @@ func (templateFinder) GetFunc(name string) (reflect.Value, bool) {
func TestApply(t *testing.T) {
t.Parallel()
c := qt.New(t)
ns := New(&deps.Deps{Tmpl: new(templateFinder)})
d := &deps.Deps{}
d.SetTmpl(new(templateFinder))
ns := New(d)
strings := []interface{}{"a\n", "b\n"}

6
tpl/partials/partials.go
View File

@@ -105,11 +105,11 @@ func (ns *Namespace) Include(name string, contextList ...interface{}) (interface
}
n := "partials/" + name
templ, found := ns.deps.Tmpl.Lookup(n)
templ, found := ns.deps.Tmpl().Lookup(n)
if !found {
// For legacy reasons.
templ, found = ns.deps.Tmpl.Lookup(n + ".html")
templ, found = ns.deps.Tmpl().Lookup(n + ".html")
}
if !found {
@@ -139,7 +139,7 @@ func (ns *Namespace) Include(name string, contextList ...interface{}) (interface
w = b
}
if err := ns.deps.Tmpl.Execute(templ, w, context); err != nil {
if err := ns.deps.Tmpl().Execute(templ, w, context); err != nil {
return "", err
}

3
tpl/resources/resources.go
View File

@@ -45,6 +45,7 @@ func New(deps *deps.Deps) (*Namespace, error) {
if err != nil {
return nil, err
}
return &Namespace{
deps: deps,
scssClient: scssClient,
@@ -53,7 +54,7 @@ func New(deps *deps.Deps) (*Namespace, error) {
integrityClient: integrity.New(deps.ResourceSpec),
minifyClient: minifier.New(deps.ResourceSpec),
postcssClient: postcss.New(deps.ResourceSpec),
templatesClient: templates.New(deps.ResourceSpec, deps.Tmpl, deps.TextTmpl),
templatesClient: templates.New(deps.ResourceSpec, deps),
}, nil
}

12
tpl/template.go
View File

@@ -30,9 +30,7 @@ type TemplateManager interface {
TemplateFuncGetter
AddTemplate(name, tpl string) error
AddLateTemplate(name, tpl string) error
LoadTemplates(prefix string) error
RebuildClone()
MarkReady() error
}
// TemplateVariants describes the possible variants of a template.
@@ -52,6 +50,8 @@ type TemplateFinder interface {
type TemplateHandler interface {
TemplateFinder
Execute(t Template, wr io.Writer, data interface{}) error
LookupLayout(d output.LayoutDescriptor, f output.Format) (Template, bool, error)
HasTemplate(name string) bool
}
type TemplateLookup interface {
@@ -105,6 +105,12 @@ type templateInfoManager struct {
InfoManager
}
// TemplatesProvider as implemented by deps.Deps.
type TemplatesProvider interface {
Tmpl() TemplateHandler
TextTmpl() TemplateParseFinder
}
// WithInfo wraps the info in a template.
func WithInfo(templ Template, info Info) Template {
if manager, ok := info.(InfoManager); ok {

2
tpl/templates/templates.go
View File

@@ -34,7 +34,7 @@ type Namespace struct {
// Note that this is the Unix-styled relative path including filename suffix,
// e.g. partials/header.html
func (ns *Namespace) Exists(name string) bool {
_, found := ns.deps.Tmpl.Lookup(name)
_, found := ns.deps.Tmpl().Lookup(name)
return found
}

3
tpl/tplimpl/shortcodes.go
View File

@@ -32,8 +32,7 @@ type shortcodeVariant struct {
// A slice of length numTemplateVariants.
variants []string
info tpl.Info
templ tpl.Template
ts *templateState
}
type shortcodeTemplates struct {

1305
tpl/tplimpl/template.go
View File

File diff suppressed because it is too large Load Diff

27
tpl/tplimpl/templateProvider.go
View File

@@ -25,32 +25,17 @@ var DefaultTemplateProvider *TemplateProvider
// Update updates the Hugo Template System in the provided Deps
// with all the additional features, templates & functions.
func (*TemplateProvider) Update(deps *deps.Deps) error {
newTmpl := newTemplateAdapter(deps)
deps.Tmpl = newTmpl
deps.TextTmpl = newTmpl.wrapTextTemplate(newTmpl.text.standalone)
// These needs to be there at parse time.
newTmpl.initTemplateExecuter()
if err := newTmpl.loadEmbedded(); err != nil {
func (*TemplateProvider) Update(d *deps.Deps) error {
tmpl, err := newTemplateExec(d)
if err != nil {
return err
}
if deps.WithTemplate != nil {
err := deps.WithTemplate(newTmpl)
if err != nil {
return err
}
}
return newTmpl.markReady()
return tmpl.postTransform()
}
// Clone clones.
func (*TemplateProvider) Clone(d *deps.Deps) error {
t := d.Tmpl.(*templateHandler)
t.clone(d)
t := d.Tmpl().(*templateExec)
d.SetTmpl(t.Clone(d))
return nil
}

130
tpl/tplimpl/template_ast_transformers.go
View File

@@ -17,10 +17,9 @@ import (
"regexp"
"strings"
"github.com/gohugoio/hugo/identity"
template "github.com/gohugoio/hugo/tpl/internal/go_templates/htmltemplate"
htmltemplate "github.com/gohugoio/hugo/tpl/internal/go_templates/htmltemplate"
texttemplate "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate"
"github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse"
"github.com/gohugoio/hugo/common/maps"
@@ -41,25 +40,21 @@ type templateContext struct {
visited map[string]bool
templateNotFound map[string]bool
identityNotFound map[string]bool
lookupFn func(name string) *templateInfoTree
lookupFn func(name string) *templateState
// The last error encountered.
err error
typ templateType
// Set when we're done checking for config header.
configChecked bool
// Contains some info about the template
parseInfo *tpl.ParseInfo
id identity.Manager
t *templateState
// Store away the return node in partials.
returnNode *parse.CommandNode
}
func (c templateContext) getIfNotVisited(name string) *templateInfoTree {
func (c templateContext) getIfNotVisited(name string) *templateState {
if c.visited[name] {
return nil
}
@@ -76,13 +71,11 @@ func (c templateContext) getIfNotVisited(name string) *templateInfoTree {
}
func newTemplateContext(
id identity.Manager,
info *tpl.ParseInfo,
lookupFn func(name string) *templateInfoTree) *templateContext {
t *templateState,
lookupFn func(name string) *templateState) *templateContext {
return &templateContext{
id: id,
parseInfo: info,
t: t,
lookupFn: lookupFn,
visited: make(map[string]bool),
templateNotFound: make(map[string]bool),
@@ -90,79 +83,36 @@ func newTemplateContext(
}
}
func createGetTemplateInfoTreeFor(getID func(name string) *templateInfoTree) func(nn string) *templateInfoTree {
return func(nn string) *templateInfoTree {
return getID(nn)
}
}
func (t *templateHandler) applyTemplateTransformersToHMLTTemplate(typ templateType, templ *template.Template) (*templateContext, error) {
id, info := t.createTemplateInfo(templ.Name())
ti := &templateInfoTree{
tree: templ.Tree,
templ: templ,
typ: typ,
id: id,
info: info,
}
t.templateInfoTree[templ.Name()] = ti
getTemplateInfoTree := createGetTemplateInfoTreeFor(func(name string) *templateInfoTree {
return t.templateInfoTree[name]
})
return applyTemplateTransformers(typ, ti, getTemplateInfoTree)
}
func (t *templateHandler) applyTemplateTransformersToTextTemplate(typ templateType, templ *texttemplate.Template) (*templateContext, error) {
id, info := t.createTemplateInfo(templ.Name())
ti := &templateInfoTree{
tree: templ.Tree,
templ: templ,
typ: typ,
id: id,
info: info,
}
t.templateInfoTree[templ.Name()] = ti
getTemplateInfoTree := createGetTemplateInfoTreeFor(func(name string) *templateInfoTree {
return t.templateInfoTree[name]
})
return applyTemplateTransformers(typ, ti, getTemplateInfoTree)
}
type templateInfoTree struct {
info tpl.ParseInfo
typ templateType
id identity.Manager
templ tpl.Template
tree *parse.Tree
}
func applyTemplateTransformers(
typ templateType,
templ *templateInfoTree,
lookupFn func(name string) *templateInfoTree) (*templateContext, error) {
t *templateState,
lookupFn func(name string) *templateState) (*templateContext, error) {
if templ == nil {
if t == nil {
return nil, errors.New("expected template, but none provided")
}
c := newTemplateContext(templ.id, &templ.info, lookupFn)
c.typ = typ
c := newTemplateContext(t, lookupFn)
tree := getParseTree(t.Template)
_, err := c.applyTransformations(templ.tree.Root)
_, err := c.applyTransformations(tree.Root)
if err == nil && c.returnNode != nil {
// This is a partial with a return statement.
c.parseInfo.HasReturn = true
templ.tree.Root = c.wrapInPartialReturnWrapper(templ.tree.Root)
c.t.parseInfo.HasReturn = true
tree.Root = c.wrapInPartialReturnWrapper(tree.Root)
}
return c, err
}
func getParseTree(templ tpl.Template) *parse.Tree {
templ = unwrap(templ)
if text, ok := templ.(*texttemplate.Template); ok {
return text.Tree
}
return templ.(*htmltemplate.Template).Tree
}
const (
partialReturnWrapperTempl = `{{ $_hugo_dot := $ }}{{ $ := .Arg }}{{ with .Arg }}{{ $_hugo_dot.Set ("PLACEHOLDER") }}{{ end }}`
)
@@ -215,7 +165,7 @@ func (c *templateContext) applyTransformations(n parse.Node) (bool, error) {
case *parse.TemplateNode:
subTempl := c.getIfNotVisited(x.Name)
if subTempl != nil {
c.applyTransformationsToNodes(subTempl.tree.Root)
c.applyTransformationsToNodes(getParseTree(subTempl.Template).Root)
}
case *parse.PipeNode:
c.collectConfig(x)
@@ -263,7 +213,7 @@ func (c *templateContext) hasIdent(idents []string, ident string) bool {
// on the form:
// {{ $_hugo_config:= `{ "version": 1 }` }}
func (c *templateContext) collectConfig(n *parse.PipeNode) {
if c.typ != templateShortcode {
if c.t.typ != templateShortcode {
return
}
if c.configChecked {
@@ -295,7 +245,7 @@ func (c *templateContext) collectConfig(n *parse.PipeNode) {
c.err = errors.Wrap(err, errMsg)
return
}
if err := mapstructure.WeakDecode(m, &c.parseInfo.Config); err != nil {
if err := mapstructure.WeakDecode(m, &c.t.parseInfo.Config); err != nil {
c.err = errors.Wrap(err, errMsg)
}
}
@@ -304,10 +254,10 @@ func (c *templateContext) collectConfig(n *parse.PipeNode) {
// collectInner determines if the given CommandNode represents a
// shortcode call to its .Inner.
func (c *templateContext) collectInner(n *parse.CommandNode) {
if c.typ != templateShortcode {
if c.t.typ != templateShortcode {
return
}
if c.parseInfo.IsInner || len(n.Args) == 0 {
if c.t.parseInfo.IsInner || len(n.Args) == 0 {
return
}
@@ -321,7 +271,7 @@ func (c *templateContext) collectInner(n *parse.CommandNode) {
}
if c.hasIdent(idents, "Inner") {
c.parseInfo.IsInner = true
c.t.parseInfo.IsInner = true
break
}
}
@@ -351,8 +301,9 @@ func (c *templateContext) collectPartialInfo(x *parse.CommandNode) {
}
partialName = "partials/" + partialName
info := c.lookupFn(partialName)
if info != nil {
c.id.Add(info.id)
c.t.Add(info)
} else {
// Delay for later
c.identityNotFound[partialName] = true
@@ -361,7 +312,7 @@ func (c *templateContext) collectPartialInfo(x *parse.CommandNode) {
}
func (c *templateContext) collectReturnNode(n *parse.CommandNode) bool {
if c.typ != templatePartial || c.returnNode != nil {
if c.t.typ != templatePartial || c.returnNode != nil {
return true
}
@@ -381,3 +332,18 @@ func (c *templateContext) collectReturnNode(n *parse.CommandNode) bool {
return false
}
func findTemplateIn(name string, in tpl.Template) (tpl.Template, bool) {
in = unwrap(in)
if text, ok := in.(*texttemplate.Template); ok {
if templ := text.Lookup(name); templ != nil {
return templ, true
}
return nil, false
}
if templ := in.(*htmltemplate.Template).Lookup(name); templ != nil {
return templ, true
}
return nil, false
}

82
tpl/tplimpl/template_ast_transformers_test.go
View File

@@ -13,15 +13,11 @@
package tplimpl
import (
"github.com/gohugoio/hugo/hugofs/files"
"testing"
template "github.com/gohugoio/hugo/tpl/internal/go_templates/htmltemplate"
"github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse"
qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/tpl"
)
@@ -33,7 +29,7 @@ func TestTransformRecursiveTemplate(t *testing.T) {
{{ define "menu-nodes" }}
{{ template "menu-node" }}
{{ end }}
{{ define "menu-nßode" }}
{{ define "menu-node" }}
{{ template "menu-node" }}
{{ end }}
{{ template "menu-nodes" }}
@@ -41,36 +37,46 @@ func TestTransformRecursiveTemplate(t *testing.T) {
templ, err := template.New("foo").Parse(recursive)
c.Assert(err, qt.IsNil)
parseInfo := tpl.DefaultParseInfo
ts := newTestTemplate(templ)
ctx := newTemplateContext(
newTemplateInfo("test").(identity.Manager),
&parseInfo,
createGetTemplateInfoTree(templ.Tree),
ts,
newTestTemplateLookup(ts),
)
ctx.applyTransformations(templ.Tree.Root)
}
func createGetTemplateInfoTree(tree *parse.Tree) func(name string) *templateInfoTree {
return func(name string) *templateInfoTree {
return &templateInfoTree{
tree: tree,
func newTestTemplate(templ tpl.Template) *templateState {
return newTemplateState(
templ,
templateInfo{
name: templ.Name(),
},
)
}
func newTestTemplateLookup(in *templateState) func(name string) *templateState {
m := make(map[string]*templateState)
return func(name string) *templateState {
if in.Name() == name {
return in
}
if ts, found := m[name]; found {
return ts
}
if templ, found := findTemplateIn(name, in); found {
ts := newTestTemplate(templ)
m[name] = ts
return ts
}
return nil
}
}
type I interface {
Method0()
}
type T struct {
NonEmptyInterfaceTypedNil I
}
func (T) Method0() {
}
func TestCollectInfo(t *testing.T) {
configStr := `{ "version": 42 }`
@@ -98,13 +104,14 @@ func TestCollectInfo(t *testing.T) {
templ, err := template.New("foo").Funcs(funcs).Parse(test.tplString)
c.Assert(err, qt.IsNil)
parseInfo := tpl.DefaultParseInfo
ts := newTestTemplate(templ)
ts.typ = templateShortcode
ctx := newTemplateContext(
newTemplateInfo("test").(identity.Manager), &parseInfo, createGetTemplateInfoTree(templ.Tree))
ctx.typ = templateShortcode
ts,
newTestTemplateLookup(ts),
)
ctx.applyTransformations(templ.Tree.Root)
c.Assert(ctx.parseInfo, qt.DeepEquals, &test.expected)
c.Assert(ctx.t.parseInfo, qt.DeepEquals, test.expected)
})
}
@@ -141,11 +148,13 @@ func TestPartialReturn(t *testing.T) {
templ, err := template.New("foo").Funcs(funcs).Parse(test.tplString)
c.Assert(err, qt.IsNil)
ts := newTestTemplate(templ)
ctx := newTemplateContext(
ts,
newTestTemplateLookup(ts),
)
_, err = applyTemplateTransformers(
templatePartial,
&templateInfoTree{tree: templ.Tree, info: tpl.DefaultParseInfo},
createGetTemplateInfoTree(templ.Tree))
_, err = ctx.applyTransformations(templ.Tree.Root)
// Just check that it doesn't fail in this test. We have functional tests
// in hugoblib.
@@ -155,10 +164,3 @@ func TestPartialReturn(t *testing.T) {
}
}
func newTemplateInfo(name string) tpl.Info {
return tpl.NewInfo(
identity.NewManager(identity.NewPathIdentity(files.ComponentFolderLayouts, name)),
tpl.DefaultParseInfo,
)
}

6
tpl/tplimpl/template_errors.go
View File

@@ -20,7 +20,9 @@ import (
)
type templateInfo struct {
name string
template string
isText bool // HTML or plain text template.
// Used to create some error context in error situations
fs afero.Fs
@@ -32,6 +34,10 @@ type templateInfo struct {
realFilename string
}
func (t templateInfo) resolveType() templateType {
return resolveTemplateType(t.name)
}
func (info templateInfo) errWithFileContext(what string, err error) error {
err = errors.Wrapf(err, what)

4
tpl/tplimpl/template_funcs_test.go
View File

@@ -127,8 +127,8 @@ func TestTemplateFuncsExamples(t *testing.T) {
c.Assert(d.LoadResources(), qt.IsNil)
var b bytes.Buffer
templ, _ := d.Tmpl.Lookup("test")
c.Assert(d.Tmpl.Execute(templ, &b, &data), qt.IsNil)
templ, _ := d.Tmpl().Lookup("test")
c.Assert(d.Tmpl().Execute(templ, &b, &data), qt.IsNil)
if b.String() != expected {
t.Fatalf("%s[%d]: got %q expected %q", ns.Name, i, b.String(), expected)
}

7
tpl/tplimpl/template_info_test.go
View File

@@ -24,14 +24,15 @@ import (
func TestTemplateInfoShortcode(t *testing.T) {
c := qt.New(t)
d := newD(c)
h := d.Tmpl.(*templateHandler)
h := d.Tmpl().(*templateExec)
c.Assert(h.AddTemplate("shortcodes/mytemplate.html", `
{{ .Inner }}
`), qt.IsNil)
c.Assert(h.markReady(), qt.IsNil)
tt, found, _ := d.Tmpl.LookupVariant("mytemplate", tpl.TemplateVariants{})
c.Assert(h.postTransform(), qt.IsNil)
tt, found, _ := d.Tmpl().LookupVariant("mytemplate", tpl.TemplateVariants{})
c.Assert(found, qt.Equals, true)
tti, ok := tt.(tpl.Info)