mirror/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2025-08-30 22:39:58 +02:00
Code Issues Packages Projects Releases Wiki Activity

parser/pageparser: Don't store the byte slices

Browse Source 
On its own this change doesn't do any magic, but this is part of a bigger picture about making Hugo leaner in the
memory usage department.
This commit is contained in:
Bjørn Erik Pedersen
2022-07-07 16:11:47 +02:00
parent 72b0ccdb01
commit 223bf28004
13 changed files with 385 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
hugolib/page.go
View File

@@ -639,7 +639,7 @@ func (p *pageState) mapContentForResult(
if fe, ok := err.(herrors.FileError); ok {
return fe
}
return p.parseError(err, iter.Input(), i.Pos)
return p.parseError(err, result.Input(), i.Pos())
}
// the parser is guaranteed to return items in proper order or fail, so …
@@ -656,14 +656,14 @@ Loop:
case it.Type == pageparser.TypeIgnore:
case it.IsFrontMatter():
f := pageparser.FormatFromFrontMatterType(it.Type)
m, err := metadecoders.Default.UnmarshalToMap(it.Val, f)
m, err := metadecoders.Default.UnmarshalToMap(it.Val(result.Input()), f)
if err != nil {
if fe, ok := err.(herrors.FileError); ok {
pos := fe.Position()
// Apply the error to the content file.
pos.Filename = p.File().Filename()
// Offset the starting position of front matter.
offset := iter.LineNumber() - 1
offset := iter.LineNumber(result.Input()) - 1
if f == metadecoders.YAML {
offset -= 1
}
@@ -687,7 +687,7 @@ Loop:
next := iter.Peek()
if !next.IsDone() {
p.source.posMainContent = next.Pos
p.source.posMainContent = next.Pos()
}
if !p.s.shouldBuild(p) {
@@ -699,10 +699,10 @@ Loop:
posBody := -1
f := func(item pageparser.Item) bool {
if posBody == -1 && !item.IsDone() {
posBody = item.Pos
posBody = item.Pos()
}
if item.IsNonWhitespace() {
if item.IsNonWhitespace(result.Input()) {
p.truncated = true
// Done
@@ -712,7 +712,7 @@ Loop:
}
iter.PeekWalk(f)
p.source.posSummaryEnd = it.Pos
p.source.posSummaryEnd = it.Pos()
p.source.posBodyStart = posBody
p.source.hasSummaryDivider = true
@@ -727,13 +727,13 @@ Loop:
// let extractShortcode handle left delim (will do so recursively)
iter.Backup()
currShortcode, err := s.extractShortcode(ordinal, 0, iter)
currShortcode, err := s.extractShortcode(ordinal, 0, result.Input(), iter)
if err != nil {
return fail(err, it)
}
currShortcode.pos = it.Pos
currShortcode.length = iter.Current().Pos - it.Pos
currShortcode.pos = it.Pos()
currShortcode.length = iter.Current().Pos() - it.Pos()
if currShortcode.placeholder == "" {
currShortcode.placeholder = createShortcodePlaceholder("s", currShortcode.ordinal)
}
@@ -754,7 +754,7 @@ Loop:
rn.AddShortcode(currShortcode)
case it.Type == pageparser.TypeEmoji:
if emoji := helpers.Emoji(it.ValStr()); emoji != nil {
if emoji := helpers.Emoji(it.ValStr(result.Input())); emoji != nil {
rn.AddReplacement(emoji, it)
} else {
rn.AddBytes(it)
@@ -762,7 +762,7 @@ Loop:
case it.IsEOF():
break Loop
case it.IsError():
err := fail(errors.New(it.ValStr()), it)
err := fail(errors.New(it.ValStr(result.Input())), it)
currShortcode.err = err
return err

2
hugolib/page__content.go
View File

@@ -45,7 +45,7 @@ func (p pageContent) contentToRender(parsed pageparser.Result, pm *pageContentMa
for _, it := range pm.items {
switch v := it.(type) {
case pageparser.Item:
c = append(c, source[v.Pos:v.Pos+len(v.Val)]...)
c = append(c, source[v.Pos():v.Pos()+len(v.Val(source))]...)
case pageContentReplacement:
c = append(c, v.val...)
case *shortcode:

26
hugolib/shortcode.go
View File

@@ -509,7 +509,7 @@ func (s *shortcodeHandler) parseError(err error, input []byte, pos int) error {
// pageTokens state:
// - before: positioned just before the shortcode start
// - after: shortcode(s) consumed (plural when they are nested)
func (s *shortcodeHandler) extractShortcode(ordinal, level int, pt *pageparser.Iterator) (*shortcode, error) {
func (s *shortcodeHandler) extractShortcode(ordinal, level int, source []byte, pt *pageparser.Iterator) (*shortcode, error) {
if s == nil {
panic("handler nil")
}
@@ -520,7 +520,7 @@ func (s *shortcodeHandler) extractShortcode(ordinal, level int, pt *pageparser.I
pt.Backup()
item := pt.Next()
if item.IsIndentation() {
sc.indentation = string(item.Val)
sc.indentation = item.ValStr(source)
}
}
@@ -530,7 +530,7 @@ func (s *shortcodeHandler) extractShortcode(ordinal, level int, pt *pageparser.I
const errorPrefix = "failed to extract shortcode"
fail := func(err error, i pageparser.Item) error {
return s.parseError(fmt.Errorf("%s: %w", errorPrefix, err), pt.Input(), i.Pos)
return s.parseError(fmt.Errorf("%s: %w", errorPrefix, err), source, i.Pos())
}
Loop:
@@ -550,7 +550,7 @@ Loop:
if cnt > 0 {
// nested shortcode; append it to inner content
pt.Backup()
nested, err := s.extractShortcode(nestedOrdinal, nextLevel, pt)
nested, err := s.extractShortcode(nestedOrdinal, nextLevel, source, pt)
nestedOrdinal++
if nested != nil && nested.name != "" {
s.addName(nested.name)
@@ -589,7 +589,7 @@ Loop:
// return that error, more specific
continue
}
return sc, fail(fmt.Errorf("shortcode %q has no .Inner, yet a closing tag was provided", next.Val), next)
return sc, fail(fmt.Errorf("shortcode %q has no .Inner, yet a closing tag was provided", next.ValStr(source)), next)
}
}
if next.IsRightShortcodeDelim() {
@@ -602,11 +602,11 @@ Loop:
return sc, nil
case currItem.IsText():
sc.inner = append(sc.inner, currItem.ValStr())
sc.inner = append(sc.inner, currItem.ValStr(source))
case currItem.Type == pageparser.TypeEmoji:
// TODO(bep) avoid the duplication of these "text cases", to prevent
// more of #6504 in the future.
val := currItem.ValStr()
val := currItem.ValStr(source)
if emoji := helpers.Emoji(val); emoji != nil {
sc.inner = append(sc.inner, string(emoji))
} else {
@@ -614,7 +614,7 @@ Loop:
}
case currItem.IsShortcodeName():
sc.name = currItem.ValStr()
sc.name = currItem.ValStr(source)
// Used to check if the template expects inner content.
templs := s.s.Tmpl().LookupVariants(sc.name)
@@ -625,7 +625,7 @@ Loop:
sc.info = templs[0].(tpl.Info)
sc.templs = templs
case currItem.IsInlineShortcodeName():
sc.name = currItem.ValStr()
sc.name = currItem.ValStr(source)
sc.isInline = true
case currItem.IsShortcodeParam():
if !pt.IsValueNext() {
@@ -634,11 +634,11 @@ Loop:
// named params
if sc.params == nil {
params := make(map[string]any)
params[currItem.ValStr()] = pt.Next().ValTyped()
params[currItem.ValStr(source)] = pt.Next().ValTyped(source)
sc.params = params
} else {
if params, ok := sc.params.(map[string]any); ok {
params[currItem.ValStr()] = pt.Next().ValTyped()
params[currItem.ValStr(source)] = pt.Next().ValTyped(source)
} else {
return sc, errShortCodeIllegalState
}
@@ -647,11 +647,11 @@ Loop:
// positional params
if sc.params == nil {
var params []any
params = append(params, currItem.ValTyped())
params = append(params, currItem.ValTyped(source))
sc.params = params
} else {
if params, ok := sc.params.([]any); ok {
params = append(params, currItem.ValTyped())
params = append(params, currItem.ValTyped(source))
sc.params = params
} else {
return sc, errShortCodeIllegalState

6
hugolib/shortcode_test.go
View File

@@ -112,7 +112,7 @@ title: "Shortcodes Galore!"
handler := newShortcodeHandler(nil, s)
iter := p.Iterator()
short, err := handler.extractShortcode(0, 0, iter)
short, err := handler.extractShortcode(0, 0, p.Input(), iter)
test.check(c, short, err)
})
@@ -763,7 +763,7 @@ title: "Hugo Rocks!"
)
}
func TestShortcodeTypedParams(t *testing.T) {
func TestShortcodeParams(t *testing.T) {
t.Parallel()
c := qt.New(t)
@@ -778,6 +778,7 @@ title: "Hugo Rocks!"
types positional: {{< hello true false 33 3.14 >}}
types named: {{< hello b1=true b2=false i1=33 f1=3.14 >}}
types string: {{< hello "true" trues "33" "3.14" >}}
escaped quoute: {{< hello "hello \"world\"." >}}
`).WithTemplatesAdded(
@@ -796,6 +797,7 @@ Get: {{ printf "%v (%T)" $b1 $b1 | safeHTML }}
"types positional: - 0: true (bool) - 1: false (bool) - 2: 33 (int) - 3: 3.14 (float64)",
"types named: - b1: true (bool) - b2: false (bool) - f1: 3.14 (float64) - i1: 33 (int) Get: true (bool) ",
"types string: - 0: true (string) - 1: trues (string) - 2: 33 (string) - 3: 3.14 (string) ",
"hello &#34;world&#34;. (string)",
)
}