mirror of
https://github.com/gohugoio/hugo.git
synced 2025-08-27 22:09:53 +02:00
Create lightweight forks of text/template and html/template
This commit also removes support for Ace and Amber templates. Updates #6594
This commit is contained in:
Bjørn Erik Pedersen
556
tpl/internal/go_templates/texttemplate/parse/lex_test.go
Normal file
556
tpl/internal/go_templates/texttemplate/parse/lex_test.go
Normal file
@@ -0,0 +1,556 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build go1.13
|
||||
|
||||
package parse
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Make the types prettyprint.
|
||||
var itemName = map[itemType]string{
|
||||
itemError: "error",
|
||||
itemBool: "bool",
|
||||
itemChar: "char",
|
||||
itemCharConstant: "charconst",
|
||||
itemComplex: "complex",
|
||||
itemDeclare: ":=",
|
||||
itemEOF: "EOF",
|
||||
itemField: "field",
|
||||
itemIdentifier: "identifier",
|
||||
itemLeftDelim: "left delim",
|
||||
itemLeftParen: "(",
|
||||
itemNumber: "number",
|
||||
itemPipe: "pipe",
|
||||
itemRawString: "raw string",
|
||||
itemRightDelim: "right delim",
|
||||
itemRightParen: ")",
|
||||
itemSpace: "space",
|
||||
itemString: "string",
|
||||
itemVariable: "variable",
|
||||
|
||||
// keywords
|
||||
itemDot: ".",
|
||||
itemBlock: "block",
|
||||
itemDefine: "define",
|
||||
itemElse: "else",
|
||||
itemIf: "if",
|
||||
itemEnd: "end",
|
||||
itemNil: "nil",
|
||||
itemRange: "range",
|
||||
itemTemplate: "template",
|
||||
itemWith: "with",
|
||||
}
|
||||
|
||||
func (i itemType) String() string {
|
||||
s := itemName[i]
|
||||
if s == "" {
|
||||
return fmt.Sprintf("item%d", int(i))
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
type lexTest struct {
|
||||
name string
|
||||
input string
|
||||
items []item
|
||||
}
|
||||
|
||||
func mkItem(typ itemType, text string) item {
|
||||
return item{
|
||||
typ: typ,
|
||||
val: text,
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
tDot = mkItem(itemDot, ".")
|
||||
tBlock = mkItem(itemBlock, "block")
|
||||
tEOF = mkItem(itemEOF, "")
|
||||
tFor = mkItem(itemIdentifier, "for")
|
||||
tLeft = mkItem(itemLeftDelim, "{{")
|
||||
tLpar = mkItem(itemLeftParen, "(")
|
||||
tPipe = mkItem(itemPipe, "|")
|
||||
tQuote = mkItem(itemString, `"abc \n\t\" "`)
|
||||
tRange = mkItem(itemRange, "range")
|
||||
tRight = mkItem(itemRightDelim, "}}")
|
||||
tRpar = mkItem(itemRightParen, ")")
|
||||
tSpace = mkItem(itemSpace, " ")
|
||||
raw = "`" + `abc\n\t\" ` + "`"
|
||||
rawNL = "`now is{{\n}}the time`" // Contains newline inside raw quote.
|
||||
tRawQuote = mkItem(itemRawString, raw)
|
||||
tRawQuoteNL = mkItem(itemRawString, rawNL)
|
||||
)
|
||||
|
||||
var lexTests = []lexTest{
|
||||
{"empty", "", []item{tEOF}},
|
||||
{"spaces", " \t\n", []item{mkItem(itemText, " \t\n"), tEOF}},
|
||||
{"text", `now is the time`, []item{mkItem(itemText, "now is the time"), tEOF}},
|
||||
{"text with comment", "hello-{{/* this is a comment */}}-world", []item{
|
||||
mkItem(itemText, "hello-"),
|
||||
mkItem(itemText, "-world"),
|
||||
tEOF,
|
||||
}},
|
||||
{"punctuation", "{{,@% }}", []item{
|
||||
tLeft,
|
||||
mkItem(itemChar, ","),
|
||||
mkItem(itemChar, "@"),
|
||||
mkItem(itemChar, "%"),
|
||||
tSpace,
|
||||
tRight,
|
||||
tEOF,
|
||||
}},
|
||||
{"parens", "{{((3))}}", []item{
|
||||
tLeft,
|
||||
tLpar,
|
||||
tLpar,
|
||||
mkItem(itemNumber, "3"),
|
||||
tRpar,
|
||||
tRpar,
|
||||
tRight,
|
||||
tEOF,
|
||||
}},
|
||||
{"empty action", `{{}}`, []item{tLeft, tRight, tEOF}},
|
||||
{"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}},
|
||||
{"block", `{{block "foo" .}}`, []item{
|
||||
tLeft, tBlock, tSpace, mkItem(itemString, `"foo"`), tSpace, tDot, tRight, tEOF,
|
||||
}},
|
||||
{"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}},
|
||||
{"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}},
|
||||
{"raw quote with newline", "{{" + rawNL + "}}", []item{tLeft, tRawQuoteNL, tRight, tEOF}},
|
||||
{"numbers", "{{1 02 0x14 0X14 -7.2i 1e3 1E3 +1.2e-4 4.2i 1+2i 1_2 0x1.e_fp4 0X1.E_FP4}}", []item{
|
||||
tLeft,
|
||||
mkItem(itemNumber, "1"),
|
||||
tSpace,
|
||||
mkItem(itemNumber, "02"),
|
||||
tSpace,
|
||||
mkItem(itemNumber, "0x14"),
|
||||
tSpace,
|
||||
mkItem(itemNumber, "0X14"),
|
||||
tSpace,
|
||||
mkItem(itemNumber, "-7.2i"),
|
||||
tSpace,
|
||||
mkItem(itemNumber, "1e3"),
|
||||
tSpace,
|
||||
mkItem(itemNumber, "1E3"),
|
||||
tSpace,
|
||||
mkItem(itemNumber, "+1.2e-4"),
|
||||
tSpace,
|
||||
mkItem(itemNumber, "4.2i"),
|
||||
tSpace,
|
||||
mkItem(itemComplex, "1+2i"),
|
||||
tSpace,
|
||||
mkItem(itemNumber, "1_2"),
|
||||
tSpace,
|
||||
mkItem(itemNumber, "0x1.e_fp4"),
|
||||
tSpace,
|
||||
mkItem(itemNumber, "0X1.E_FP4"),
|
||||
tRight,
|
||||
tEOF,
|
||||
}},
|
||||
{"characters", `{{'a' '\n' '\'' '\\' '\u00FF' '\xFF' '本'}}`, []item{
|
||||
tLeft,
|
||||
mkItem(itemCharConstant, `'a'`),
|
||||
tSpace,
|
||||
mkItem(itemCharConstant, `'\n'`),
|
||||
tSpace,
|
||||
mkItem(itemCharConstant, `'\''`),
|
||||
tSpace,
|
||||
mkItem(itemCharConstant, `'\\'`),
|
||||
tSpace,
|
||||
mkItem(itemCharConstant, `'\u00FF'`),
|
||||
tSpace,
|
||||
mkItem(itemCharConstant, `'\xFF'`),
|
||||
tSpace,
|
||||
mkItem(itemCharConstant, `'本'`),
|
||||
tRight,
|
||||
tEOF,
|
||||
}},
|
||||
{"bools", "{{true false}}", []item{
|
||||
tLeft,
|
||||
mkItem(itemBool, "true"),
|
||||
tSpace,
|
||||
mkItem(itemBool, "false"),
|
||||
tRight,
|
||||
tEOF,
|
||||
}},
|
||||
{"dot", "{{.}}", []item{
|
||||
tLeft,
|
||||
tDot,
|
||||
tRight,
|
||||
tEOF,
|
||||
}},
|
||||
{"nil", "{{nil}}", []item{
|
||||
tLeft,
|
||||
mkItem(itemNil, "nil"),
|
||||
tRight,
|
||||
tEOF,
|
||||
}},
|
||||
{"dots", "{{.x . .2 .x.y.z}}", []item{
|
||||
tLeft,
|
||||
mkItem(itemField, ".x"),
|
||||
tSpace,
|
||||
tDot,
|
||||
tSpace,
|
||||
mkItem(itemNumber, ".2"),
|
||||
tSpace,
|
||||
mkItem(itemField, ".x"),
|
||||
mkItem(itemField, ".y"),
|
||||
mkItem(itemField, ".z"),
|
||||
tRight,
|
||||
tEOF,
|
||||
}},
|
||||
{"keywords", "{{range if else end with}}", []item{
|
||||
tLeft,
|
||||
mkItem(itemRange, "range"),
|
||||
tSpace,
|
||||
mkItem(itemIf, "if"),
|
||||
tSpace,
|
||||
mkItem(itemElse, "else"),
|
||||
tSpace,
|
||||
mkItem(itemEnd, "end"),
|
||||
tSpace,
|
||||
mkItem(itemWith, "with"),
|
||||
tRight,
|
||||
tEOF,
|
||||
}},
|
||||
{"variables", "{{$c := printf $ $hello $23 $ $var.Field .Method}}", []item{
|
||||
tLeft,
|
||||
mkItem(itemVariable, "$c"),
|
||||
tSpace,
|
||||
mkItem(itemDeclare, ":="),
|
||||
tSpace,
|
||||
mkItem(itemIdentifier, "printf"),
|
||||
tSpace,
|
||||
mkItem(itemVariable, "$"),
|
||||
tSpace,
|
||||
mkItem(itemVariable, "$hello"),
|
||||
tSpace,
|
||||
mkItem(itemVariable, "$23"),
|
||||
tSpace,
|
||||
mkItem(itemVariable, "$"),
|
||||
tSpace,
|
||||
mkItem(itemVariable, "$var"),
|
||||
mkItem(itemField, ".Field"),
|
||||
tSpace,
|
||||
mkItem(itemField, ".Method"),
|
||||
tRight,
|
||||
tEOF,
|
||||
}},
|
||||
{"variable invocation", "{{$x 23}}", []item{
|
||||
tLeft,
|
||||
mkItem(itemVariable, "$x"),
|
||||
tSpace,
|
||||
mkItem(itemNumber, "23"),
|
||||
tRight,
|
||||
tEOF,
|
||||
}},
|
||||
{"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{
|
||||
mkItem(itemText, "intro "),
|
||||
tLeft,
|
||||
mkItem(itemIdentifier, "echo"),
|
||||
tSpace,
|
||||
mkItem(itemIdentifier, "hi"),
|
||||
tSpace,
|
||||
mkItem(itemNumber, "1.2"),
|
||||
tSpace,
|
||||
tPipe,
|
||||
mkItem(itemIdentifier, "noargs"),
|
||||
tPipe,
|
||||
mkItem(itemIdentifier, "args"),
|
||||
tSpace,
|
||||
mkItem(itemNumber, "1"),
|
||||
tSpace,
|
||||
mkItem(itemString, `"hi"`),
|
||||
tRight,
|
||||
mkItem(itemText, " outro"),
|
||||
tEOF,
|
||||
}},
|
||||
{"declaration", "{{$v := 3}}", []item{
|
||||
tLeft,
|
||||
mkItem(itemVariable, "$v"),
|
||||
tSpace,
|
||||
mkItem(itemDeclare, ":="),
|
||||
tSpace,
|
||||
mkItem(itemNumber, "3"),
|
||||
tRight,
|
||||
tEOF,
|
||||
}},
|
||||
{"2 declarations", "{{$v , $w := 3}}", []item{
|
||||
tLeft,
|
||||
mkItem(itemVariable, "$v"),
|
||||
tSpace,
|
||||
mkItem(itemChar, ","),
|
||||
tSpace,
|
||||
mkItem(itemVariable, "$w"),
|
||||
tSpace,
|
||||
mkItem(itemDeclare, ":="),
|
||||
tSpace,
|
||||
mkItem(itemNumber, "3"),
|
||||
tRight,
|
||||
tEOF,
|
||||
}},
|
||||
{"field of parenthesized expression", "{{(.X).Y}}", []item{
|
||||
tLeft,
|
||||
tLpar,
|
||||
mkItem(itemField, ".X"),
|
||||
tRpar,
|
||||
mkItem(itemField, ".Y"),
|
||||
tRight,
|
||||
tEOF,
|
||||
}},
|
||||
{"trimming spaces before and after", "hello- {{- 3 -}} -world", []item{
|
||||
mkItem(itemText, "hello-"),
|
||||
tLeft,
|
||||
mkItem(itemNumber, "3"),
|
||||
tRight,
|
||||
mkItem(itemText, "-world"),
|
||||
tEOF,
|
||||
}},
|
||||
{"trimming spaces before and after comment", "hello- {{- /* hello */ -}} -world", []item{
|
||||
mkItem(itemText, "hello-"),
|
||||
mkItem(itemText, "-world"),
|
||||
tEOF,
|
||||
}},
|
||||
// errors
|
||||
{"badchar", "#{{\x01}}", []item{
|
||||
mkItem(itemText, "#"),
|
||||
tLeft,
|
||||
mkItem(itemError, "unrecognized character in action: U+0001"),
|
||||
}},
|
||||
{"unclosed action", "{{\n}}", []item{
|
||||
tLeft,
|
||||
mkItem(itemError, "unclosed action"),
|
||||
}},
|
||||
{"EOF in action", "{{range", []item{
|
||||
tLeft,
|
||||
tRange,
|
||||
mkItem(itemError, "unclosed action"),
|
||||
}},
|
||||
{"unclosed quote", "{{\"\n\"}}", []item{
|
||||
tLeft,
|
||||
mkItem(itemError, "unterminated quoted string"),
|
||||
}},
|
||||
{"unclosed raw quote", "{{`xx}}", []item{
|
||||
tLeft,
|
||||
mkItem(itemError, "unterminated raw quoted string"),
|
||||
}},
|
||||
{"unclosed char constant", "{{'\n}}", []item{
|
||||
tLeft,
|
||||
mkItem(itemError, "unterminated character constant"),
|
||||
}},
|
||||
{"bad number", "{{3k}}", []item{
|
||||
tLeft,
|
||||
mkItem(itemError, `bad number syntax: "3k"`),
|
||||
}},
|
||||
{"unclosed paren", "{{(3}}", []item{
|
||||
tLeft,
|
||||
tLpar,
|
||||
mkItem(itemNumber, "3"),
|
||||
mkItem(itemError, `unclosed left paren`),
|
||||
}},
|
||||
{"extra right paren", "{{3)}}", []item{
|
||||
tLeft,
|
||||
mkItem(itemNumber, "3"),
|
||||
tRpar,
|
||||
mkItem(itemError, `unexpected right paren U+0029 ')'`),
|
||||
}},
|
||||
|
||||
// Fixed bugs
|
||||
// Many elements in an action blew the lookahead until
|
||||
// we made lexInsideAction not loop.
|
||||
{"long pipeline deadlock", "{{|||||}}", []item{
|
||||
tLeft,
|
||||
tPipe,
|
||||
tPipe,
|
||||
tPipe,
|
||||
tPipe,
|
||||
tPipe,
|
||||
tRight,
|
||||
tEOF,
|
||||
}},
|
||||
{"text with bad comment", "hello-{{/*/}}-world", []item{
|
||||
mkItem(itemText, "hello-"),
|
||||
mkItem(itemError, `unclosed comment`),
|
||||
}},
|
||||
{"text with comment close separated from delim", "hello-{{/* */ }}-world", []item{
|
||||
mkItem(itemText, "hello-"),
|
||||
mkItem(itemError, `comment ends before closing delimiter`),
|
||||
}},
|
||||
// This one is an error that we can't catch because it breaks templates with
|
||||
// minimized JavaScript. Should have fixed it before Go 1.1.
|
||||
{"unmatched right delimiter", "hello-{.}}-world", []item{
|
||||
mkItem(itemText, "hello-{.}}-world"),
|
||||
tEOF,
|
||||
}},
|
||||
}
|
||||
|
||||
// collect gathers the emitted items into a slice.
|
||||
func collect(t *lexTest, left, right string) (items []item) {
|
||||
l := lex(t.name, t.input, left, right)
|
||||
for {
|
||||
item := l.nextItem()
|
||||
items = append(items, item)
|
||||
if item.typ == itemEOF || item.typ == itemError {
|
||||
break
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func equal(i1, i2 []item, checkPos bool) bool {
|
||||
if len(i1) != len(i2) {
|
||||
return false
|
||||
}
|
||||
for k := range i1 {
|
||||
if i1[k].typ != i2[k].typ {
|
||||
return false
|
||||
}
|
||||
if i1[k].val != i2[k].val {
|
||||
return false
|
||||
}
|
||||
if checkPos && i1[k].pos != i2[k].pos {
|
||||
return false
|
||||
}
|
||||
if checkPos && i1[k].line != i2[k].line {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func TestLex(t *testing.T) {
|
||||
for _, test := range lexTests {
|
||||
items := collect(&test, "", "")
|
||||
if !equal(items, test.items, false) {
|
||||
t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Some easy cases from above, but with delimiters $$ and @@
|
||||
var lexDelimTests = []lexTest{
|
||||
{"punctuation", "$$,@%{{}}@@", []item{
|
||||
tLeftDelim,
|
||||
mkItem(itemChar, ","),
|
||||
mkItem(itemChar, "@"),
|
||||
mkItem(itemChar, "%"),
|
||||
mkItem(itemChar, "{"),
|
||||
mkItem(itemChar, "{"),
|
||||
mkItem(itemChar, "}"),
|
||||
mkItem(itemChar, "}"),
|
||||
tRightDelim,
|
||||
tEOF,
|
||||
}},
|
||||
{"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}},
|
||||
{"for", `$$for@@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}},
|
||||
{"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}},
|
||||
{"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}},
|
||||
}
|
||||
|
||||
var (
|
||||
tLeftDelim = mkItem(itemLeftDelim, "$$")
|
||||
tRightDelim = mkItem(itemRightDelim, "@@")
|
||||
)
|
||||
|
||||
func TestDelims(t *testing.T) {
|
||||
for _, test := range lexDelimTests {
|
||||
items := collect(&test, "$$", "@@")
|
||||
if !equal(items, test.items, false) {
|
||||
t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var lexPosTests = []lexTest{
|
||||
{"empty", "", []item{{itemEOF, 0, "", 1}}},
|
||||
{"punctuation", "{{,@%#}}", []item{
|
||||
{itemLeftDelim, 0, "{{", 1},
|
||||
{itemChar, 2, ",", 1},
|
||||
{itemChar, 3, "@", 1},
|
||||
{itemChar, 4, "%", 1},
|
||||
{itemChar, 5, "#", 1},
|
||||
{itemRightDelim, 6, "}}", 1},
|
||||
{itemEOF, 8, "", 1},
|
||||
}},
|
||||
{"sample", "0123{{hello}}xyz", []item{
|
||||
{itemText, 0, "0123", 1},
|
||||
{itemLeftDelim, 4, "{{", 1},
|
||||
{itemIdentifier, 6, "hello", 1},
|
||||
{itemRightDelim, 11, "}}", 1},
|
||||
{itemText, 13, "xyz", 1},
|
||||
{itemEOF, 16, "", 1},
|
||||
}},
|
||||
{"trimafter", "{{x -}}\n{{y}}", []item{
|
||||
{itemLeftDelim, 0, "{{", 1},
|
||||
{itemIdentifier, 2, "x", 1},
|
||||
{itemRightDelim, 5, "}}", 1},
|
||||
{itemLeftDelim, 8, "{{", 2},
|
||||
{itemIdentifier, 10, "y", 2},
|
||||
{itemRightDelim, 11, "}}", 2},
|
||||
{itemEOF, 13, "", 2},
|
||||
}},
|
||||
{"trimbefore", "{{x}}\n{{- y}}", []item{
|
||||
{itemLeftDelim, 0, "{{", 1},
|
||||
{itemIdentifier, 2, "x", 1},
|
||||
{itemRightDelim, 3, "}}", 1},
|
||||
{itemLeftDelim, 6, "{{", 2},
|
||||
{itemIdentifier, 10, "y", 2},
|
||||
{itemRightDelim, 11, "}}", 2},
|
||||
{itemEOF, 13, "", 2},
|
||||
}},
|
||||
}
|
||||
|
||||
// The other tests don't check position, to make the test cases easier to construct.
|
||||
// This one does.
|
||||
func TestPos(t *testing.T) {
|
||||
for _, test := range lexPosTests {
|
||||
items := collect(&test, "", "")
|
||||
if !equal(items, test.items, true) {
|
||||
t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
|
||||
if len(items) == len(test.items) {
|
||||
// Detailed print; avoid item.String() to expose the position value.
|
||||
for i := range items {
|
||||
if !equal(items[i:i+1], test.items[i:i+1], true) {
|
||||
i1 := items[i]
|
||||
i2 := test.items[i]
|
||||
t.Errorf("\t#%d: got {%v %d %q %d} expected {%v %d %q %d}",
|
||||
i, i1.typ, i1.pos, i1.val, i1.line, i2.typ, i2.pos, i2.val, i2.line)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test that an error shuts down the lexing goroutine.
|
||||
func TestShutdown(t *testing.T) {
|
||||
// We need to duplicate template.Parse here to hold on to the lexer.
|
||||
const text = "erroneous{{define}}{{else}}1234"
|
||||
lexer := lex("foo", text, "{{", "}}")
|
||||
_, err := New("root").parseLexer(lexer)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error")
|
||||
}
|
||||
// The error should have drained the input. Therefore, the lexer should be shut down.
|
||||
token, ok := <-lexer.items
|
||||
if ok {
|
||||
t.Fatalf("input was not drained; got %v", token)
|
||||
}
|
||||
}
|
||||
|
||||
// parseLexer is a local version of parse that lets us pass in the lexer instead of building it.
|
||||
// We expect an error, so the tree set and funcs list are explicitly nil.
|
||||
func (t *Tree) parseLexer(lex *lexer) (tree *Tree, err error) {
|
||||
defer t.recover(&err)
|
||||
t.ParseName = t.Name
|
||||
t.startParse(nil, lex, map[string]*Tree{})
|
||||
t.parse()
|
||||
t.add()
|
||||
t.stopParse()
|
||||
return t, nil
|
||||
}
|
||||
Reference in New Issue
Block a user