tpl/internal: Synch Go templates fork with Go 1.16dev

tpl/internal/go_templates/texttemplate/doc.go

@@ -40,16 +40,17 @@ More intricate examples appear below.
 Text and spaces
 
 By default, all text between actions is copied verbatim when the template is
-executed. For example, the string " items are made of " in the example above appears
-on standard output when the program is run.
+executed. For example, the string " items are made of " in the example above
+appears on standard output when the program is run.
 
-However, to aid in formatting template source code, if an action's left delimiter
-(by default "{{") is followed immediately by a minus sign and ASCII space character
-("{{- "), all trailing white space is trimmed from the immediately preceding text.
-Similarly, if the right delimiter ("}}") is preceded by a space and minus sign
-(" -}}"), all leading white space is trimmed from the immediately following text.
-In these trim markers, the ASCII space must be present; "{{-3}}" parses as an
-action containing the number -3.
+However, to aid in formatting template source code, if an action's left
+delimiter (by default "{{") is followed immediately by a minus sign and white
+space, all trailing white space is trimmed from the immediately preceding text.
+Similarly, if the right delimiter ("}}") is preceded by white space and a minus
+sign, all leading white space is trimmed from the immediately following text.
+In these trim markers, the white space must be present:
+"{{- 3}}" is like "{{3}}" but trims the immediately preceding text, while
+"{{-3}}" parses as an action containing the number -3.
 
 For instance, when executing the template whose source is
 

tpl/internal/go_templates/texttemplate/exec.go

@@ -256,6 +256,7 @@ func (s *state) walk(dot reflect.Value, node parse.Node) {
 		if len(node.Pipe.Decl) == 0 {
 			s.printValue(node, val)
 		}
+	case *parse.CommentNode:
 	case *parse.IfNode:
 		s.walkIfOrWith(parse.NodeIf, dot, node.Pipe, node.List, node.ElseList)
 	case *parse.ListNode:

tpl/internal/go_templates/texttemplate/exec_test.go

@@ -11,7 +11,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"reflect"
 	"strings"
 	"testing"
@@ -1297,7 +1297,7 @@ func TestUnterminatedStringError(t *testing.T) {
 		t.Fatal("expected error")
 	}
 	str := err.Error()
-	if !strings.Contains(str, "X:3: unexpected unterminated raw quoted string") {
+	if !strings.Contains(str, "X:3: unterminated raw quoted string") {
 		t.Fatalf("unexpected error: %s", str)
 	}
 }
@@ -1330,7 +1330,7 @@ func TestExecuteGivesExecError(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = tmpl.Execute(ioutil.Discard, 0)
+	err = tmpl.Execute(io.Discard, 0)
 	if err == nil {
 		t.Fatal("expected error; got none")
 	}
@@ -1476,7 +1476,7 @@ func TestEvalFieldErrors(t *testing.T) {
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
 			tmpl := Must(New("tmpl").Parse(tc.src))
-			err := tmpl.Execute(ioutil.Discard, tc.value)
+			err := tmpl.Execute(io.Discard, tc.value)
 			got := "<nil>"
 			if err != nil {
 				got = err.Error()
@@ -1493,7 +1493,7 @@ func TestMaxExecDepth(t *testing.T) {
 		t.Skip("skipping in -short mode")
 	}
 	tmpl := Must(New("tmpl").Parse(`{{template "tmpl" .}}`))
-	err := tmpl.Execute(ioutil.Discard, nil)
+	err := tmpl.Execute(io.Discard, nil)
 	got := "<nil>"
 	if err != nil {
 		got = err.Error()

tpl/internal/go_templates/texttemplate/helper.go

@@ -8,7 +8,9 @@ package template
 
 import (
 	"fmt"
+	"io/fs"
 	"io/ioutil"
+	"path"
 	"path/filepath"
 )
 
@@ -35,7 +37,7 @@ func Must(t *Template, err error) *Template {
 // For instance, ParseFiles("a/foo", "b/foo") stores "b/foo" as the template
 // named "foo", while "a/foo" is unavailable.
 func ParseFiles(filenames ...string) (*Template, error) {
-	return parseFiles(nil, filenames...)
+	return parseFiles(nil, readFileOS, filenames...)
 }
 
 // ParseFiles parses the named files and associates the resulting templates with
@@ -51,23 +53,22 @@ func ParseFiles(filenames ...string) (*Template, error) {
 // the last one mentioned will be the one that results.
 func (t *Template) ParseFiles(filenames ...string) (*Template, error) {
 	t.init()
-	return parseFiles(t, filenames...)
+	return parseFiles(t, readFileOS, filenames...)
 }
 
 // parseFiles is the helper for the method and function. If the argument
 // template is nil, it is created from the first file.
-func parseFiles(t *Template, filenames ...string) (*Template, error) {
+func parseFiles(t *Template, readFile func(string) (string, []byte, error), filenames ...string) (*Template, error) {
 	if len(filenames) == 0 {
 		// Not really a problem, but be consistent.
 		return nil, fmt.Errorf("template: no files named in call to ParseFiles")
 	}
 	for _, filename := range filenames {
-		b, err := ioutil.ReadFile(filename)
+		name, b, err := readFile(filename)
 		if err != nil {
 			return nil, err
 		}
 		s := string(b)
-		name := filepath.Base(filename)
 		// First template becomes return value if not already defined,
 		// and we use that one for subsequent New calls to associate
 		// all the templates together. Also, if this file has the same name
@@ -126,5 +127,51 @@ func parseGlob(t *Template, pattern string) (*Template, error) {
 	if len(filenames) == 0 {
 		return nil, fmt.Errorf("template: pattern matches no files: %#q", pattern)
 	}
-	return parseFiles(t, filenames...)
+	return parseFiles(t, readFileOS, filenames...)
+}
+
+// ParseFS is like ParseFiles or ParseGlob but reads from the file system fsys
+// instead of the host operating system's file system.
+// It accepts a list of glob patterns.
+// (Note that most file names serve as glob patterns matching only themselves.)
+func ParseFS(fsys fs.FS, patterns ...string) (*Template, error) {
+	return parseFS(nil, fsys, patterns)
+}
+
+// ParseFS is like ParseFiles or ParseGlob but reads from the file system fsys
+// instead of the host operating system's file system.
+// It accepts a list of glob patterns.
+// (Note that most file names serve as glob patterns matching only themselves.)
+func (t *Template) ParseFS(fsys fs.FS, patterns ...string) (*Template, error) {
+	t.init()
+	return parseFS(t, fsys, patterns)
+}
+
+func parseFS(t *Template, fsys fs.FS, patterns []string) (*Template, error) {
+	var filenames []string
+	for _, pattern := range patterns {
+		list, err := fs.Glob(fsys, pattern)
+		if err != nil {
+			return nil, err
+		}
+		if len(list) == 0 {
+			return nil, fmt.Errorf("template: pattern matches no files: %#q", pattern)
+		}
+		filenames = append(filenames, list...)
+	}
+	return parseFiles(t, readFileFS(fsys), filenames...)
+}
+
+func readFileOS(file string) (name string, b []byte, err error) {
+	name = filepath.Base(file)
+	b, err = ioutil.ReadFile(file)
+	return
+}
+
+func readFileFS(fsys fs.FS) func(string) (string, []byte, error) {
+	return func(file string) (name string, b []byte, err error) {
+		name = path.Base(file)
+		b, err = fs.ReadFile(fsys, file)
+		return
+	}
 }

tpl/internal/go_templates/texttemplate/link_test.go

@@ -0,0 +1,66 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.13
+
+package template_test
+
+import (
+	"bytes"
+	"github.com/gohugoio/hugo/tpl/internal/go_templates/testenv"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+)
+
+// Issue 36021: verify that text/template doesn't prevent the linker from removing
+// unused methods.
+func _TestLinkerGC(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping in short mode")
+	}
+	testenv.MustHaveGoBuild(t)
+	const prog = `package main
+
+import (
+	_ "text/template"
+)
+
+type T struct{}
+
+func (t *T) Unused() { println("THIS SHOULD BE ELIMINATED") }
+func (t *T) Used() {}
+
+var sink *T
+
+func main() {
+	var t T
+	sink = &t
+	t.Used()
+}
+`
+	td, err := ioutil.TempDir("", "text_template_TestDeadCodeElimination")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(td)
+
+	if err := ioutil.WriteFile(filepath.Join(td, "x.go"), []byte(prog), 0644); err != nil {
+		t.Fatal(err)
+	}
+	cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", "x.exe", "x.go")
+	cmd.Dir = td
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("go build: %v, %s", err, out)
+	}
+	slurp, err := ioutil.ReadFile(filepath.Join(td, "x.exe"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if bytes.Contains(slurp, []byte("THIS SHOULD BE ELIMINATED")) {
+		t.Error("binary contains code that should be deadcode eliminated")
+	}
+}

tpl/internal/go_templates/texttemplate/multi_test.go

@@ -12,6 +12,7 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse"
+	"os"
 	"testing"
 )
 
@@ -155,6 +156,35 @@ func TestParseGlob(t *testing.T) {
 	testExecute(multiExecTests, template, t)
 }
 
+func TestParseFS(t *testing.T) {
+	fs := os.DirFS("testdata")
+
+	{
+		_, err := ParseFS(fs, "DOES NOT EXIST")
+		if err == nil {
+			t.Error("expected error for non-existent file; got none")
+		}
+	}
+
+	{
+		template := New("root")
+		_, err := template.ParseFS(fs, "file1.tmpl", "file2.tmpl")
+		if err != nil {
+			t.Fatalf("error parsing files: %v", err)
+		}
+		testExecute(multiExecTests, template, t)
+	}
+
+	{
+		template := New("root")
+		_, err := template.ParseFS(fs, "file*.tmpl")
+		if err != nil {
+			t.Fatalf("error parsing files: %v", err)
+		}
+		testExecute(multiExecTests, template, t)
+	}
+}
+
 // In these tests, actual content (not just template definitions) comes from the parsed files.
 
 var templateFileExecTests = []execTest{
@@ -361,6 +391,7 @@ func TestEmptyTemplate(t *testing.T) {
 		in   string
 		want string
 	}{
+		{[]string{"x", "y"}, "", "y"},
 		{[]string{""}, "once", ""},
 		{[]string{"", ""}, "twice", ""},
 		{[]string{"{{.}}", "{{.}}"}, "twice", "twice"},

tpl/internal/go_templates/texttemplate/parse/lex.go

@@ -41,6 +41,7 @@ const (
 	itemBool                         // boolean constant
 	itemChar                         // printable ASCII character; grab bag for comma etc.
 	itemCharConstant                 // character constant
+	itemComment                      // comment text
 	itemComplex                      // complex constant (1+2i); imaginary is just a number
 	itemAssign                       // equals ('=') introducing an assignment
 	itemDeclare                      // colon-equals (':=') introducing a declaration
@@ -91,15 +92,14 @@ const eof = -1
 // If the action begins "{{- " rather than "{{", then all space/tab/newlines
 // preceding the action are trimmed; conversely if it ends " -}}" the
 // leading spaces are trimmed. This is done entirely in the lexer; the
-// parser never sees it happen. We require an ASCII space to be
-// present to avoid ambiguity with things like "{{-3}}". It reads
+// parser never sees it happen. We require an ASCII space (' ', \t, \r, \n)
+// to be present to avoid ambiguity with things like "{{-3}}". It reads
 // better with the space present anyway. For simplicity, only ASCII
-// space does the job.
+// does the job.
 const (
-	spaceChars      = " \t\r\n" // These are the space characters defined by Go itself.
-	leftTrimMarker  = "- "      // Attached to left delimiter, trims trailing spaces from preceding text.
-	rightTrimMarker = " -"      // Attached to right delimiter, trims leading spaces from following text.
-	trimMarkerLen   = Pos(len(leftTrimMarker))
+	spaceChars    = " \t\r\n"  // These are the space characters defined by Go itself.
+	trimMarker    = '-'        // Attached to left/right delimiter, trims trailing spaces from preceding/following text.
+	trimMarkerLen = Pos(1 + 1) // marker plus space before or after
 )
 
 // stateFn represents the state of the scanner as a function that returns the next state.
@@ -107,18 +107,18 @@ type stateFn func(*lexer) stateFn
 
 // lexer holds the state of the scanner.
 type lexer struct {
-	name           string    // the name of the input; used only for error reports
-	input          string    // the string being scanned
-	leftDelim      string    // start of action
-	rightDelim     string    // end of action
-	trimRightDelim string    // end of action with trim marker
-	pos            Pos       // current position in the input
-	start          Pos       // start position of this item
-	width          Pos       // width of last rune read from input
-	items          chan item // channel of scanned items
-	parenDepth     int       // nesting depth of ( ) exprs
-	line           int       // 1+number of newlines seen
-	startLine      int       // start line of this item
+	name        string    // the name of the input; used only for error reports
+	input       string    // the string being scanned
+	leftDelim   string    // start of action
+	rightDelim  string    // end of action
+	emitComment bool      // emit itemComment tokens.
+	pos         Pos       // current position in the input
+	start       Pos       // start position of this item
+	width       Pos       // width of last rune read from input
+	items       chan item // channel of scanned items
+	parenDepth  int       // nesting depth of ( ) exprs
+	line        int       // 1+number of newlines seen
+	startLine   int       // start line of this item
 }
 
 // next returns the next rune in the input.
@@ -203,7 +203,7 @@ func (l *lexer) drain() {
 }
 
 // lex creates a new scanner for the input string.
-func lex(name, input, left, right string) *lexer {
+func lex(name, input, left, right string, emitComment bool) *lexer {
 	if left == "" {
 		left = leftDelim
 	}
@@ -211,14 +211,14 @@ func lex(name, input, left, right string) *lexer {
 		right = rightDelim
 	}
 	l := &lexer{
-		name:           name,
-		input:          input,
-		leftDelim:      left,
-		rightDelim:     right,
-		trimRightDelim: rightTrimMarker + right,
-		items:          make(chan item),
-		line:           1,
-		startLine:      1,
+		name:        name,
+		input:       input,
+		leftDelim:   left,
+		rightDelim:  right,
+		emitComment: emitComment,
+		items:       make(chan item),
+		line:        1,
+		startLine:   1,
 	}
 	go l.run()
 	return l
@@ -248,7 +248,7 @@ func lexText(l *lexer) stateFn {
 		ldn := Pos(len(l.leftDelim))
 		l.pos += Pos(x)
 		trimLength := Pos(0)
-		if strings.HasPrefix(l.input[l.pos+ldn:], leftTrimMarker) {
+		if hasLeftTrimMarker(l.input[l.pos+ldn:]) {
 			trimLength = rightTrimLength(l.input[l.start:l.pos])
 		}
 		l.pos -= trimLength
@@ -277,7 +277,7 @@ func rightTrimLength(s string) Pos {
 
 // atRightDelim reports whether the lexer is at a right delimiter, possibly preceded by a trim marker.
 func (l *lexer) atRightDelim() (delim, trimSpaces bool) {
-	if strings.HasPrefix(l.input[l.pos:], l.trimRightDelim) { // With trim marker.
+	if hasRightTrimMarker(l.input[l.pos:]) && strings.HasPrefix(l.input[l.pos+trimMarkerLen:], l.rightDelim) { // With trim marker.
 		return true, true
 	}
 	if strings.HasPrefix(l.input[l.pos:], l.rightDelim) { // Without trim marker.
@@ -294,7 +294,7 @@ func leftTrimLength(s string) Pos {
 // lexLeftDelim scans the left delimiter, which is known to be present, possibly with a trim marker.
 func lexLeftDelim(l *lexer) stateFn {
 	l.pos += Pos(len(l.leftDelim))
-	trimSpace := strings.HasPrefix(l.input[l.pos:], leftTrimMarker)
+	trimSpace := hasLeftTrimMarker(l.input[l.pos:])
 	afterMarker := Pos(0)
 	if trimSpace {
 		afterMarker = trimMarkerLen
@@ -323,6 +323,9 @@ func lexComment(l *lexer) stateFn {
 	if !delim {
 		return l.errorf("comment ends before closing delimiter")
 	}
+	if l.emitComment {
+		l.emit(itemComment)
+	}
 	if trimSpace {
 		l.pos += trimMarkerLen
 	}
@@ -336,7 +339,7 @@ func lexComment(l *lexer) stateFn {
 
 // lexRightDelim scans the right delimiter, which is known to be present, possibly with a trim marker.
 func lexRightDelim(l *lexer) stateFn {
-	trimSpace := strings.HasPrefix(l.input[l.pos:], rightTrimMarker)
+	trimSpace := hasRightTrimMarker(l.input[l.pos:])
 	if trimSpace {
 		l.pos += trimMarkerLen
 		l.ignore()
@@ -363,7 +366,7 @@ func lexInsideAction(l *lexer) stateFn {
 		return l.errorf("unclosed left paren")
 	}
 	switch r := l.next(); {
-	case r == eof || isEndOfLine(r):
+	case r == eof:
 		return l.errorf("unclosed action")
 	case isSpace(r):
 		l.backup() // Put space back in case we have " -}}".
@@ -433,7 +436,7 @@ func lexSpace(l *lexer) stateFn {
 	}
 	// Be careful about a trim-marked closing delimiter, which has a minus
 	// after a space. We know there is a space, so check for the '-' that might follow.
-	if strings.HasPrefix(l.input[l.pos-1:], l.trimRightDelim) {
+	if hasRightTrimMarker(l.input[l.pos-1:]) && strings.HasPrefix(l.input[l.pos-1+trimMarkerLen:], l.rightDelim) {
 		l.backup() // Before the space.
 		if numSpaces == 1 {
 			return lexRightDelim // On the delim, so go right to that.
@@ -520,7 +523,7 @@ func lexFieldOrVariable(l *lexer, typ itemType) stateFn {
 // day to implement arithmetic.
 func (l *lexer) atTerminator() bool {
 	r := l.peek()
-	if isSpace(r) || isEndOfLine(r) {
+	if isSpace(r) {
 		return true
 	}
 	switch r {
@@ -651,15 +654,18 @@ Loop:
 
 // isSpace reports whether r is a space character.
 func isSpace(r rune) bool {
-	return r == ' ' || r == '\t'
-}
-
-// isEndOfLine reports whether r is an end-of-line character.
-func isEndOfLine(r rune) bool {
-	return r == '\r' || r == '\n'
+	return r == ' ' || r == '\t' || r == '\r' || r == '\n'
 }
 
 // isAlphaNumeric reports whether r is an alphabetic, digit, or underscore.
 func isAlphaNumeric(r rune) bool {
 	return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
 }
+
+func hasLeftTrimMarker(s string) bool {
+	return len(s) >= 2 && s[0] == trimMarker && isSpace(rune(s[1]))
+}
+
+func hasRightTrimMarker(s string) bool {
+	return len(s) >= 2 && isSpace(rune(s[0])) && s[1] == trimMarker
+}

tpl/internal/go_templates/texttemplate/parse/lex_test.go

@@ -17,6 +17,7 @@ var itemName = map[itemType]string{
 	itemBool:         "bool",
 	itemChar:         "char",
 	itemCharConstant: "charconst",
+	itemComment:      "comment",
 	itemComplex:      "complex",
 	itemDeclare:      ":=",
 	itemEOF:          "EOF",
@@ -92,6 +93,7 @@ var lexTests = []lexTest{
 	{"text", `now is the time`, []item{mkItem(itemText, "now is the time"), tEOF}},
 	{"text with comment", "hello-{{/* this is a comment */}}-world", []item{
 		mkItem(itemText, "hello-"),
+		mkItem(itemComment, "/* this is a comment */"),
 		mkItem(itemText, "-world"),
 		tEOF,
 	}},
@@ -313,6 +315,7 @@ var lexTests = []lexTest{
 	}},
 	{"trimming spaces before and after comment", "hello- {{- /* hello */ -}} -world", []item{
 		mkItem(itemText, "hello-"),
+		mkItem(itemComment, "/* hello */"),
 		mkItem(itemText, "-world"),
 		tEOF,
 	}},
@@ -322,7 +325,7 @@ var lexTests = []lexTest{
 		tLeft,
 		mkItem(itemError, "unrecognized character in action: U+0001"),
 	}},
-	{"unclosed action", "{{\n}}", []item{
+	{"unclosed action", "{{", []item{
 		tLeft,
 		mkItem(itemError, "unclosed action"),
 	}},
@@ -391,7 +394,7 @@ var lexTests = []lexTest{
 
 // collect gathers the emitted items into a slice.
 func collect(t *lexTest, left, right string) (items []item) {
-	l := lex(t.name, t.input, left, right)
+	l := lex(t.name, t.input, left, right, true)
 	for {
 		item := l.nextItem()
 		items = append(items, item)
@@ -531,7 +534,7 @@ func TestPos(t *testing.T) {
 func TestShutdown(t *testing.T) {
 	// We need to duplicate template.Parse here to hold on to the lexer.
 	const text = "erroneous{{define}}{{else}}1234"
-	lexer := lex("foo", text, "{{", "}}")
+	lexer := lex("foo", text, "{{", "}}", false)
 	_, err := New("root").parseLexer(lexer)
 	if err == nil {
 		t.Fatalf("expected error")

tpl/internal/go_templates/texttemplate/parse/node.go

@@ -70,6 +70,7 @@ const (
 	NodeTemplate                   // A template invocation action.
 	NodeVariable                   // A $ variable.
 	NodeWith                       // A with action.
+	NodeComment                    // A comment.
 )
 
 // Nodes.
@@ -149,6 +150,38 @@ func (t *TextNode) Copy() Node {
 	return &TextNode{tr: t.tr, NodeType: NodeText, Pos: t.Pos, Text: append([]byte{}, t.Text...)}
 }
 
+// CommentNode holds a comment.
+type CommentNode struct {
+	NodeType
+	Pos
+	tr   *Tree
+	Text string // Comment text.
+}
+
+func (t *Tree) newComment(pos Pos, text string) *CommentNode {
+	return &CommentNode{tr: t, NodeType: NodeComment, Pos: pos, Text: text}
+}
+
+func (c *CommentNode) String() string {
+	var sb strings.Builder
+	c.writeTo(&sb)
+	return sb.String()
+}
+
+func (c *CommentNode) writeTo(sb *strings.Builder) {
+	sb.WriteString("{{")
+	sb.WriteString(c.Text)
+	sb.WriteString("}}")
+}
+
+func (c *CommentNode) tree() *Tree {
+	return c.tr
+}
+
+func (c *CommentNode) Copy() Node {
+	return &CommentNode{tr: c.tr, NodeType: NodeComment, Pos: c.Pos, Text: c.Text}
+}
+
 // PipeNode holds a pipeline with optional declaration
 type PipeNode struct {
 	NodeType
@@ -349,7 +382,7 @@ func (i *IdentifierNode) Copy() Node {
 	return NewIdentifier(i.Ident).SetTree(i.tr).SetPos(i.Pos)
 }
 
-// AssignNode holds a list of variable names, possibly with chained field
+// VariableNode holds a list of variable names, possibly with chained field
 // accesses. The dollar sign is part of the (first) name.
 type VariableNode struct {
 	NodeType

tpl/internal/go_templates/texttemplate/parse/parse.go

@@ -21,16 +21,26 @@ type Tree struct {
 	Name      string    // name of the template represented by the tree.
 	ParseName string    // name of the top-level template during parsing, for error messages.
 	Root      *ListNode // top-level root of the tree.
+	Mode      Mode      // parsing mode.
 	text      string    // text parsed to create the template (or its parent)
 	// Parsing only; cleared after parse.
-	funcs     []map[string]interface{}
-	lex       *lexer
-	token     [3]item // three-token lookahead for parser.
-	peekCount int
-	vars      []string // variables defined at the moment.
-	treeSet   map[string]*Tree
+	funcs      []map[string]interface{}
+	lex        *lexer
+	token      [3]item // three-token lookahead for parser.
+	peekCount  int
+	vars       []string // variables defined at the moment.
+	treeSet    map[string]*Tree
+	actionLine int // line of left delim starting action
+	mode       Mode
 }
 
+// A mode value is a set of flags (or 0). Modes control parser behavior.
+type Mode uint
+
+const (
+	ParseComments Mode = 1 << iota // parse comments and add them to AST
+)
+
 // Copy returns a copy of the Tree. Any parsing state is discarded.
 func (t *Tree) Copy() *Tree {
 	if t == nil {
@@ -178,6 +188,16 @@ func (t *Tree) expectOneOf(expected1, expected2 itemType, context string) item {
 
 // unexpected complains about the token and terminates processing.
 func (t *Tree) unexpected(token item, context string) {
+	if token.typ == itemError {
+		extra := ""
+		if t.actionLine != 0 && t.actionLine != token.line {
+			extra = fmt.Sprintf(" in action started at %s:%d", t.ParseName, t.actionLine)
+			if strings.HasSuffix(token.val, " action") {
+				extra = extra[len(" in action"):] // avoid "action in action"
+			}
+		}
+		t.errorf("%s%s", token, extra)
+	}
 	t.errorf("unexpected %s in %s", token, context)
 }
 
@@ -220,7 +240,8 @@ func (t *Tree) stopParse() {
 func (t *Tree) Parse(text, leftDelim, rightDelim string, treeSet map[string]*Tree, funcs ...map[string]interface{}) (tree *Tree, err error) {
 	defer t.recover(&err)
 	t.ParseName = t.Name
-	t.startParse(funcs, lex(t.Name, text, leftDelim, rightDelim), treeSet)
+	emitComment := t.Mode&ParseComments != 0
+	t.startParse(funcs, lex(t.Name, text, leftDelim, rightDelim, emitComment), treeSet)
 	t.text = text
 	t.parse()
 	t.add()
@@ -240,12 +261,14 @@ func (t *Tree) add() {
 	}
 }
 
-// IsEmptyTree reports whether this tree (node) is empty of everything but space.
+// IsEmptyTree reports whether this tree (node) is empty of everything but space or comments.
 func IsEmptyTree(n Node) bool {
 	switch n := n.(type) {
 	case nil:
 		return true
 	case *ActionNode:
+	case *CommentNode:
+		return true
 	case *IfNode:
 	case *ListNode:
 		for _, node := range n.Nodes {
@@ -276,6 +299,7 @@ func (t *Tree) parse() {
 			if t.nextNonSpace().typ == itemDefine {
 				newT := New("definition") // name will be updated once we know it.
 				newT.text = t.text
+				newT.Mode = t.Mode
 				newT.ParseName = t.ParseName
 				newT.startParse(t.funcs, t.lex, t.treeSet)
 				newT.parseDefinition()
@@ -331,19 +355,27 @@ func (t *Tree) itemList() (list *ListNode, next Node) {
 }
 
 // textOrAction:
-//	text | action
+//	text | comment | action
 func (t *Tree) textOrAction() Node {
 	switch token := t.nextNonSpace(); token.typ {
 	case itemText:
 		return t.newText(token.pos, token.val)
 	case itemLeftDelim:
+		t.actionLine = token.line
+		defer t.clearActionLine()
 		return t.action()
+	case itemComment:
+		return t.newComment(token.pos, token.val)
 	default:
 		t.unexpected(token, "input")
 	}
 	return nil
 }
 
+func (t *Tree) clearActionLine() {
+	t.actionLine = 0
+}
+
 // Action:
 //	control
 //	command ("|" command)*
@@ -369,12 +401,12 @@ func (t *Tree) action() (n Node) {
 	t.backup()
 	token := t.peek()
 	// Do not pop variables; they persist until "end".
-	return t.newAction(token.pos, token.line, t.pipeline("command"))
+	return t.newAction(token.pos, token.line, t.pipeline("command", itemRightDelim))
 }
 
 // Pipeline:
 //	declarations? command ('|' command)*
-func (t *Tree) pipeline(context string) (pipe *PipeNode) {
+func (t *Tree) pipeline(context string, end itemType) (pipe *PipeNode) {
 	token := t.peekNonSpace()
 	pipe = t.newPipeline(token.pos, token.line, nil)
 	// Are there declarations or assignments?
@@ -415,12 +447,9 @@ decls:
 	}
 	for {
 		switch token := t.nextNonSpace(); token.typ {
-		case itemRightDelim, itemRightParen:
+		case end:
 			// At this point, the pipeline is complete
 			t.checkPipeline(pipe, context)
-			if token.typ == itemRightParen {
-				t.backup()
-			}
 			return
 		case itemBool, itemCharConstant, itemComplex, itemDot, itemField, itemIdentifier,
 			itemNumber, itemNil, itemRawString, itemString, itemVariable, itemLeftParen:
@@ -449,7 +478,7 @@ func (t *Tree) checkPipeline(pipe *PipeNode, context string) {
 
 func (t *Tree) parseControl(allowElseIf bool, context string) (pos Pos, line int, pipe *PipeNode, list, elseList *ListNode) {
 	defer t.popVars(len(t.vars))
-	pipe = t.pipeline(context)
+	pipe = t.pipeline(context, itemRightDelim)
 	var next Node
 	list, next = t.itemList()
 	switch next.Type() {
@@ -535,10 +564,11 @@ func (t *Tree) blockControl() Node {
 
 	token := t.nextNonSpace()
 	name := t.parseTemplateName(token, context)
-	pipe := t.pipeline(context)
+	pipe := t.pipeline(context, itemRightDelim)
 
 	block := New(name) // name will be updated once we know it.
 	block.text = t.text
+	block.Mode = t.Mode
 	block.ParseName = t.ParseName
 	block.startParse(t.funcs, t.lex, t.treeSet)
 	var end Node
@@ -564,7 +594,7 @@ func (t *Tree) templateControl() Node {
 	if t.nextNonSpace().typ != itemRightDelim {
 		t.backup()
 		// Do not pop variables; they persist until "end".
-		pipe = t.pipeline(context)
+		pipe = t.pipeline(context, itemRightDelim)
 	}
 	return t.newTemplate(token.pos, token.line, name, pipe)
 }
@@ -598,13 +628,12 @@ func (t *Tree) command() *CommandNode {
 		switch token := t.next(); token.typ {
 		case itemSpace:
 			continue
-		case itemError:
-			t.errorf("%s", token.val)
 		case itemRightDelim, itemRightParen:
 			t.backup()
 		case itemPipe:
+			// nothing here; break loop below
 		default:
-			t.errorf("unexpected %s in operand", token)
+			t.unexpected(token, "operand")
 		}
 		break
 	}
@@ -659,8 +688,6 @@ func (t *Tree) operand() Node {
 // A nil return means the next item is not a term.
 func (t *Tree) term() Node {
 	switch token := t.nextNonSpace(); token.typ {
-	case itemError:
-		t.errorf("%s", token.val)
 	case itemIdentifier:
 		if !t.hasFunction(token.val) {
 			t.errorf("function %q not defined", token.val)
@@ -683,11 +710,7 @@ func (t *Tree) term() Node {
 		}
 		return number
 	case itemLeftParen:
-		pipe := t.pipeline("parenthesized pipeline")
-		if token := t.next(); token.typ != itemRightParen {
-			t.errorf("unclosed right paren: unexpected %s", token)
-		}
-		return pipe
+		return t.pipeline("parenthesized pipeline", itemRightParen)
 	case itemString, itemRawString:
 		s, err := strconv.Unquote(token.val)
 		if err != nil {

tpl/internal/go_templates/texttemplate/parse/parse_test.go

@@ -252,6 +252,13 @@ var parseTests = []parseTest{
 	{"comment trim left and right", "x \r\n\t{{- /* */ -}}\n\n\ty", noError, `"x""y"`},
 	{"block definition", `{{block "foo" .}}hello{{end}}`, noError,
 		`{{template "foo" .}}`},
+
+	{"newline in assignment", "{{ $x \n := \n 1 \n }}", noError, "{{$x := 1}}"},
+	{"newline in empty action", "{{\n}}", hasError, "{{\n}}"},
+	{"newline in pipeline", "{{\n\"x\"\n|\nprintf\n}}", noError, `{{"x" | printf}}`},
+	{"newline in comment", "{{/*\nhello\n*/}}", noError, ""},
+	{"newline in comment", "{{-\n/*\nhello\n*/\n-}}", noError, ""},
+
 	// Errors.
 	{"unclosed action", "hello{{range", hasError, ""},
 	{"unmatched end", "{{end}}", hasError, ""},
@@ -350,6 +357,30 @@ func TestParseCopy(t *testing.T) {
 	testParse(true, t)
 }
 
+func TestParseWithComments(t *testing.T) {
+	textFormat = "%q"
+	defer func() { textFormat = "%s" }()
+	tests := [...]parseTest{
+		{"comment", "{{/*\n\n\n*/}}", noError, "{{/*\n\n\n*/}}"},
+		{"comment trim left", "x \r\n\t{{- /* hi */}}", noError, `"x"{{/* hi */}}`},
+		{"comment trim right", "{{/* hi */ -}}\n\n\ty", noError, `{{/* hi */}}"y"`},
+		{"comment trim left and right", "x \r\n\t{{- /* */ -}}\n\n\ty", noError, `"x"{{/* */}}"y"`},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			tr := New(test.name)
+			tr.Mode = ParseComments
+			tmpl, err := tr.Parse(test.input, "", "", make(map[string]*Tree))
+			if err != nil {
+				t.Errorf("%q: expected error; got none", test.name)
+			}
+			if result := tmpl.Root.String(); result != test.result {
+				t.Errorf("%s=(%q): got\n\t%v\nexpected\n\t%v", test.name, test.input, result, test.result)
+			}
+		})
+	}
+}
+
 type isEmptyTest struct {
 	name  string
 	input string
@@ -360,6 +391,7 @@ var isEmptyTests = []isEmptyTest{
 	{"empty", ``, true},
 	{"nonempty", `hello`, false},
 	{"spaces only", " \t\n \t\n", true},
+	{"comment only", "{{/* comment */}}", true},
 	{"definition", `{{define "x"}}something{{end}}`, true},
 	{"definitions and space", "{{define `x`}}something{{end}}\n\n{{define `y`}}something{{end}}\n\n", true},
 	{"definitions and text", "{{define `x`}}something{{end}}\nx\n{{define `y`}}something{{end}}\ny\n", false},
@@ -403,23 +435,38 @@ var errorTests = []parseTest{
 	// Check line numbers are accurate.
 	{"unclosed1",
 		"line1\n{{",
-		hasError, `unclosed1:2: unexpected unclosed action in command`},
+		hasError, `unclosed1:2: unclosed action`},
 	{"unclosed2",
 		"line1\n{{define `x`}}line2\n{{",
-		hasError, `unclosed2:3: unexpected unclosed action in command`},
+		hasError, `unclosed2:3: unclosed action`},
+	{"unclosed3",
+		"line1\n{{\"x\"\n\"y\"\n",
+		hasError, `unclosed3:4: unclosed action started at unclosed3:2`},
+	{"unclosed4",
+		"{{\n\n\n\n\n",
+		hasError, `unclosed4:6: unclosed action started at unclosed4:1`},
+	{"var1",
+		"line1\n{{\nx\n}}",
+		hasError, `var1:3: function "x" not defined`},
 	// Specific errors.
 	{"function",
 		"{{foo}}",
 		hasError, `function "foo" not defined`},
-	{"comment",
+	{"comment1",
 		"{{/*}}",
-		hasError, `unclosed comment`},
+		hasError, `comment1:1: unclosed comment`},
+	{"comment2",
+		"{{/*\nhello\n}}",
+		hasError, `comment2:1: unclosed comment`},
 	{"lparen",
 		"{{.X (1 2 3}}",
 		hasError, `unclosed left paren`},
 	{"rparen",
-		"{{.X 1 2 3)}}",
-		hasError, `unexpected ")"`},
+		"{{.X 1 2 3 ) }}",
+		hasError, `unexpected ")" in command`},
+	{"rparen2",
+		"{{(.X 1 2 3",
+		hasError, `unclosed action`},
 	{"space",
 		"{{`x`3}}",
 		hasError, `in operand`},
@@ -465,7 +512,7 @@ var errorTests = []parseTest{
 		hasError, `missing value for parenthesized pipeline`},
 	{"multilinerawstring",
 		"{{ $v := `\n` }} {{",
-		hasError, `multilinerawstring:2: unexpected unclosed action`},
+		hasError, `multilinerawstring:2: unclosed action`},
 	{"rangeundefvar",
 		"{{range $k}}{{end}}",
 		hasError, `undefined variable`},