commands: Show server error info in browser

The main item in this commit is showing of errors with a file context when running `hugo server`.

This can be turned off: `hugo server --disableBrowserError` (can also be set in `config.toml`).

But to get there, the error handling in Hugo needed a revision. There are some items left TODO for commits soon to follow, most notable errors in content and config files.

Fixes #5284
Fixes #5290
See #5325
See #5324

common/herrors/error_locator.go

@@ -0,0 +1,194 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package errors contains common Hugo errors and error related utilities.
+package herrors
+
+import (
+	"bufio"
+	"io"
+	"strings"
+
+	"github.com/spf13/afero"
+)
+
+// LineMatcher is used to match a line with an error.
+type LineMatcher func(le FileError, lineNumber int, line string) bool
+
+// SimpleLineMatcher matches if the current line number matches the line number
+// in the error.
+var SimpleLineMatcher = func(le FileError, lineNumber int, line string) bool {
+	return le.LineNumber() == lineNumber
+}
+
+// ErrorContext contains contextual information about an error. This will
+// typically be the lines surrounding some problem in a file.
+type ErrorContext struct {
+
+	// If a match will contain the matched line and up to 2 lines before and after.
+	// Will be empty if no match.
+	Lines []string
+
+	// The position of the error in the Lines above. 0 based.
+	Pos int
+
+	// The linenumber in the source file from where the Lines start. Starting at 1.
+	LineNumber int
+
+	// The lexer to use for syntax highlighting.
+	// https://gohugo.io/content-management/syntax-highlighting/#list-of-chroma-highlighting-languages
+	ChromaLexer string
+}
+
+var _ causer = (*ErrorWithFileContext)(nil)
+
+// ErrorWithFileContext is an error with some additional file context related
+// to that error.
+type ErrorWithFileContext struct {
+	cause error
+	ErrorContext
+}
+
+func (e *ErrorWithFileContext) Error() string {
+	return e.cause.Error()
+}
+
+func (e *ErrorWithFileContext) Cause() error {
+	return e.cause
+}
+
+// WithFileContextForFile will try to add a file context with lines matching the given matcher.
+// If no match could be found, the original error is returned with false as the second return value.
+func WithFileContextForFile(e error, filename string, fs afero.Fs, chromaLexer string, matcher LineMatcher) (error, bool) {
+	f, err := fs.Open(filename)
+	if err != nil {
+		return e, false
+	}
+	defer f.Close()
+	return WithFileContext(e, f, chromaLexer, matcher)
+}
+
+// WithFileContextForFile will try to add a file context with lines matching the given matcher.
+// If no match could be found, the original error is returned with false as the second return value.
+func WithFileContext(e error, r io.Reader, chromaLexer string, matcher LineMatcher) (error, bool) {
+	if e == nil {
+		panic("error missing")
+	}
+	le := UnwrapFileError(e)
+	if le == nil {
+		var ok bool
+		if le, ok = ToFileError("bash", e).(FileError); !ok {
+			return e, false
+		}
+	}
+
+	errCtx := locateError(r, le, matcher)
+
+	if errCtx.LineNumber == -1 {
+		return e, false
+	}
+
+	if chromaLexer != "" {
+		errCtx.ChromaLexer = chromaLexer
+	} else {
+		errCtx.ChromaLexer = chromaLexerFromType(le.Type())
+	}
+
+	return &ErrorWithFileContext{cause: e, ErrorContext: errCtx}, true
+}
+
+// UnwrapErrorWithFileContext tries to unwrap an ErrorWithFileContext from err.
+// It returns nil if this is not possible.
+func UnwrapErrorWithFileContext(err error) *ErrorWithFileContext {
+	for err != nil {
+		switch v := err.(type) {
+		case *ErrorWithFileContext:
+			return v
+		case causer:
+			err = v.Cause()
+		default:
+			return nil
+		}
+	}
+	return nil
+}
+
+func chromaLexerFromType(fileType string) string {
+	return fileType
+}
+
+func locateErrorInString(le FileError, src string, matcher LineMatcher) ErrorContext {
+	return locateError(strings.NewReader(src), nil, matcher)
+}
+
+func locateError(r io.Reader, le FileError, matches LineMatcher) ErrorContext {
+	var errCtx ErrorContext
+	s := bufio.NewScanner(r)
+
+	lineNo := 0
+
+	var buff [6]string
+	i := 0
+	errCtx.Pos = -1
+
+	for s.Scan() {
+		lineNo++
+		txt := s.Text()
+		buff[i] = txt
+
+		if errCtx.Pos != -1 && i >= 5 {
+			break
+		}
+
+		if errCtx.Pos == -1 && matches(le, lineNo, txt) {
+			errCtx.Pos = i
+			errCtx.LineNumber = lineNo - i
+		}
+
+		if errCtx.Pos == -1 && i == 2 {
+			// Shift left
+			buff[0], buff[1] = buff[i-1], buff[i]
+		} else {
+			i++
+		}
+	}
+
+	// Go's template parser will typically report "unexpected EOF" errors on the
+	// empty last line that is supressed by the scanner.
+	// Do an explicit check for that.
+	if errCtx.Pos == -1 {
+		lineNo++
+		if matches(le, lineNo, "") {
+			buff[i] = ""
+			errCtx.Pos = i
+			errCtx.LineNumber = lineNo - 1
+
+			i++
+		}
+	}
+
+	if errCtx.Pos != -1 {
+		low := errCtx.Pos - 2
+		if low < 0 {
+			low = 0
+		}
+		high := i
+		errCtx.Lines = buff[low:high]
+
+	} else {
+		errCtx.Pos = -1
+		errCtx.LineNumber = -1
+	}
+
+	return errCtx
+}

common/herrors/error_locator_test.go

@@ -0,0 +1,112 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package errors contains common Hugo errors and error related utilities.
+package herrors
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestErrorLocator(t *testing.T) {
+	assert := require.New(t)
+
+	lineMatcher := func(le FileError, lineno int, line string) bool {
+		return strings.Contains(line, "THEONE")
+	}
+
+	lines := `LINE 1
+LINE 2
+LINE 3
+LINE 4
+This is THEONE
+LINE 6
+LINE 7
+LINE 8
+`
+
+	location := locateErrorInString(nil, lines, lineMatcher)
+	assert.Equal([]string{"LINE 3", "LINE 4", "This is THEONE", "LINE 6", "LINE 7"}, location.Lines)
+
+	assert.Equal(3, location.LineNumber)
+	assert.Equal(2, location.Pos)
+
+	assert.Equal([]string{"This is THEONE"}, locateErrorInString(nil, `This is THEONE`, lineMatcher).Lines)
+
+	location = locateErrorInString(nil, `L1
+This is THEONE
+L2
+`, lineMatcher)
+	assert.Equal(1, location.Pos)
+	assert.Equal([]string{"L1", "This is THEONE", "L2"}, location.Lines)
+
+	location = locateErrorInString(nil, `This is THEONE
+L2
+`, lineMatcher)
+	assert.Equal(0, location.Pos)
+	assert.Equal([]string{"This is THEONE", "L2"}, location.Lines)
+
+	location = locateErrorInString(nil, `L1
+This THEONE
+`, lineMatcher)
+	assert.Equal([]string{"L1", "This THEONE"}, location.Lines)
+	assert.Equal(1, location.Pos)
+
+	location = locateErrorInString(nil, `L1
+L2
+This THEONE
+`, lineMatcher)
+	assert.Equal([]string{"L1", "L2", "This THEONE"}, location.Lines)
+	assert.Equal(2, location.Pos)
+
+	location = locateErrorInString(nil, "NO MATCH", lineMatcher)
+	assert.Equal(-1, location.LineNumber)
+	assert.Equal(-1, location.Pos)
+	assert.Equal(0, len(location.Lines))
+
+	lineMatcher = func(le FileError, lineno int, line string) bool {
+		return lineno == 6
+	}
+	location = locateErrorInString(nil, `A
+B
+C
+D
+E
+F
+G
+H
+I
+J`, lineMatcher)
+
+	assert.Equal([]string{"D", "E", "F", "G", "H"}, location.Lines)
+	assert.Equal(4, location.LineNumber)
+	assert.Equal(2, location.Pos)
+
+	// Test match EOF
+	lineMatcher = func(le FileError, lineno int, line string) bool {
+		return lineno == 4
+	}
+
+	location = locateErrorInString(nil, `A
+B
+C
+`, lineMatcher)
+
+	assert.Equal([]string{"B", "C", ""}, location.Lines)
+	assert.Equal(3, location.LineNumber)
+	assert.Equal(2, location.Pos)
+
+}

common/herrors/errors.go

@@ -11,13 +11,41 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package errors contains common Hugo errors and error related utilities.
-package errors
+// Package herrors contains common Hugo errors and error related utilities.
+package herrors
 
 import (
 	"errors"
+	"fmt"
+	"io"
+	"os"
+
+	_errors "github.com/pkg/errors"
 )
 
+// As defined in https://godoc.org/github.com/pkg/errors
+type causer interface {
+	Cause() error
+}
+
+type stackTracer interface {
+	StackTrace() _errors.StackTrace
+}
+
+// PrintStackTrace prints the error's stack trace to stdoud.
+func PrintStackTrace(err error) {
+	FprintStackTrace(os.Stdout, err)
+}
+
+// FprintStackTrace prints the error's stack trace to w.
+func FprintStackTrace(w io.Writer, err error) {
+	if err, ok := err.(stackTracer); ok {
+		for _, f := range err.StackTrace() {
+			fmt.Fprintf(w, "%+s:%d\n", f, f)
+		}
+	}
+}
+
 // ErrFeatureNotAvailable denotes that a feature is unavailable.
 //
 // We will, at least to begin with, make some Hugo features (SCSS with libsass) optional,

common/herrors/file_error.go

@@ -0,0 +1,111 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitatio	ns under the License.
+
+package herrors
+
+import (
+	"fmt"
+)
+
+var _ causer = (*fileError)(nil)
+
+// FileError represents an error when handling a file: Parsing a config file,
+// execute a template etc.
+type FileError interface {
+	error
+
+	// LineNumber gets the error location, starting at line 1.
+	LineNumber() int
+
+	// A string identifying the type of file, e.g. JSON, TOML, markdown etc.
+	Type() string
+}
+
+var _ FileError = (*fileError)(nil)
+
+type fileError struct {
+	lineNumber int
+	fileType   string
+	msg        string
+
+	cause error
+}
+
+func (e *fileError) LineNumber() int {
+	return e.lineNumber
+}
+
+func (e *fileError) Type() string {
+	return e.fileType
+}
+
+func (e *fileError) Error() string {
+	return e.msg
+}
+
+func (f *fileError) Cause() error {
+	return f.cause
+}
+
+func (e *fileError) Format(s fmt.State, verb rune) {
+	switch verb {
+	case 'v':
+		fallthrough
+	case 's':
+		fmt.Fprintf(s, "%s:%d: %s:%s", e.fileType, e.lineNumber, e.msg, e.cause)
+	case 'q':
+		fmt.Fprintf(s, "%q:%d: %q:%q", e.fileType, e.lineNumber, e.msg, e.cause)
+	}
+}
+
+// NewFileError creates a new FileError.
+func NewFileError(fileType string, lineNumber int, msg string, err error) FileError {
+	return &fileError{cause: err, fileType: fileType, lineNumber: lineNumber, msg: msg}
+}
+
+// UnwrapFileError tries to unwrap a FileError from err.
+// It returns nil if this is not possible.
+func UnwrapFileError(err error) FileError {
+	for err != nil {
+		switch v := err.(type) {
+		case FileError:
+			return v
+		case causer:
+			err = v.Cause()
+		default:
+			return nil
+		}
+	}
+	return nil
+}
+
+// ToFileError will try to convert the given error to an error supporting
+// the FileError interface.
+// If will fall back to returning the original error if a line number cannot be extracted.
+func ToFileError(fileType string, err error) error {
+	return ToFileErrorWithOffset(fileType, err, 0)
+}
+
+// ToFileErrorWithOffset will try to convert the given error to an error supporting
+// the FileError interface. It will take any line number offset given into account.
+// If will fall back to returning the original error if a line number cannot be extracted.
+func ToFileErrorWithOffset(fileType string, err error, offset int) error {
+	for _, handle := range lineNumberExtractors {
+		lno, msg := handle(err, offset)
+		if lno > 0 {
+			return NewFileError(fileType, lno, msg, err)
+		}
+	}
+	// Fall back to the original.
+	return err
+}

common/herrors/file_error_test.go

@@ -0,0 +1,56 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package herrors
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestToLineNumberError(t *testing.T) {
+	t.Parallel()
+
+	assert := require.New(t)
+
+	for i, test := range []struct {
+		in         error
+		offset     int
+		lineNumber int
+	}{
+		{errors.New("no line number for you"), 0, -1},
+		{errors.New(`template: _default/single.html:2:15: executing "_default/single.html" at <.Titles>: can't evaluate field`), 0, 2},
+		{errors.New("parse failed: template: _default/bundle-resource-meta.html:11: unexpected in operand"), 0, 11},
+		{errors.New(`failed:: template: _default/bundle-resource-meta.html:2:7: executing "main" at <.Titles>`), 0, 2},
+		{errors.New("error in front matter: Near line 32 (last key parsed 'title')"), 0, 32},
+		{errors.New("error in front matter: Near line 32 (last key parsed 'title')"), 2, 34},
+	} {
+
+		got := ToFileErrorWithOffset("template", test.in, test.offset)
+
+		errMsg := fmt.Sprintf("[%d][%T]", i, got)
+		le, ok := got.(FileError)
+
+		if test.lineNumber > 0 {
+			assert.True(ok)
+			assert.Equal(test.lineNumber, le.LineNumber(), errMsg)
+			assert.Contains(got.Error(), strconv.Itoa(le.LineNumber()))
+		} else {
+			assert.False(ok)
+		}
+	}
+}

common/herrors/line_number_extractors.go

@@ -0,0 +1,59 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitatio	ns under the License.
+
+package herrors
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+)
+
+var lineNumberExtractors = []lineNumberExtractor{
+	// Template/shortcode parse errors
+	newLineNumberErrHandlerFromRegexp("(.*?:)(\\d+)(:.*)"),
+
+	// TOML parse errors
+	newLineNumberErrHandlerFromRegexp("(.*Near line )(\\d+)(\\s.*)"),
+
+	// YAML parse errors
+	newLineNumberErrHandlerFromRegexp("(line )(\\d+)(:)"),
+}
+
+type lineNumberExtractor func(e error, offset int) (int, string)
+
+func newLineNumberErrHandlerFromRegexp(expression string) lineNumberExtractor {
+	re := regexp.MustCompile(expression)
+	return extractLineNo(re)
+}
+
+func extractLineNo(re *regexp.Regexp) lineNumberExtractor {
+	return func(e error, offset int) (int, string) {
+		if e == nil {
+			panic("no error")
+		}
+		s := e.Error()
+		m := re.FindStringSubmatch(s)
+		if len(m) == 4 {
+			i, _ := strconv.Atoi(m[2])
+			msg := e.Error()
+			if offset != 0 {
+				i = i + offset
+				msg = re.ReplaceAllString(s, fmt.Sprintf("${1}%d${3}", i))
+			}
+			return i, msg
+		}
+
+		return -1, ""
+	}
+}

common/loggers/loggers.go

@@ -14,6 +14,8 @@
 package loggers
 
 import (
+	"bytes"
+	"io"
 	"io/ioutil"
 	"log"
 	"os"
@@ -21,17 +23,78 @@ import (
 	jww "github.com/spf13/jwalterweatherman"
 )
 
+var (
+	// Counts ERROR logs to the global jww logger.
+	GlobalErrorCounter *jww.Counter
+)
+
+func init() {
+	GlobalErrorCounter = &jww.Counter{}
+	jww.SetLogListeners(jww.LogCounter(GlobalErrorCounter, jww.LevelError))
+}
+
+// Logger wraps a *loggers.Logger and some other related logging state.
+type Logger struct {
+	*jww.Notepad
+	ErrorCounter *jww.Counter
+
+	// This is only set in server mode.
+	Errors *bytes.Buffer
+}
+
+// Reset resets the logger's internal state.
+func (l *Logger) Reset() {
+	l.ErrorCounter.Reset()
+	if l.Errors != nil {
+		l.Errors.Reset()
+	}
+}
+
+//  NewLogger creates a new Logger for the given thresholds
+func NewLogger(stdoutThreshold, logThreshold jww.Threshold, outHandle, logHandle io.Writer, saveErrors bool) *Logger {
+	return newLogger(stdoutThreshold, logThreshold, outHandle, logHandle, saveErrors)
+}
+
 // NewDebugLogger is a convenience function to create a debug logger.
-func NewDebugLogger() *jww.Notepad {
-	return jww.NewNotepad(jww.LevelDebug, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime)
+func NewDebugLogger() *Logger {
+	return newBasicLogger(jww.LevelDebug)
 }
 
 // NewWarningLogger is a convenience function to create a warning logger.
-func NewWarningLogger() *jww.Notepad {
-	return jww.NewNotepad(jww.LevelWarn, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime)
+func NewWarningLogger() *Logger {
+	return newBasicLogger(jww.LevelWarn)
 }
 
 // NewErrorLogger is a convenience function to create an error logger.
-func NewErrorLogger() *jww.Notepad {
-	return jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime)
+func NewErrorLogger() *Logger {
+	return newBasicLogger(jww.LevelError)
+}
+
+func newLogger(stdoutThreshold, logThreshold jww.Threshold, outHandle, logHandle io.Writer, saveErrors bool) *Logger {
+	errorCounter := &jww.Counter{}
+	listeners := []jww.LogListener{jww.LogCounter(errorCounter, jww.LevelError)}
+	var errorBuff *bytes.Buffer
+	if saveErrors {
+		errorBuff = new(bytes.Buffer)
+		errorCapture := func(t jww.Threshold) io.Writer {
+			if t != jww.LevelError {
+				// Only interested in ERROR
+				return nil
+			}
+
+			return errorBuff
+		}
+
+		listeners = append(listeners, errorCapture)
+	}
+
+	return &Logger{
+		Notepad:      jww.NewNotepad(stdoutThreshold, logThreshold, outHandle, logHandle, "", log.Ldate|log.Ltime, listeners...),
+		ErrorCounter: errorCounter,
+		Errors:       errorBuff,
+	}
+}
+
+func newBasicLogger(t jww.Threshold) *Logger {
+	return newLogger(t, jww.LevelError, os.Stdout, ioutil.Discard, false)
 }