Support typed bool, int and float in shortcode params

This means that you now can do:

    {{< vidur 9KvBeKu false true 32 3.14 >}}

And the boolean and numeric values will be converted to `bool`, `int` and `float64`.

If you want these to be  strings, they must be quoted:

    {{< vidur 9KvBeKu "false" "true" "32" "3.14" >}}

Fixes #6371

parser/pageparser/item.go

@@ -16,12 +16,15 @@ package pageparser
 import (
 	"bytes"
 	"fmt"
+	"regexp"
+	"strconv"
 )
 
 type Item struct {
-	Type ItemType
-	Pos  int
-	Val  []byte
+	Type     ItemType
+	Pos      int
+	Val      []byte
+	isString bool
 }
 
 type Items []Item
@@ -30,6 +33,36 @@ func (i Item) ValStr() string {
 	return string(i.Val)
 }
 
+func (i Item) ValTyped() interface{} {
+	str := i.ValStr()
+	if i.isString {
+		// A quoted value that is a string even if it looks like a number etc.
+		return str
+	}
+
+	if boolRe.MatchString(str) {
+		return str == "true"
+	}
+
+	if intRe.MatchString(str) {
+		num, err := strconv.Atoi(str)
+		if err != nil {
+			return str
+		}
+		return num
+	}
+
+	if floatRe.MatchString(str) {
+		num, err := strconv.ParseFloat(str, 64)
+		if err != nil {
+			return str
+		}
+		return num
+	}
+
+	return str
+}
+
 func (i Item) IsText() bool {
 	return i.Type == tText
 }
@@ -132,3 +165,9 @@ const (
 	// preserved for later - keywords come after this
 	tKeywordMarker
 )
+
+var (
+	boolRe  = regexp.MustCompile(`^(true$)|(false$)`)
+	intRe   = regexp.MustCompile(`^[-+]?\d+$`)
+	floatRe = regexp.MustCompile(`^[-+]?\d*\.\d+$`)
+)

parser/pageparser/item_test.go

@@ -0,0 +1,35 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pageparser
+
+import (
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestItemValTyped(t *testing.T) {
+	c := qt.New(t)
+
+	c.Assert(Item{Val: []byte("3.14")}.ValTyped(), qt.Equals, float64(3.14))
+	c.Assert(Item{Val: []byte(".14")}.ValTyped(), qt.Equals, float64(.14))
+	c.Assert(Item{Val: []byte("314")}.ValTyped(), qt.Equals, 314)
+	c.Assert(Item{Val: []byte("314x")}.ValTyped(), qt.Equals, "314x")
+	c.Assert(Item{Val: []byte("314 ")}.ValTyped(), qt.Equals, "314 ")
+	c.Assert(Item{Val: []byte("314"), isString: true}.ValTyped(), qt.Equals, "314")
+	c.Assert(Item{Val: []byte("true")}.ValTyped(), qt.Equals, true)
+	c.Assert(Item{Val: []byte("false")}.ValTyped(), qt.Equals, false)
+	c.Assert(Item{Val: []byte("trues")}.ValTyped(), qt.Equals, "trues")
+
+}

parser/pageparser/pagelexer.go

@@ -142,7 +142,13 @@ func (l *pageLexer) backup() {
 
 // sends an item back to the client.
 func (l *pageLexer) emit(t ItemType) {
-	l.items = append(l.items, Item{t, l.start, l.input[l.start:l.pos]})
+	l.items = append(l.items, Item{t, l.start, l.input[l.start:l.pos], false})
+	l.start = l.pos
+}
+
+// sends a string item back to the client.
+func (l *pageLexer) emitString(t ItemType) {
+	l.items = append(l.items, Item{t, l.start, l.input[l.start:l.pos], true})
 	l.start = l.pos
 }
 
@@ -151,14 +157,14 @@ func (l *pageLexer) isEOF() bool {
 }
 
 // special case, do not send '\\' back to client
-func (l *pageLexer) ignoreEscapesAndEmit(t ItemType) {
+func (l *pageLexer) ignoreEscapesAndEmit(t ItemType, isString bool) {
 	val := bytes.Map(func(r rune) rune {
 		if r == '\\' {
 			return -1
 		}
 		return r
 	}, l.input[l.start:l.pos])
-	l.items = append(l.items, Item{t, l.start, val})
+	l.items = append(l.items, Item{t, l.start, val, isString})
 	l.start = l.pos
 }
 
@@ -176,7 +182,7 @@ var lf = []byte("\n")
 
 // nil terminates the parser
 func (l *pageLexer) errorf(format string, args ...interface{}) stateFunc {
-	l.items = append(l.items, Item{tError, l.start, []byte(fmt.Sprintf(format, args...))})
+	l.items = append(l.items, Item{tError, l.start, []byte(fmt.Sprintf(format, args...)), true})
 	return nil
 }
 
@@ -201,6 +207,16 @@ func (l *pageLexer) consumeToNextLine() {
 	}
 }
 
+func (l *pageLexer) consumeToSpace() {
+	for {
+		r := l.next()
+		if r == eof || unicode.IsSpace(r) {
+			l.backup()
+			return
+		}
+	}
+}
+
 func (l *pageLexer) consumeSpace() {
 	for {
 		r := l.next()

parser/pageparser/pagelexer_shortcode.go

@@ -112,7 +112,7 @@ func lexShortcodeParam(l *pageLexer, escapedQuoteStart bool) stateFunc {
 			break
 		}
 
-		if !isAlphaNumericOrHyphen(r) {
+		if !isAlphaNumericOrHyphen(r) && r != '.' { // Floats have period
 			l.backup()
 			break
 		}
@@ -137,6 +137,12 @@ func lexShortcodeParam(l *pageLexer, escapedQuoteStart bool) stateFunc {
 
 }
 
+func lexShortcodeParamVal(l *pageLexer) stateFunc {
+	l.consumeToSpace()
+	l.emit(tScParamVal)
+	return lexInsideShortcode
+}
+
 func lexShortcodeQuotedParamVal(l *pageLexer, escapedQuotedValuesAllowed bool, typ ItemType) stateFunc {
 	openQuoteFound := false
 	escapedInnerQuoteFound := false
@@ -176,9 +182,9 @@ Loop:
 	}
 
 	if escapedInnerQuoteFound {
-		l.ignoreEscapesAndEmit(typ)
+		l.ignoreEscapesAndEmit(typ, true)
 	} else {
-		l.emit(typ)
+		l.emitString(typ)
 	}
 
 	r := l.next()
@@ -273,8 +279,13 @@ func lexInsideShortcode(l *pageLexer) stateFunc {
 	case isSpace(r), isEndOfLine(r):
 		l.ignore()
 	case r == '=':
+		l.consumeSpace()
 		l.ignore()
-		return lexShortcodeQuotedParamVal(l, l.peek() != '\\', tScParamVal)
+		peek := l.peek()
+		if peek == '"' || peek == '\\' {
+			return lexShortcodeQuotedParamVal(l, peek != '\\', tScParamVal)
+		}
+		return lexShortcodeParamVal
 	case r == '/':
 		if l.currShortcodeName == "" {
 			return l.errorf("got closing shortcode, but none is open")

parser/pageparser/pageparser.go

@@ -80,7 +80,7 @@ func (t *Iterator) Input() []byte {
 	return t.l.Input()
 }
 
-var errIndexOutOfBounds = Item{tError, 0, []byte("no more tokens")}
+var errIndexOutOfBounds = Item{tError, 0, []byte("no more tokens"), true}
 
 // Current will repeatably return the current item.
 func (t *Iterator) Current() Item {

parser/pageparser/pageparser_intro_test.go

@@ -27,7 +27,7 @@ type lexerTest struct {
 }
 
 func nti(tp ItemType, val string) Item {
-	return Item{tp, 0, []byte(val)}
+	return Item{tp, 0, []byte(val), false}
 }
 
 var (
@@ -119,6 +119,7 @@ func equal(i1, i2 []Item) bool {
 		if i1[k].Type != i2[k].Type {
 			return false
 		}
+
 		if !reflect.DeepEqual(i1[k].Val, i2[k].Val) {
 			return false
 		}

parser/pageparser/pageparser_shortcode_test.go

@@ -16,22 +16,26 @@ package pageparser
 import "testing"
 
 var (
-	tstEOF       = nti(tEOF, "")
-	tstLeftNoMD  = nti(tLeftDelimScNoMarkup, "{{<")
-	tstRightNoMD = nti(tRightDelimScNoMarkup, ">}}")
-	tstLeftMD    = nti(tLeftDelimScWithMarkup, "{{%")
-	tstRightMD   = nti(tRightDelimScWithMarkup, "%}}")
-	tstSCClose   = nti(tScClose, "/")
-	tstSC1       = nti(tScName, "sc1")
-	tstSC1Inline = nti(tScNameInline, "sc1.inline")
-	tstSC2Inline = nti(tScNameInline, "sc2.inline")
-	tstSC2       = nti(tScName, "sc2")
-	tstSC3       = nti(tScName, "sc3")
-	tstSCSlash   = nti(tScName, "sc/sub")
-	tstParam1    = nti(tScParam, "param1")
-	tstParam2    = nti(tScParam, "param2")
-	tstVal       = nti(tScParamVal, "Hello World")
-	tstText      = nti(tText, "Hello World")
+	tstEOF            = nti(tEOF, "")
+	tstLeftNoMD       = nti(tLeftDelimScNoMarkup, "{{<")
+	tstRightNoMD      = nti(tRightDelimScNoMarkup, ">}}")
+	tstLeftMD         = nti(tLeftDelimScWithMarkup, "{{%")
+	tstRightMD        = nti(tRightDelimScWithMarkup, "%}}")
+	tstSCClose        = nti(tScClose, "/")
+	tstSC1            = nti(tScName, "sc1")
+	tstSC1Inline      = nti(tScNameInline, "sc1.inline")
+	tstSC2Inline      = nti(tScNameInline, "sc2.inline")
+	tstSC2            = nti(tScName, "sc2")
+	tstSC3            = nti(tScName, "sc3")
+	tstSCSlash        = nti(tScName, "sc/sub")
+	tstParam1         = nti(tScParam, "param1")
+	tstParam2         = nti(tScParam, "param2")
+	tstParamBoolTrue  = nti(tScParam, "true")
+	tstParamBoolFalse = nti(tScParam, "false")
+	tstParamInt       = nti(tScParam, "32")
+	tstParamFloat     = nti(tScParam, "3.14")
+	tstVal            = nti(tScParamVal, "Hello World")
+	tstText           = nti(tText, "Hello World")
 )
 
 var shortCodeLexerTests = []lexerTest{
@@ -69,6 +73,12 @@ var shortCodeLexerTests = []lexerTest{
 	{"close with extra keyword", `{{< sc1 >}}{{< /sc1 keyword>}}`, []Item{
 		tstLeftNoMD, tstSC1, tstRightNoMD, tstLeftNoMD, tstSCClose, tstSC1,
 		nti(tError, "unclosed shortcode")}},
+	{"float param, positional", `{{< sc1 3.14 >}}`, []Item{
+		tstLeftNoMD, tstSC1, nti(tScParam, "3.14"), tstRightNoMD, tstEOF}},
+	{"float param, named", `{{< sc1 param1=3.14 >}}`, []Item{
+		tstLeftNoMD, tstSC1, tstParam1, nti(tScParamVal, "3.14"), tstRightNoMD, tstEOF}},
+	{"float param, named, space before", `{{< sc1 param1= 3.14 >}}`, []Item{
+		tstLeftNoMD, tstSC1, tstParam1, nti(tScParamVal, "3.14"), tstRightNoMD, tstEOF}},
 	{"Youtube id", `{{< sc1 -ziL-Q_456igdO-4 >}}`, []Item{
 		tstLeftNoMD, tstSC1, nti(tScParam, "-ziL-Q_456igdO-4"), tstRightNoMD, tstEOF}},
 	{"non-alphanumerics param quoted", `{{< sc1 "-ziL-.%QigdO-4" >}}`, []Item{