Add CSV support to transform.Unmarshal

Fixes #5555

tpl/transform/remarshal.go

@@ -35,12 +35,12 @@ func (ns *Namespace) Remarshal(format string, data interface{}) (string, error)
 		return "", err
 	}
 
-	fromFormat := metadecoders.FormatFromContentString(from)
+	fromFormat := metadecoders.Default.FormatFromContentString(from)
 	if fromFormat == "" {
 		return "", errors.New("failed to detect format from content")
 	}
 
-	meta, err := metadecoders.UnmarshalToMap([]byte(from), fromFormat)
+	meta, err := metadecoders.Default.UnmarshalToMap([]byte(from), fromFormat)
 
 	var result bytes.Buffer
 	if err := parser.InterfaceToConfig(meta, mark, &result); err != nil {

tpl/transform/unmarshal.go

@@ -15,8 +15,10 @@ package transform
 
 import (
 	"io/ioutil"
+	"strings"
 
 	"github.com/gohugoio/hugo/common/hugio"
+	"github.com/mitchellh/mapstructure"
 
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/parser/metadecoders"
@@ -27,8 +29,33 @@ import (
 )
 
 // Unmarshal unmarshals the data given, which can be either a string
-// or a Resource. Supported formats are JSON, TOML and YAML.
-func (ns *Namespace) Unmarshal(data interface{}) (interface{}, error) {
+// or a Resource. Supported formats are JSON, TOML, YAML, and CSV.
+// You can optional provide an Options object as the first argument.
+func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) {
+	if len(args) < 1 || len(args) > 2 {
+		return nil, errors.New("unmarshal takes 1 or 2 arguments")
+	}
+
+	var data interface{}
+	var decoder = metadecoders.Default
+
+	if len(args) == 1 {
+		data = args[0]
+	} else {
+		m, ok := args[0].(map[string]interface{})
+		if !ok {
+			return nil, errors.New("first argument must be a map")
+		}
+
+		var err error
+
+		data = args[1]
+		decoder, err = decodeDecoder(m)
+		if err != nil {
+			return nil, errors.WithMessage(err, "failed to decode options")
+		}
+
+	}
 
 	// All the relevant Resource types implements ReadSeekCloserResource,
 	// which should be the most effective way to get the content.
@@ -75,7 +102,7 @@ func (ns *Namespace) Unmarshal(data interface{}) (interface{}, error) {
 				return nil, err
 			}
 
-			return metadecoders.Unmarshal(b, f)
+			return decoder.Unmarshal(b, f)
 		})
 
 	}
@@ -88,11 +115,67 @@ func (ns *Namespace) Unmarshal(data interface{}) (interface{}, error) {
 	key := helpers.MD5String(dataStr)
 
 	return ns.cache.GetOrCreate(key, func() (interface{}, error) {
-		f := metadecoders.FormatFromContentString(dataStr)
+		f := decoder.FormatFromContentString(dataStr)
 		if f == "" {
 			return nil, errors.New("unknown format")
 		}
 
-		return metadecoders.Unmarshal([]byte(dataStr), f)
+		return decoder.Unmarshal([]byte(dataStr), f)
 	})
 }
+
+func decodeDecoder(m map[string]interface{}) (metadecoders.Decoder, error) {
+	opts := metadecoders.Default
+
+	if m == nil {
+		return opts, nil
+	}
+
+	// mapstructure does not support string to rune conversion, so do that manually.
+	// See https://github.com/mitchellh/mapstructure/issues/151
+	for k, v := range m {
+		if strings.EqualFold(k, "Comma") {
+			r, err := stringToRune(v)
+			if err != nil {
+				return opts, err
+			}
+			opts.Comma = r
+			delete(m, k)
+
+		} else if strings.EqualFold(k, "Comment") {
+			r, err := stringToRune(v)
+			if err != nil {
+				return opts, err
+			}
+			opts.Comment = r
+			delete(m, k)
+		}
+	}
+
+	err := mapstructure.WeakDecode(m, &opts)
+
+	return opts, err
+}
+
+func stringToRune(v interface{}) (rune, error) {
+	s, err := cast.ToStringE(v)
+	if err != nil {
+		return 0, err
+	}
+
+	if len(s) == 0 {
+		return 0, nil
+	}
+
+	var r rune
+
+	for i, rr := range s {
+		if i == 0 {
+			r = rr
+		} else {
+			return 0, errors.Errorf("invalid character: %q", v)
+		}
+	}
+
+	return r, nil
+}

tpl/transform/unmarshal_test.go

@@ -89,38 +89,74 @@ func TestUnmarshal(t *testing.T) {
 	}
 
 	for i, test := range []struct {
-		data   interface{}
-		expect interface{}
+		data    interface{}
+		options interface{}
+		expect  interface{}
 	}{
-		{`{ "slogan": "Hugo Rocks!" }`, func(m map[string]interface{}) {
+		{`{ "slogan": "Hugo Rocks!" }`, nil, func(m map[string]interface{}) {
 			assertSlogan(m)
 		}},
-		{`slogan: "Hugo Rocks!"`, func(m map[string]interface{}) {
+		{`slogan: "Hugo Rocks!"`, nil, func(m map[string]interface{}) {
 			assertSlogan(m)
 		}},
-		{`slogan = "Hugo Rocks!"`, func(m map[string]interface{}) {
+		{`slogan = "Hugo Rocks!"`, nil, func(m map[string]interface{}) {
 			assertSlogan(m)
 		}},
-		{testContentResource{content: `slogan: "Hugo Rocks!"`, mime: media.YAMLType}, func(m map[string]interface{}) {
+		{testContentResource{content: `slogan: "Hugo Rocks!"`, mime: media.YAMLType}, nil, func(m map[string]interface{}) {
 			assertSlogan(m)
 		}},
-		{testContentResource{content: `{ "slogan": "Hugo Rocks!" }`, mime: media.JSONType}, func(m map[string]interface{}) {
+		{testContentResource{content: `{ "slogan": "Hugo Rocks!" }`, mime: media.JSONType}, nil, func(m map[string]interface{}) {
 			assertSlogan(m)
 		}},
-		{testContentResource{content: `slogan = "Hugo Rocks!"`, mime: media.TOMLType}, func(m map[string]interface{}) {
+		{testContentResource{content: `slogan = "Hugo Rocks!"`, mime: media.TOMLType}, nil, func(m map[string]interface{}) {
 			assertSlogan(m)
 		}},
+		{testContentResource{content: `1997,Ford,E350,"ac, abs, moon",3000.00
+1999,Chevy,"Venture ""Extended Edition""","",4900.00`, mime: media.CSVType}, nil, func(r [][]string) {
+			assert.Equal(2, len(r))
+			first := r[0]
+			assert.Equal(5, len(first))
+			assert.Equal("Ford", first[1])
+		}},
+		{testContentResource{content: `a;b;c`, mime: media.CSVType}, map[string]interface{}{"comma": ";"}, func(r [][]string) {
+			assert.Equal(r, [][]string{[]string{"a", "b", "c"}})
+
+		}},
+		{"a,b,c", nil, func(r [][]string) {
+			assert.Equal(r, [][]string{[]string{"a", "b", "c"}})
+
+		}},
+		{"a;b;c", map[string]interface{}{"comma": ";"}, func(r [][]string) {
+			assert.Equal(r, [][]string{[]string{"a", "b", "c"}})
+
+		}},
+		{testContentResource{content: `
+% This is a comment
+a;b;c`, mime: media.CSVType}, map[string]interface{}{"CommA": ";", "Comment": "%"}, func(r [][]string) {
+			assert.Equal(r, [][]string{[]string{"a", "b", "c"}})
+
+		}},
 		// errors
-		{"thisisnotavaliddataformat", false},
-		{testContentResource{content: `invalid&toml"`, mime: media.TOMLType}, false},
-		{testContentResource{content: `unsupported: MIME"`, mime: media.CalendarType}, false},
-		{"thisisnotavaliddataformat", false},
-		{`{ notjson }`, false},
-		{tstNoStringer{}, false},
+		{"thisisnotavaliddataformat", nil, false},
+		{testContentResource{content: `invalid&toml"`, mime: media.TOMLType}, nil, false},
+		{testContentResource{content: `unsupported: MIME"`, mime: media.CalendarType}, nil, false},
+		{"thisisnotavaliddataformat", nil, false},
+		{`{ notjson }`, nil, false},
+		{tstNoStringer{}, nil, false},
 	} {
 		errMsg := fmt.Sprintf("[%d]", i)
 
-		result, err := ns.Unmarshal(test.data)
+		ns.cache.Clear()
+
+		var args []interface{}
+
+		if test.options != nil {
+			args = []interface{}{test.options, test.data}
+		} else {
+			args = []interface{}{test.data}
+		}
+
+		result, err := ns.Unmarshal(args...)
 
 		if b, ok := test.expect.(bool); ok && !b {
 			assert.Error(err, errMsg)
@@ -129,6 +165,11 @@ func TestUnmarshal(t *testing.T) {
 			m, ok := result.(map[string]interface{})
 			assert.True(ok, errMsg)
 			fn(m)
+		} else if fn, ok := test.expect.(func(r [][]string)); ok {
+			assert.NoError(err, errMsg)
+			r, ok := result.([][]string)
+			assert.True(ok, errMsg)
+			fn(r)
 		} else {
 			assert.NoError(err, errMsg)
 			assert.Equal(test.expect, result, errMsg)