tpl/transform: Add transform.Unmarshal func

Fixes #5428

tpl/transform/init.go

@@ -95,6 +95,14 @@ func init() {
 			},
 		)
 
+		ns.AddMethodMapping(ctx.Unmarshal,
+			[]string{"unmarshal"},
+			[][2]string{
+				{`{{ "hello = \"Hello World\"" | transform.Unmarshal }}`, "map[hello:Hello World]"},
+				{`{{ "hello = \"Hello World\"" | resources.FromString "data/greetings.toml" | transform.Unmarshal }}`, "map[hello:Hello World]"},
+			},
+		)
+
 		return ns
 
 	}

tpl/transform/remarshal.go

@@ -2,9 +2,10 @@ package transform
 
 import (
 	"bytes"
-	"errors"
 	"strings"
 
+	"github.com/pkg/errors"
+
 	"github.com/gohugoio/hugo/parser"
 	"github.com/gohugoio/hugo/parser/metadecoders"
 	"github.com/spf13/cast"
@@ -34,9 +35,9 @@ func (ns *Namespace) Remarshal(format string, data interface{}) (string, error)
 		return "", err
 	}
 
-	fromFormat, err := detectFormat(from)
-	if err != nil {
-		return "", err
+	fromFormat := metadecoders.FormatFromContentString(from)
+	if fromFormat == "" {
+		return "", errors.New("failed to detect format from content")
 	}
 
 	meta, err := metadecoders.UnmarshalToMap([]byte(from), fromFormat)
@@ -56,24 +57,3 @@ func toFormatMark(format string) (metadecoders.Format, error) {
 
 	return "", errors.New("failed to detect target data serialization format")
 }
-
-func detectFormat(data string) (metadecoders.Format, error) {
-	jsonIdx := strings.Index(data, "{")
-	yamlIdx := strings.Index(data, ":")
-	tomlIdx := strings.Index(data, "=")
-
-	if jsonIdx != -1 && (yamlIdx == -1 || jsonIdx < yamlIdx) && (tomlIdx == -1 || jsonIdx < tomlIdx) {
-		return metadecoders.JSON, nil
-	}
-
-	if yamlIdx != -1 && (tomlIdx == -1 || yamlIdx < tomlIdx) {
-		return metadecoders.YAML, nil
-	}
-
-	if tomlIdx != -1 {
-		return metadecoders.TOML, nil
-	}
-
-	return "", errors.New("failed to detect data serialization format")
-
-}

tpl/transform/remarshal_test.go

@@ -18,7 +18,6 @@ import (
 	"testing"
 
 	"github.com/gohugoio/hugo/helpers"
-	"github.com/gohugoio/hugo/parser/metadecoders"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
 )
@@ -171,34 +170,3 @@ func TestTestRemarshalError(t *testing.T) {
 	assert.Error(err)
 
 }
-
-func TestRemarshalDetectFormat(t *testing.T) {
-	t.Parallel()
-	assert := require.New(t)
-
-	for i, test := range []struct {
-		data   string
-		expect interface{}
-	}{
-		{`foo = "bar"`, metadecoders.TOML},
-		{`   foo = "bar"`, metadecoders.TOML},
-		{`foo="bar"`, metadecoders.TOML},
-		{`foo: "bar"`, metadecoders.YAML},
-		{`foo:"bar"`, metadecoders.YAML},
-		{`{ "foo": "bar"`, metadecoders.JSON},
-		{`asdfasdf`, false},
-		{``, false},
-	} {
-		errMsg := fmt.Sprintf("[%d] %s", i, test.data)
-
-		result, err := detectFormat(test.data)
-
-		if b, ok := test.expect.(bool); ok && !b {
-			assert.Error(err, errMsg)
-			continue
-		}
-
-		assert.NoError(err, errMsg)
-		assert.Equal(test.expect, result)
-	}
-}

tpl/transform/transform.go

@@ -19,6 +19,8 @@ import (
 	"html"
 	"html/template"
 
+	"github.com/gohugoio/hugo/cache/namedmemcache"
+
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/spf13/cast"
@@ -26,14 +28,22 @@ import (
 
 // New returns a new instance of the transform-namespaced template functions.
 func New(deps *deps.Deps) *Namespace {
+	cache := namedmemcache.New()
+	deps.BuildStartListeners.Add(
+		func() {
+			cache.Clear()
+		})
+
 	return &Namespace{
-		deps: deps,
+		cache: cache,
+		deps:  deps,
 	}
 }
 
 // Namespace provides template functions for the "transform" namespace.
 type Namespace struct {
-	deps *deps.Deps
+	cache *namedmemcache.Cache
+	deps  *deps.Deps
 }
 
 // Emojify returns a copy of s with all emoji codes replaced with actual emojis.

tpl/transform/transform_test.go

@@ -34,7 +34,6 @@ func TestEmojify(t *testing.T) {
 	t.Parallel()
 
 	v := viper.New()
-	v.Set("contentDir", "content")
 	ns := New(newDeps(v))
 
 	for i, test := range []struct {
@@ -215,7 +214,6 @@ func TestPlainify(t *testing.T) {
 	t.Parallel()
 
 	v := viper.New()
-	v.Set("contentDir", "content")
 	ns := New(newDeps(v))
 
 	for i, test := range []struct {
@@ -241,8 +239,11 @@ func TestPlainify(t *testing.T) {
 }
 
 func newDeps(cfg config.Provider) *deps.Deps {
+	cfg.Set("contentDir", "content")
+	cfg.Set("i18nDir", "i18n")
+
 	l := langs.NewLanguage("en", cfg)
-	l.Set("i18nDir", "i18n")
+
 	cs, err := helpers.NewContentSpec(l)
 	if err != nil {
 		panic(err)

tpl/transform/unmarshal.go

@@ -0,0 +1,98 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transform
+
+import (
+	"io/ioutil"
+
+	"github.com/gohugoio/hugo/common/hugio"
+
+	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/parser/metadecoders"
+	"github.com/gohugoio/hugo/resource"
+	"github.com/pkg/errors"
+
+	"github.com/spf13/cast"
+)
+
+// Unmarshal unmarshals the data given, which can be either a string
+// or a Resource. Supported formats are JSON, TOML and YAML.
+func (ns *Namespace) Unmarshal(data interface{}) (interface{}, error) {
+
+	// All the relevant Resource types implements ReadSeekCloserResource,
+	// which should be the most effective way to get the content.
+	if r, ok := data.(resource.ReadSeekCloserResource); ok {
+		var key string
+		var reader hugio.ReadSeekCloser
+
+		if k, ok := r.(resource.Identifier); ok {
+			key = k.Key()
+		}
+
+		if key == "" {
+			reader, err := r.ReadSeekCloser()
+			if err != nil {
+				return nil, err
+			}
+			defer reader.Close()
+
+			key, err = helpers.MD5FromReader(reader)
+			if err != nil {
+				return nil, err
+			}
+
+			reader.Seek(0, 0)
+		}
+
+		return ns.cache.GetOrCreate(key, func() (interface{}, error) {
+			f := metadecoders.FormatFromMediaType(r.MediaType())
+			if f == "" {
+				return nil, errors.Errorf("MIME %q not supported", r.MediaType())
+			}
+
+			if reader == nil {
+				var err error
+				reader, err = r.ReadSeekCloser()
+				if err != nil {
+					return nil, err
+				}
+				defer reader.Close()
+			}
+
+			b, err := ioutil.ReadAll(reader)
+			if err != nil {
+				return nil, err
+			}
+
+			return metadecoders.Unmarshal(b, f)
+		})
+
+	}
+
+	dataStr, err := cast.ToStringE(data)
+	if err != nil {
+		return nil, errors.Errorf("type %T not supported", data)
+	}
+
+	key := helpers.MD5String(dataStr)
+
+	return ns.cache.GetOrCreate(key, func() (interface{}, error) {
+		f := metadecoders.FormatFromContentString(dataStr)
+		if f == "" {
+			return nil, errors.New("unknown format")
+		}
+
+		return metadecoders.Unmarshal([]byte(dataStr), f)
+	})
+}

tpl/transform/unmarshal_test.go

@@ -0,0 +1,185 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transform
+
+import (
+	"fmt"
+	"math/rand"
+	"strings"
+	"testing"
+
+	"github.com/gohugoio/hugo/common/hugio"
+
+	"github.com/gohugoio/hugo/media"
+
+	"github.com/gohugoio/hugo/resource"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	testJSON = `
+	
+{
+    "ROOT_KEY": {
+        "title": "example glossary",
+		"GlossDiv": {
+            "title": "S",
+			"GlossList": {
+                "GlossEntry": {
+                    "ID": "SGML",
+					"SortAs": "SGML",
+					"GlossTerm": "Standard Generalized Markup Language",
+					"Acronym": "SGML",
+					"Abbrev": "ISO 8879:1986",
+					"GlossDef": {
+                        "para": "A meta-markup language, used to create markup languages such as DocBook.",
+						"GlossSeeAlso": ["GML", "XML"]
+                    },
+					"GlossSee": "markup"
+                }
+            }
+        }
+    }
+}
+
+	`
+)
+
+var _ resource.ReadSeekCloserResource = (*testContentResource)(nil)
+
+type testContentResource struct {
+	content string
+	mime    media.Type
+
+	key string
+}
+
+func (t testContentResource) ReadSeekCloser() (hugio.ReadSeekCloser, error) {
+	return hugio.NewReadSeekerNoOpCloserFromString(t.content), nil
+}
+
+func (t testContentResource) MediaType() media.Type {
+	return t.mime
+}
+
+func (t testContentResource) Key() string {
+	return t.key
+}
+
+func TestUnmarshal(t *testing.T) {
+
+	v := viper.New()
+	ns := New(newDeps(v))
+	assert := require.New(t)
+
+	assertSlogan := func(m map[string]interface{}) {
+		assert.Equal("Hugo Rocks!", m["slogan"])
+	}
+
+	for i, test := range []struct {
+		data   interface{}
+		expect interface{}
+	}{
+		{`{ "slogan": "Hugo Rocks!" }`, func(m map[string]interface{}) {
+			assertSlogan(m)
+		}},
+		{`slogan: "Hugo Rocks!"`, func(m map[string]interface{}) {
+			assertSlogan(m)
+		}},
+		{`slogan = "Hugo Rocks!"`, func(m map[string]interface{}) {
+			assertSlogan(m)
+		}},
+		{testContentResource{content: `slogan: "Hugo Rocks!"`, mime: media.YAMLType}, func(m map[string]interface{}) {
+			assertSlogan(m)
+		}},
+		{testContentResource{content: `{ "slogan": "Hugo Rocks!" }`, mime: media.JSONType}, func(m map[string]interface{}) {
+			assertSlogan(m)
+		}},
+		{testContentResource{content: `slogan = "Hugo Rocks!"`, mime: media.TOMLType}, func(m map[string]interface{}) {
+			assertSlogan(m)
+		}},
+		// errors
+		{"thisisnotavaliddataformat", false},
+		{testContentResource{content: `invalid&toml"`, mime: media.TOMLType}, false},
+		{testContentResource{content: `unsupported: MIME"`, mime: media.CalendarType}, false},
+		{"thisisnotavaliddataformat", false},
+		{`{ notjson }`, false},
+		{tstNoStringer{}, false},
+	} {
+		errMsg := fmt.Sprintf("[%d]", i)
+
+		result, err := ns.Unmarshal(test.data)
+
+		if b, ok := test.expect.(bool); ok && !b {
+			assert.Error(err, errMsg)
+		} else if fn, ok := test.expect.(func(m map[string]interface{})); ok {
+			assert.NoError(err, errMsg)
+			m, ok := result.(map[string]interface{})
+			assert.True(ok, errMsg)
+			fn(m)
+		} else {
+			assert.NoError(err, errMsg)
+			assert.Equal(test.expect, result, errMsg)
+		}
+
+	}
+}
+
+func BenchmarkUnmarshalString(b *testing.B) {
+	v := viper.New()
+	ns := New(newDeps(v))
+
+	const numJsons = 100
+
+	var jsons [numJsons]string
+	for i := 0; i < numJsons; i++ {
+		jsons[i] = strings.Replace(testJSON, "ROOT_KEY", fmt.Sprintf("root%d", i), 1)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		result, err := ns.Unmarshal(jsons[rand.Intn(numJsons)])
+		if err != nil {
+			b.Fatal(err)
+		}
+		if result == nil {
+			b.Fatal("no result")
+		}
+	}
+}
+
+func BenchmarkUnmarshalResource(b *testing.B) {
+	v := viper.New()
+	ns := New(newDeps(v))
+
+	const numJsons = 100
+
+	var jsons [numJsons]testContentResource
+	for i := 0; i < numJsons; i++ {
+		key := fmt.Sprintf("root%d", i)
+		jsons[i] = testContentResource{key: key, content: strings.Replace(testJSON, "ROOT_KEY", key, 1), mime: media.JSONType}
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		result, err := ns.Unmarshal(jsons[rand.Intn(numJsons)])
+		if err != nil {
+			b.Fatal(err)
+		}
+		if result == nil {
+			b.Fatal("no result")
+		}
+	}
+}