mirror/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2025-08-16 20:53:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

parser/metadecoders: Consolidate the metadata decoders

Browse Source 
See #5324
This commit is contained in:
Bjørn Erik Pedersen
2018-10-20 11:16:18 +02:00
parent 44da60d869
commit 129c27ee6e
22 changed files with 624 additions and 808 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
parser/metadecoders/decoder.go
View File

@@ -15,81 +15,139 @@ package metadecoders
import (
"encoding/json"
"fmt"
"github.com/BurntSushi/toml"
"github.com/chaseadamsio/goorgeous"
"github.com/gohugoio/hugo/parser/pageparser"
"github.com/pkg/errors"
"github.com/spf13/cast"
yaml "gopkg.in/yaml.v2"
)
type Format string
const (
// These are the supported metdata formats in Hugo. Most of these are also
// supported as /data formats.
ORG Format = "org"
JSON Format = "json"
TOML Format = "toml"
YAML Format = "yaml"
)
// FormatFromFrontMatterType will return empty if not supported.
func FormatFromFrontMatterType(typ pageparser.ItemType) Format {
switch typ {
case pageparser.TypeFrontMatterJSON:
return JSON
case pageparser.TypeFrontMatterORG:
return ORG
case pageparser.TypeFrontMatterTOML:
return TOML
case pageparser.TypeFrontMatterYAML:
return YAML
default:
return ""
}
}
// UnmarshalToMap will unmarshall data in format f into a new map. This is
// what's needed for Hugo's front matter decoding.
func UnmarshalToMap(data []byte, f Format) (map[string]interface{}, error) {
m := make(map[string]interface{})
if data == nil {
return m, nil
}
err := unmarshal(data, f, &m)
return m, err
}
// Unmarshal will unmarshall data in format f into an interface{}.
// This is what's needed for Hugo's /data handling.
func Unmarshal(data []byte, f Format) (interface{}, error) {
if data == nil {
return make(map[string]interface{}), nil
}
var v interface{}
err := unmarshal(data, f, &v)
return v, err
}
// unmarshal unmarshals data in format f into v.
func unmarshal(data []byte, f Format, v interface{}) error {
var err error
switch f {
case ORG:
m, err = goorgeous.OrgHeaders(data)
vv, err := goorgeous.OrgHeaders(data)
if err != nil {
return err
}
switch v.(type) {
case *map[string]interface{}:
*v.(*map[string]interface{}) = vv
default:
*v.(*interface{}) = vv
}
case JSON:
err = json.Unmarshal(data, &m)
err = json.Unmarshal(data, v)
case TOML:
_, err = toml.Decode(string(data), &m)
err = toml.Unmarshal(data, v)
case YAML:
err = yaml.Unmarshal(data, &m)
err = yaml.Unmarshal(data, v)
// To support boolean keys, the `yaml` package unmarshals maps to
// To support boolean keys, the YAML package unmarshals maps to
// map[interface{}]interface{}. Here we recurse through the result
// and change all maps to map[string]interface{} like we would've
// gotten from `json`.
if err == nil {
for k, v := range m {
if vv, changed := stringifyMapKeys(v); changed {
m[k] = vv
}
var ptr interface{}
switch v.(type) {
case *map[string]interface{}:
ptr = *v.(*map[string]interface{})
case *interface{}:
ptr = *v.(*interface{})
default:
return errors.Errorf("unknown type %T in YAML unmarshal", v)
}
if mm, changed := stringifyMapKeys(ptr); changed {
switch v.(type) {
case *map[string]interface{}:
*v.(*map[string]interface{}) = mm.(map[string]interface{})
case *interface{}:
*v.(*interface{}) = mm
}
}
default:
return nil, errors.Errorf("unmarshal of format %q is not supported", f)
return errors.Errorf("unmarshal of format %q is not supported", f)
}
if err != nil {
return nil, errors.Wrapf(err, "unmarshal failed for format %q", f)
}
return m, nil
return err
}
// stringifyMapKeys recurses into in and changes all instances of
// map[interface{}]interface{} to map[string]interface{}. This is useful to
// work around the impedence mismatch between JSON and YAML unmarshaling that's
// described here: https://github.com/go-yaml/yaml/issues/139
//
// Inspired by https://github.com/stripe/stripe-mock, MIT licensed
func stringifyMapKeys(in interface{}) (interface{}, bool) {
switch in := in.(type) {
case []interface{}:
for i, v := range in {
if vv, replaced := stringifyMapKeys(v); replaced {
in[i] = vv
}
}
case map[string]interface{}:
for k, v := range in {
if vv, changed := stringifyMapKeys(v); changed {
in[k] = vv
}
}
case map[interface{}]interface{}:
res := make(map[string]interface{})
var (
ok bool
err error
)
for k, v := range in {
var ks string
if ks, ok = k.(string); !ok {
ks, err = cast.ToStringE(k)
if err != nil {
ks = fmt.Sprintf("%v", k)
}
}
if vv, replaced := stringifyMapKeys(v); replaced {
res[ks] = vv
} else {
res[ks] = v
}
}
return res, true
}
return nil, false
}

207
parser/metadecoders/decoder_test.go Normal file
View File

@@ -0,0 +1,207 @@
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package metadecoders
import (
"fmt"
"reflect"
"testing"
"github.com/stretchr/testify/require"
)
func TestUnmarshalToMap(t *testing.T) {
assert := require.New(t)
expect := map[string]interface{}{"a": "b"}
for i, test := range []struct {
data string
format Format
expect interface{}
}{
{`a = "b"`, TOML, expect},
{`a: "b"`, YAML, expect},
// Make sure we get all string keys, even for YAML
{"a: Easy!\nb:\n c: 2\n d: [3, 4]", YAML, map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}},
{"a:\n true: 1\n false: 2", YAML, map[string]interface{}{"a": map[string]interface{}{"true": 1, "false": 2}}},
{`{ "a": "b" }`, JSON, expect},
{`#+a: b`, ORG, expect},
// errors
{`a = b`, TOML, false},
} {
msg := fmt.Sprintf("%d: %s", i, test.format)
m, err := UnmarshalToMap([]byte(test.data), test.format)
if b, ok := test.expect.(bool); ok && !b {
assert.Error(err, msg)
} else {
assert.NoError(err, msg)
assert.Equal(test.expect, m, msg)
}
}
}
func TestUnmarshalToInterface(t *testing.T) {
assert := require.New(t)
expect := map[string]interface{}{"a": "b"}
for i, test := range []struct {
data string
format Format
expect interface{}
}{
{`[ "Brecker", "Blake", "Redman" ]`, JSON, []interface{}{"Brecker", "Blake", "Redman"}},
{`{ "a": "b" }`, JSON, expect},
{`#+a: b`, ORG, expect},
{`a = "b"`, TOML, expect},
{`a: "b"`, YAML, expect},
{"a: Easy!\nb:\n c: 2\n d: [3, 4]", YAML, map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}},
// errors
{`a = "`, TOML, false},
} {
msg := fmt.Sprintf("%d: %s", i, test.format)
m, err := Unmarshal([]byte(test.data), test.format)
if b, ok := test.expect.(bool); ok && !b {
assert.Error(err, msg)
} else {
assert.NoError(err, msg)
assert.Equal(test.expect, m, msg)
}
}
}
func TestStringifyYAMLMapKeys(t *testing.T) {
cases := []struct {
input interface{}
want interface{}
replaced bool
}{
{
map[interface{}]interface{}{"a": 1, "b": 2},
map[string]interface{}{"a": 1, "b": 2},
true,
},
{
map[interface{}]interface{}{"a": []interface{}{1, map[interface{}]interface{}{"b": 2}}},
map[string]interface{}{"a": []interface{}{1, map[string]interface{}{"b": 2}}},
true,
},
{
map[interface{}]interface{}{true: 1, "b": false},
map[string]interface{}{"true": 1, "b": false},
true,
},
{
map[interface{}]interface{}{1: "a", 2: "b"},
map[string]interface{}{"1": "a", "2": "b"},
true,
},
{
map[interface{}]interface{}{"a": map[interface{}]interface{}{"b": 1}},
map[string]interface{}{"a": map[string]interface{}{"b": 1}},
true,
},
{
map[string]interface{}{"a": map[string]interface{}{"b": 1}},
map[string]interface{}{"a": map[string]interface{}{"b": 1}},
false,
},
{
[]interface{}{map[interface{}]interface{}{1: "a", 2: "b"}},
[]interface{}{map[string]interface{}{"1": "a", "2": "b"}},
false,
},
}
for i, c := range cases {
res, replaced := stringifyMapKeys(c.input)
if c.replaced != replaced {
t.Fatalf("[%d] Replaced mismatch: %t", i, replaced)
}
if !c.replaced {
res = c.input
}
if !reflect.DeepEqual(res, c.want) {
t.Errorf("[%d] given %q\nwant: %q\n got: %q", i, c.input, c.want, res)
}
}
}
func BenchmarkStringifyMapKeysStringsOnlyInterfaceMaps(b *testing.B) {
maps := make([]map[interface{}]interface{}, b.N)
for i := 0; i < b.N; i++ {
maps[i] = map[interface{}]interface{}{
"a": map[interface{}]interface{}{
"b": 32,
"c": 43,
"d": map[interface{}]interface{}{
"b": 32,
"c": 43,
},
},
"b": []interface{}{"a", "b"},
"c": "d",
}
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
stringifyMapKeys(maps[i])
}
}
func BenchmarkStringifyMapKeysStringsOnlyStringMaps(b *testing.B) {
m := map[string]interface{}{
"a": map[string]interface{}{
"b": 32,
"c": 43,
"d": map[string]interface{}{
"b": 32,
"c": 43,
},
},
"b": []interface{}{"a", "b"},
"c": "d",
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
stringifyMapKeys(m)
}
}
func BenchmarkStringifyMapKeysIntegers(b *testing.B) {
maps := make([]map[interface{}]interface{}, b.N)
for i := 0; i < b.N; i++ {
maps[i] = map[interface{}]interface{}{
1: map[interface{}]interface{}{
4: 32,
5: 43,
6: map[interface{}]interface{}{
7: 32,
8: 43,
},
},
2: []interface{}{"a", "b"},
3: "d",
}
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
stringifyMapKeys(maps[i])
}
}

66
parser/metadecoders/format.go Normal file
View File

@@ -0,0 +1,66 @@
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package metadecoders
import (
"strings"
"github.com/gohugoio/hugo/parser/pageparser"
)
type Format string
const (
// These are the supported metdata formats in Hugo. Most of these are also
// supported as /data formats.
ORG Format = "org"
JSON Format = "json"
TOML Format = "toml"
YAML Format = "yaml"
)
// FormatFromString turns formatStr, typically a file extension without any ".",
// into a Format. It returns an empty string for unknown formats.
func FormatFromString(formatStr string) Format {
formatStr = strings.ToLower(formatStr)
switch formatStr {
case "yaml", "yml":
return YAML
case "json":
return JSON
case "toml":
return TOML
case "org":
return ORG
}
return ""
}
// FormatFromFrontMatterType will return empty if not supported.
func FormatFromFrontMatterType(typ pageparser.ItemType) Format {
switch typ {
case pageparser.TypeFrontMatterJSON:
return JSON
case pageparser.TypeFrontMatterORG:
return ORG
case pageparser.TypeFrontMatterTOML:
return TOML
case pageparser.TypeFrontMatterYAML:
return YAML
default:
return ""
}
}

57
parser/metadecoders/format_test.go Normal file
View File

@@ -0,0 +1,57 @@
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package metadecoders
import (
"fmt"
"testing"
"github.com/gohugoio/hugo/parser/pageparser"
"github.com/stretchr/testify/require"
)
func TestFormatFromString(t *testing.T) {
assert := require.New(t)
for i, test := range []struct {
s string
expect Format
}{
{"json", JSON},
{"yaml", YAML},
{"yml", YAML},
{"toml", TOML},
{"tOMl", TOML},
{"org", ORG},
{"foo", ""},
} {
assert.Equal(test.expect, FormatFromString(test.s), fmt.Sprintf("t%d", i))
}
}
func TestFormatFromFrontMatterType(t *testing.T) {
assert := require.New(t)
for i, test := range []struct {
typ pageparser.ItemType
expect Format
}{
{pageparser.TypeFrontMatterJSON, JSON},
{pageparser.TypeFrontMatterTOML, TOML},
{pageparser.TypeFrontMatterYAML, YAML},
{pageparser.TypeFrontMatterORG, ORG},
{pageparser.TypeIgnore, ""},
} {
assert.Equal(test.expect, FormatFromFrontMatterType(test.typ), fmt.Sprintf("t%d", i))
}
}

31
parser/metadecoders/json.go
View File

@@ -1,31 +0,0 @@
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package metadecoders
import "encoding/json"
// HandleJSONData unmarshals JSON-encoded datum and returns a Go interface
// representing the encoded data structure.
func HandleJSONData(datum []byte) (interface{}, error) {
if datum == nil {
// Package json returns on error on nil input.
// Return an empty map to be consistent with our other supported
// formats.
return make(map[string]interface{}), nil
}
var f interface{}
err := json.Unmarshal(datum, &f)
return f, err
}

84
parser/metadecoders/yaml.go
View File

@@ -1,84 +0,0 @@
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The metadecoders package contains functions to decode metadata (e.g. page front matter)
// from different formats: TOML, YAML, JSON.
package metadecoders
import (
"fmt"
"github.com/spf13/cast"
yaml "gopkg.in/yaml.v2"
)
// HandleYAMLData unmarshals YAML-encoded datum and returns a Go interface
// representing the encoded data structure.
func HandleYAMLData(datum []byte) (interface{}, error) {
var m interface{}
err := yaml.Unmarshal(datum, &m)
if err != nil {
return nil, err
}
// To support boolean keys, the `yaml` package unmarshals maps to
// map[interface{}]interface{}. Here we recurse through the result
// and change all maps to map[string]interface{} like we would've
// gotten from `json`.
if mm, changed := stringifyMapKeys(m); changed {
return mm, nil
}
return m, nil
}
// stringifyMapKeys recurses into in and changes all instances of
// map[interface{}]interface{} to map[string]interface{}. This is useful to
// work around the impedence mismatch between JSON and YAML unmarshaling that's
// described here: https://github.com/go-yaml/yaml/issues/139
//
// Inspired by https://github.com/stripe/stripe-mock, MIT licensed
func stringifyMapKeys(in interface{}) (interface{}, bool) {
switch in := in.(type) {
case []interface{}:
for i, v := range in {
if vv, replaced := stringifyMapKeys(v); replaced {
in[i] = vv
}
}
case map[interface{}]interface{}:
res := make(map[string]interface{})
var (
ok bool
err error
)
for k, v := range in {
var ks string
if ks, ok = k.(string); !ok {
ks, err = cast.ToStringE(k)
if err != nil {
ks = fmt.Sprintf("%v", k)
}
}
if vv, replaced := stringifyMapKeys(v); replaced {
res[ks] = vv
} else {
res[ks] = v
}
}
return res, true
}
return nil, false
}