mirror of
https://github.com/gohugoio/hugo.git
synced 2025-08-30 22:39:58 +02:00
Merge to stay up to date with current dev
Feature: GetJson and GetJson in short codes or other layout files. For more details please see: http://cyrillschumacher.com/2014/12/21/dynamic-pages-with-gohugo.io/
This commit is contained in:
Cyrill Schumacher
spf13
220
tpl/template_resources.go
Normal file
220
tpl/template_resources.go
Normal file
@@ -0,0 +1,220 @@
|
||||
// Copyright © 2013-14 Steve Francia <spf@spf13.com>.
|
||||
//
|
||||
// Licensed under the Simple Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://opensource.org/licenses/Simple-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpl
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/spf13/hugo/helpers"
|
||||
"github.com/spf13/hugo/hugofs"
|
||||
jww "github.com/spf13/jwalterweatherman"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
var remoteUrlLock = &remoteLock{m: make(map[string]*sync.Mutex)}
|
||||
|
||||
type remoteLock struct {
|
||||
sync.RWMutex
|
||||
m map[string]*sync.Mutex
|
||||
}
|
||||
|
||||
// resLock locks an URL during download
|
||||
func (l *remoteLock) UrlLock(url string) {
|
||||
l.Lock()
|
||||
if _, ok := l.m[url]; !ok {
|
||||
l.m[url] = &sync.Mutex{}
|
||||
}
|
||||
l.Unlock() // call this Unlock before the next lock will be called. NFI why but defer doesn't work.
|
||||
l.m[url].Lock()
|
||||
}
|
||||
|
||||
// resUnlock unlocks an URL when the download has been finished. Use only in defer calls.
|
||||
func (l *remoteLock) UrlUnlock(url string) {
|
||||
l.RLock()
|
||||
defer l.RUnlock()
|
||||
if um, ok := l.m[url]; ok {
|
||||
um.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// getFileID returns the cache ID for a string
|
||||
func getCacheFileID(id string) string {
|
||||
return viper.GetString("CacheDir") + url.QueryEscape(id)
|
||||
}
|
||||
|
||||
// resGetCache returns the content for an ID from the file cache or an error
|
||||
// if the file is not found returns nil,nil
|
||||
func resGetCache(id string, fs afero.Fs) ([]byte, error) {
|
||||
fID := getCacheFileID(id)
|
||||
isExists, err := helpers.Exists(fID, fs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !isExists {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
f, err := fs.Open(fID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ioutil.ReadAll(f)
|
||||
}
|
||||
|
||||
// resWriteCache writes bytes to an ID into the file cache
|
||||
func resWriteCache(id string, c []byte, fs afero.Fs) error {
|
||||
fID := getCacheFileID(id)
|
||||
f, err := fs.Create(fID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := f.Write(c)
|
||||
if n == 0 {
|
||||
return errors.New("No bytes written to file: " + fID)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// resGetRemote loads the content of a remote file. This method is thread safe.
|
||||
func resGetRemote(url string, fs afero.Fs, hc *http.Client) ([]byte, error) {
|
||||
|
||||
c, err := resGetCache(url, fs)
|
||||
if c != nil && err == nil {
|
||||
return c, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// avoid race condition with locks, block other goroutines if the current url is processing
|
||||
remoteUrlLock.UrlLock(url)
|
||||
defer func() { remoteUrlLock.UrlUnlock(url) }()
|
||||
|
||||
// avoid multiple locks due to calling resGetCache twice
|
||||
c, err = resGetCache(url, fs)
|
||||
if c != nil && err == nil {
|
||||
return c, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
jww.INFO.Printf("Downloading: %s ...", url)
|
||||
res, err := hc.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c, err = ioutil.ReadAll(res.Body)
|
||||
res.Body.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = resWriteCache(url, c, fs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
jww.INFO.Printf("... and cached to: %s", getCacheFileID(url))
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// resGetLocal loads the content of a local file
|
||||
func resGetLocal(url string, fs afero.Fs) ([]byte, error) {
|
||||
p := ""
|
||||
if viper.GetString("WorkingDir") != "" {
|
||||
p = viper.GetString("WorkingDir")
|
||||
if helpers.FilePathSeparator != p[len(p)-1:] {
|
||||
p = p + helpers.FilePathSeparator
|
||||
}
|
||||
}
|
||||
jFile := p + url
|
||||
if e, err := helpers.Exists(jFile, fs); !e {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
f, err := fs.Open(jFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ioutil.ReadAll(f)
|
||||
}
|
||||
|
||||
// resGetResource loads the content of a local or remote file
|
||||
func resGetResource(url string) ([]byte, error) {
|
||||
if url == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if strings.Contains(url, "://") {
|
||||
return resGetRemote(url, hugofs.SourceFs, http.DefaultClient)
|
||||
}
|
||||
return resGetLocal(url, hugofs.SourceFs)
|
||||
}
|
||||
|
||||
// GetJson expects the url to a resource which can either be a local or a remote one.
|
||||
// GetJson returns nil or parsed JSON to use in a short code.
|
||||
func GetJson(url string) interface{} {
|
||||
c, err := resGetResource(url)
|
||||
if err != nil {
|
||||
jww.ERROR.Printf("Failed to get json resource %s with error message %s", url, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
var v interface{}
|
||||
err = json.Unmarshal(c, &v)
|
||||
if err != nil {
|
||||
jww.ERROR.Printf("Cannot read json from resource %s with error message %s", url, err)
|
||||
return nil
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// parseCsv parses bytes of csv data into a slice slice string or an error
|
||||
func parseCsv(c []byte, sep string) ([][]string, error) {
|
||||
if len(sep) != 1 {
|
||||
return nil, errors.New("Incorrect length of csv separator: " + sep)
|
||||
}
|
||||
b := bytes.NewReader(c)
|
||||
r := csv.NewReader(b)
|
||||
rSep := []rune(sep)
|
||||
r.Comma = rSep[0]
|
||||
r.FieldsPerRecord = 0
|
||||
return r.ReadAll()
|
||||
}
|
||||
|
||||
// GetCsv expects the url to a resource which can either be a local or a remote one and the type
|
||||
// of the data separator which can be comma, semi-colon, pipe, but only one character.
|
||||
// GetCsv returns nil or a slice slice to use in a short code.
|
||||
func GetCsv(url string, sep string) [][]string {
|
||||
|
||||
c, err := resGetResource(url)
|
||||
if err != nil {
|
||||
jww.ERROR.Printf("Failed to get csv resource %s with error message %s", url, err)
|
||||
return nil
|
||||
}
|
||||
d, err := parseCsv(c, sep)
|
||||
if err != nil {
|
||||
jww.ERROR.Printf("Failed to read csv resource %s with error message %s", url, err)
|
||||
return nil
|
||||
}
|
||||
return d
|
||||
}
|
||||
181
tpl/template_resources_test.go
Normal file
181
tpl/template_resources_test.go
Normal file
@@ -0,0 +1,181 @@
|
||||
// Copyright © 2013-14 Steve Francia <spf@spf13.com>.
|
||||
//
|
||||
// Licensed under the Simple Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://opensource.org/licenses/Simple-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpl
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/spf13/hugo/helpers"
|
||||
)
|
||||
|
||||
func TestScpCache(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
path string
|
||||
content []byte
|
||||
}{
|
||||
{"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`)},
|
||||
{"fOO,bar:foo%bAR", []byte(`T€st Content 123 fOO,bar:foo%bAR`)},
|
||||
{"FOo/BaR.html", []byte(`FOo/BaR.html T€st Content 123`)},
|
||||
{"трям/трям", []byte(`T€st трям/трям Content 123`)},
|
||||
{"은행", []byte(`T€st C은행ontent 123`)},
|
||||
{"Банковский кассир", []byte(`Банковский кассир T€st Content 123`)},
|
||||
}
|
||||
|
||||
fs := new(afero.MemMapFs)
|
||||
|
||||
for _, test := range tests {
|
||||
c, err := resGetCache(test.path, fs)
|
||||
if err != nil {
|
||||
t.Errorf("Error getting cache: %s", err)
|
||||
}
|
||||
if c != nil {
|
||||
t.Errorf("There is content where there should not be anything: %s", string(c))
|
||||
}
|
||||
|
||||
err = resWriteCache(test.path, test.content, fs)
|
||||
if err != nil {
|
||||
t.Errorf("Error writing cache: %s", err)
|
||||
}
|
||||
|
||||
c, err = resGetCache(test.path, fs)
|
||||
if err != nil {
|
||||
t.Errorf("Error getting cache after writing: %s", err)
|
||||
}
|
||||
if bytes.Compare(c, test.content) != 0 {
|
||||
t.Errorf("\nExpected: %s\nActual: %s\n", string(test.content), string(c))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScpGetLocal(t *testing.T) {
|
||||
fs := new(afero.MemMapFs)
|
||||
ps := helpers.FilePathSeparator
|
||||
tests := []struct {
|
||||
path string
|
||||
content []byte
|
||||
}{
|
||||
{"testpath" + ps + "test.txt", []byte(`T€st Content 123 fOO,bar:foo%bAR`)},
|
||||
{"FOo" + ps + "BaR.html", []byte(`FOo/BaR.html T€st Content 123`)},
|
||||
{"трям" + ps + "трям", []byte(`T€st трям/трям Content 123`)},
|
||||
{"은행", []byte(`T€st C은행ontent 123`)},
|
||||
{"Банковский кассир", []byte(`Банковский кассир T€st Content 123`)},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
r := bytes.NewReader(test.content)
|
||||
err := helpers.WriteToDisk(test.path, r, fs)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
c, err := resGetLocal(test.path, fs)
|
||||
if err != nil {
|
||||
t.Errorf("Error getting resource content: %s", err)
|
||||
}
|
||||
if bytes.Compare(c, test.content) != 0 {
|
||||
t.Errorf("\nExpected: %s\nActual: %s\n", string(test.content), string(c))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func getTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*httptest.Server, *http.Client) {
|
||||
testServer := httptest.NewServer(http.HandlerFunc(handler))
|
||||
client := &http.Client{
|
||||
Transport: &http.Transport{Proxy: func(*http.Request) (*url.URL, error) { return url.Parse(testServer.URL) }},
|
||||
}
|
||||
return testServer, client
|
||||
}
|
||||
|
||||
func TestScpGetRemote(t *testing.T) {
|
||||
fs := new(afero.MemMapFs)
|
||||
|
||||
tests := []struct {
|
||||
path string
|
||||
content []byte
|
||||
}{
|
||||
{"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`)},
|
||||
{"http://Doppel.Gänger/foo_Bar-Foo", []byte(`T€st Cont€nt 123`)},
|
||||
{"http://Doppel.Gänger/Fizz_Bazz-Foo", []byte(`T€st Банковский кассир Cont€nt 123`)},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
|
||||
srv, cl := getTestServer(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(test.content)
|
||||
})
|
||||
defer func() { srv.Close() }()
|
||||
|
||||
c, err := resGetRemote(test.path, fs, cl)
|
||||
if err != nil {
|
||||
t.Errorf("Error getting resource content: %s", err)
|
||||
}
|
||||
if bytes.Compare(c, test.content) != 0 {
|
||||
t.Errorf("\nNet Expected: %s\nNet Actual: %s\n", string(test.content), string(c))
|
||||
}
|
||||
cc, cErr := resGetCache(test.path, fs)
|
||||
if cErr != nil {
|
||||
t.Error(cErr)
|
||||
}
|
||||
if bytes.Compare(cc, test.content) != 0 {
|
||||
t.Errorf("\nCache Expected: %s\nCache Actual: %s\n", string(test.content), string(c))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCsv(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
csv []byte
|
||||
sep string
|
||||
exp string
|
||||
err bool
|
||||
}{
|
||||
{[]byte("a,b,c\nd,e,f\n"), "", "", true},
|
||||
{[]byte("a,b,c\nd,e,f\n"), "~/", "", true},
|
||||
{[]byte("a,b,c\nd,e,f"), "|", "a,b,cd,e,f", false},
|
||||
{[]byte("q,w,e\nd,e,f"), ",", "qwedef", false},
|
||||
{[]byte("a|b|c\nd|e|f|g"), "|", "abcdefg", true},
|
||||
{[]byte("z|y|c\nd|e|f"), "|", "zycdef", false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
csv, err := parseCsv(test.csv, test.sep)
|
||||
if test.err && err == nil {
|
||||
t.Error("Expecting an error")
|
||||
}
|
||||
if test.err {
|
||||
continue
|
||||
}
|
||||
if !test.err && err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
act := ""
|
||||
for _, v := range csv {
|
||||
act = act + strings.Join(v, "")
|
||||
}
|
||||
|
||||
if act != test.exp {
|
||||
t.Errorf("\nExpected: %s\nActual: %s\n%#v\n", test.exp, act, csv)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user