mirror/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2025-08-26 22:04:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add readFile template func

Browse Source 
This also includes a refactor of the hugofs package and its usage.

The motivation for that is:

The Afero filesystems are brilliant. Hugo's way of adding a dozen of global variables for the different filesystems was a mistake. In readFile (and also in some other places in Hugo today) we need a way to restrict the access inside the working dir. We could use ioutil.ReadFile and implement the path checking, checking the base path and the dots ("..") etc. But it is obviously better to use an Afero BasePathFs combined witha ReadOnlyFs. We could create a use-once-filesystem and handle the initialization ourselves, but since this is also useful to others and the initialization depends on some other global state (which would mean to create a new file system on every invocation), we might as well do it properly and encapsulate the predefined set of filesystems. This change also leads the way, if needed, to encapsulate the file systems in a struct, making it possible to have several file system sets in action at once (parallel multilanguage site building? With Moore's law and all...)

Fixes #1551
This commit is contained in:
Bjørn Erik Pedersen
2016-03-22 00:28:42 +01:00
parent a89035bdaa
commit 4f66f790b1
32 changed files with 400 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
commands/gendoc.go
View File

@@ -1,4 +1,4 @@
// Copyright 2015 The Hugo Authors. All rights reserved.
// Copyright 2016 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -51,9 +51,9 @@ for rendering in Hugo.`,
if !strings.HasSuffix(gendocdir, helpers.FilePathSeparator) {
gendocdir += helpers.FilePathSeparator
}
if found, _ := helpers.Exists(gendocdir, hugofs.OsFs); !found {
if found, _ := helpers.Exists(gendocdir, hugofs.Os()); !found {
jww.FEEDBACK.Println("Directory", gendocdir, "does not exist, creating...")
hugofs.OsFs.MkdirAll(gendocdir, 0777)
hugofs.Os().MkdirAll(gendocdir, 0777)
}
now := time.Now().Format(time.RFC3339)
prepender := func(filename string) string {

6
commands/genman.go
View File

@@ -1,4 +1,4 @@
// Copyright 2015 The Hugo Authors. All rights reserved.
// Copyright 2016 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -41,9 +41,9 @@ in the "man" directory under the current directory.`,
if !strings.HasSuffix(genmandir, helpers.FilePathSeparator) {
genmandir += helpers.FilePathSeparator
}
if found, _ := helpers.Exists(genmandir, hugofs.OsFs); !found {
if found, _ := helpers.Exists(genmandir, hugofs.Os()); !found {
jww.FEEDBACK.Println("Directory", genmandir, "does not exist, creating...")
hugofs.OsFs.MkdirAll(genmandir, 0777)
hugofs.Os().MkdirAll(genmandir, 0777)
}
cmd.Root().DisableAutoGenTag = true

37
commands/hugo.go
View File

@@ -423,14 +423,14 @@ func InitializeConfig(subCmdVs ...*cobra.Command) error {
if helpers.FilePathSeparator != cacheDir[len(cacheDir)-1:] {
cacheDir = cacheDir + helpers.FilePathSeparator
}
isDir, err := helpers.DirExists(cacheDir, hugofs.SourceFs)
isDir, err := helpers.DirExists(cacheDir, hugofs.Source())
utils.CheckErr(err)
if isDir == false {
mkdir(cacheDir)
}
viper.Set("CacheDir", cacheDir)
} else {
viper.Set("CacheDir", helpers.GetTempDir("hugo_cache", hugofs.SourceFs))
viper.Set("CacheDir", helpers.GetTempDir("hugo_cache", hugofs.Source()))
}
if verboseLog || logging || (viper.IsSet("LogFile") && viper.GetString("LogFile") != "") {
@@ -453,6 +453,9 @@ func InitializeConfig(subCmdVs ...*cobra.Command) error {
jww.INFO.Println("Using config file:", viper.ConfigFileUsed())
// Init file systems. This may be changed at a later point.
hugofs.InitDefaultFs()
themeDir := helpers.GetThemeDir()
if themeDir != "" {
if _, err := os.Stat(themeDir); os.IsNotExist(err) {
@@ -498,7 +501,7 @@ func build(watches ...bool) error {
// This is only used for benchmark testing. Cause the content is only visible
// in memory
if renderToMemory {
hugofs.DestinationFS = new(afero.MemMapFs)
hugofs.SetDestination(new(afero.MemMapFs))
// Rendering to memoryFS, publish to Root regardless of publishDir.
viper.Set("PublishDir", "/")
}
@@ -524,7 +527,7 @@ func build(watches ...bool) error {
}
func getStaticSourceFs() afero.Fs {
source := hugofs.SourceFs
source := hugofs.Source()
themeDir, err := helpers.GetThemeStaticDirPath()
staticDir := helpers.GetStaticDirPath() + helpers.FilePathSeparator
@@ -563,8 +566,8 @@ func getStaticSourceFs() afero.Fs {
jww.INFO.Println("using a UnionFS for static directory comprised of:")
jww.INFO.Println("Base:", themeDir)
jww.INFO.Println("Overlay:", staticDir)
base := afero.NewReadOnlyFs(afero.NewBasePathFs(hugofs.SourceFs, themeDir))
overlay := afero.NewReadOnlyFs(afero.NewBasePathFs(hugofs.SourceFs, staticDir))
base := afero.NewReadOnlyFs(afero.NewBasePathFs(hugofs.Source(), themeDir))
overlay := afero.NewReadOnlyFs(afero.NewBasePathFs(hugofs.Source(), staticDir))
return afero.NewCopyOnWriteFs(base, overlay)
}
@@ -587,7 +590,7 @@ func copyStatic() error {
syncer := fsync.NewSyncer()
syncer.NoTimes = viper.GetBool("notimes")
syncer.SrcFs = staticSourceFs
syncer.DestFs = hugofs.DestinationFS
syncer.DestFs = hugofs.Destination()
// Now that we are using a unionFs for the static directories
// We can effectively clean the publishDir on initial sync
syncer.Delete = viper.GetBool("cleanDestinationDir")
@@ -653,12 +656,12 @@ func getDirList() []string {
return nil
}
helpers.SymbolicWalk(hugofs.SourceFs, dataDir, walker)
helpers.SymbolicWalk(hugofs.SourceFs, helpers.AbsPathify(viper.GetString("ContentDir")), walker)
helpers.SymbolicWalk(hugofs.SourceFs, helpers.AbsPathify(viper.GetString("LayoutDir")), walker)
helpers.SymbolicWalk(hugofs.SourceFs, helpers.AbsPathify(viper.GetString("StaticDir")), walker)
helpers.SymbolicWalk(hugofs.Source(), dataDir, walker)
helpers.SymbolicWalk(hugofs.Source(), helpers.AbsPathify(viper.GetString("ContentDir")), walker)
helpers.SymbolicWalk(hugofs.Source(), helpers.AbsPathify(viper.GetString("LayoutDir")), walker)
helpers.SymbolicWalk(hugofs.Source(), helpers.AbsPathify(viper.GetString("StaticDir")), walker)
if helpers.ThemeSet() {
helpers.SymbolicWalk(hugofs.SourceFs, helpers.AbsPathify(viper.GetString("themesDir")+"/"+viper.GetString("theme")), walker)
helpers.SymbolicWalk(hugofs.Source(), helpers.AbsPathify(viper.GetString("themesDir")+"/"+viper.GetString("theme")), walker)
}
return a
@@ -770,8 +773,8 @@ func NewWatcher(port int) error {
// recursively add new directories to watch list
// When mkdir -p is used, only the top directory triggers an event (at least on OSX)
if ev.Op&fsnotify.Create == fsnotify.Create {
if s, err := hugofs.SourceFs.Stat(ev.Name); err == nil && s.Mode().IsDir() {
helpers.SymbolicWalk(hugofs.SourceFs, ev.Name, walkAdder)
if s, err := hugofs.Source().Stat(ev.Name); err == nil && s.Mode().IsDir() {
helpers.SymbolicWalk(hugofs.Source(), ev.Name, walkAdder)
}
}
@@ -813,7 +816,7 @@ func NewWatcher(port int) error {
syncer := fsync.NewSyncer()
syncer.NoTimes = viper.GetBool("notimes")
syncer.SrcFs = staticSourceFs
syncer.DestFs = hugofs.DestinationFS
syncer.DestFs = hugofs.Destination()
// prevent spamming the log on changes
logger := helpers.NewDistinctFeedbackLogger()
@@ -858,7 +861,7 @@ func NewWatcher(port int) error {
// If file doesn't exist in any static dir, remove it
toRemove := filepath.Join(publishDir, relPath)
logger.Println("File no longer exists in static dir, removing", toRemove)
hugofs.DestinationFS.RemoveAll(toRemove)
hugofs.Destination().RemoveAll(toRemove)
} else if err == nil {
// If file still exists, sync it
logger.Println("Syncing", relPath, "to", publishDir)
@@ -939,7 +942,7 @@ func isThemeVsHugoVersionMismatch() (mismatch bool, requiredMinVersion string) {
themeDir := helpers.GetThemeDir()
fs := hugofs.SourceFs
fs := hugofs.Source()
path := filepath.Join(themeDir, "theme.toml")
exists, err := helpers.Exists(path, fs)

10
commands/import_jekyll.go
View File

@@ -1,4 +1,4 @@
// Copyright 2015 The Hugo Authors. All rights reserved.
// Copyright 2016 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -124,7 +124,7 @@ func importFromJekyll(cmd *cobra.Command, args []string) error {
return convertJekyllPost(path, relPath, targetDir, draft)
}
err = helpers.SymbolicWalk(hugofs.OsFs, jekyllRoot, callback)
err = helpers.SymbolicWalk(hugofs.Os(), jekyllRoot, callback)
if err != nil {
return err
@@ -139,7 +139,7 @@ func importFromJekyll(cmd *cobra.Command, args []string) error {
// TODO: Consider calling doNewSite() instead?
func createSiteFromJekyll(jekyllRoot, targetDir string, force bool) error {
fs := hugofs.SourceFs
fs := hugofs.Source()
if exists, _ := helpers.Exists(targetDir, fs); exists {
if isDir, _ := helpers.IsDir(targetDir, fs); !isDir {
return errors.New("Target path \"" + targetDir + "\" already exists but not a directory")
@@ -187,7 +187,7 @@ func createSiteFromJekyll(jekyllRoot, targetDir string, force bool) error {
}
func loadJekyllConfig(jekyllRoot string) map[string]interface{} {
fs := hugofs.SourceFs
fs := hugofs.Source()
path := filepath.Join(jekyllRoot, "_config.yml")
exists, err := helpers.Exists(path, fs)
@@ -252,7 +252,7 @@ func createConfigFromJekyll(inpath string, kind string, jekyllConfig map[string]
return err
}
err = helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), bytes.NewReader(by), hugofs.SourceFs)
err = helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), bytes.NewReader(by), hugofs.Source())
if err != nil {
return
}

26
commands/new.go
View File

@@ -1,4 +1,4 @@
// Copyright 2015 The Hugo Authors. All rights reserved.
// Copyright 2016 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -110,7 +110,7 @@ func NewContent(cmd *cobra.Command, args []string) error {
kind = contentType
}
return create.NewContent(hugofs.SourceFs, kind, createpath)
return create.NewContent(hugofs.Source(), kind, createpath)
}
func doNewSite(basepath string, force bool) error {
@@ -123,12 +123,12 @@ func doNewSite(basepath string, force bool) error {
filepath.Join(basepath, "themes"),
}
if exists, _ := helpers.Exists(basepath, hugofs.SourceFs); exists {
if isDir, _ := helpers.IsDir(basepath, hugofs.SourceFs); !isDir {
if exists, _ := helpers.Exists(basepath, hugofs.Source()); exists {
if isDir, _ := helpers.IsDir(basepath, hugofs.Source()); !isDir {
return errors.New(basepath + " already exists but not a directory")
}
isEmpty, _ := helpers.IsEmpty(basepath, hugofs.SourceFs)
isEmpty, _ := helpers.IsEmpty(basepath, hugofs.Source())
switch {
case !isEmpty && !force:
@@ -137,7 +137,7 @@ func doNewSite(basepath string, force bool) error {
case !isEmpty && force:
all := append(dirs, filepath.Join(basepath, "config."+configFormat))
for _, path := range all {
if exists, _ := helpers.Exists(path, hugofs.SourceFs); exists {
if exists, _ := helpers.Exists(path, hugofs.Source()); exists {
return errors.New(path + " already exists")
}
}
@@ -145,7 +145,7 @@ func doNewSite(basepath string, force bool) error {
}
for _, dir := range dirs {
hugofs.SourceFs.MkdirAll(dir, 0777)
hugofs.Source().MkdirAll(dir, 0777)
}
createConfig(basepath, configFormat)
@@ -185,7 +185,7 @@ func NewTheme(cmd *cobra.Command, args []string) error {
createpath := helpers.AbsPathify(filepath.Join(viper.GetString("themesDir"), args[0]))
jww.INFO.Println("creating theme at", createpath)
if x, _ := helpers.Exists(createpath, hugofs.SourceFs); x {
if x, _ := helpers.Exists(createpath, hugofs.Source()); x {
return newUserError(createpath, "already exists")
}
@@ -204,7 +204,7 @@ func NewTheme(cmd *cobra.Command, args []string) error {
archDefault := []byte("+++\n+++\n")
err := helpers.WriteToDisk(filepath.Join(createpath, "archetypes", "default.md"), bytes.NewReader(archDefault), hugofs.SourceFs)
err := helpers.WriteToDisk(filepath.Join(createpath, "archetypes", "default.md"), bytes.NewReader(archDefault), hugofs.Source())
if err != nil {
return err
}
@@ -234,7 +234,7 @@ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
`)
err = helpers.WriteToDisk(filepath.Join(createpath, "LICENSE.md"), bytes.NewReader(by), hugofs.SourceFs)
err = helpers.WriteToDisk(filepath.Join(createpath, "LICENSE.md"), bytes.NewReader(by), hugofs.Source())
if err != nil {
return err
}
@@ -256,7 +256,7 @@ func mkdir(x ...string) {
func touchFile(x ...string) {
inpath := filepath.Join(x...)
mkdir(filepath.Dir(inpath))
err := helpers.WriteToDisk(inpath, bytes.NewReader([]byte{}), hugofs.SourceFs)
err := helpers.WriteToDisk(inpath, bytes.NewReader([]byte{}), hugofs.Source())
if err != nil {
jww.FATAL.Fatalln(err)
}
@@ -287,7 +287,7 @@ min_version = 0.15
repo = ""
`)
err = helpers.WriteToDisk(filepath.Join(inpath, "theme.toml"), bytes.NewReader(by), hugofs.SourceFs)
err = helpers.WriteToDisk(filepath.Join(inpath, "theme.toml"), bytes.NewReader(by), hugofs.Source())
if err != nil {
return
}
@@ -321,7 +321,7 @@ func createConfig(inpath string, kind string) (err error) {
return err
}
err = helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), bytes.NewReader(by), hugofs.SourceFs)
err = helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), bytes.NewReader(by), hugofs.Source())
if err != nil {
return
}

31
commands/new_test.go
View File

@@ -1,4 +1,4 @@
// Copyright 2015 The Hugo Authors. All rights reserved.
// Copyright 2016 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -18,7 +18,6 @@ import (
"path/filepath"
"testing"
"github.com/spf13/afero"
"github.com/spf13/hugo/hugofs"
"github.com/stretchr/testify/assert"
)
@@ -41,14 +40,14 @@ func checkNewSiteInited(basepath string, t *testing.T) {
}
for _, path := range paths {
_, err := hugofs.SourceFs.Stat(path)
_, err := hugofs.Source().Stat(path)
assert.Nil(t, err)
}
}
func TestDoNewSite(t *testing.T) {
basepath := filepath.Join(os.TempDir(), "blog")
hugofs.SourceFs = new(afero.MemMapFs)
hugofs.InitMemFs()
err := doNewSite(basepath, false)
assert.Nil(t, err)
@@ -57,17 +56,17 @@ func TestDoNewSite(t *testing.T) {
func TestDoNewSite_noerror_base_exists_but_empty(t *testing.T) {
basepath := filepath.Join(os.TempDir(), "blog")
hugofs.SourceFs = new(afero.MemMapFs)
hugofs.SourceFs.MkdirAll(basepath, 777)
hugofs.InitMemFs()
hugofs.Source().MkdirAll(basepath, 777)
err := doNewSite(basepath, false)
assert.Nil(t, err)
}
func TestDoNewSite_error_base_exists(t *testing.T) {
basepath := filepath.Join(os.TempDir(), "blog")
hugofs.SourceFs = new(afero.MemMapFs)
hugofs.SourceFs.MkdirAll(basepath, 777)
hugofs.SourceFs.Create(filepath.Join(basepath, "foo"))
hugofs.InitMemFs()
hugofs.Source().MkdirAll(basepath, 777)
hugofs.Source().Create(filepath.Join(basepath, "foo"))
// Since the directory already exists and isn't empty, expect an error
err := doNewSite(basepath, false)
assert.NotNil(t, err)
@@ -75,8 +74,8 @@ func TestDoNewSite_error_base_exists(t *testing.T) {
func TestDoNewSite_force_empty_dir(t *testing.T) {
basepath := filepath.Join(os.TempDir(), "blog")
hugofs.SourceFs = new(afero.MemMapFs)
hugofs.SourceFs.MkdirAll(basepath, 777)
hugofs.InitMemFs()
hugofs.Source().MkdirAll(basepath, 777)
err := doNewSite(basepath, true)
assert.Nil(t, err)
@@ -86,8 +85,8 @@ func TestDoNewSite_force_empty_dir(t *testing.T) {
func TestDoNewSite_error_force_dir_inside_exists(t *testing.T) {
basepath := filepath.Join(os.TempDir(), "blog")
contentPath := filepath.Join(basepath, "content")
hugofs.SourceFs = new(afero.MemMapFs)
hugofs.SourceFs.MkdirAll(contentPath, 777)
hugofs.InitMemFs()
hugofs.Source().MkdirAll(contentPath, 777)
err := doNewSite(basepath, true)
assert.NotNil(t, err)
}
@@ -95,9 +94,9 @@ func TestDoNewSite_error_force_dir_inside_exists(t *testing.T) {
func TestDoNewSite_error_force_config_inside_exists(t *testing.T) {
basepath := filepath.Join(os.TempDir(), "blog")
configPath := filepath.Join(basepath, "config.toml")
hugofs.SourceFs = new(afero.MemMapFs)
hugofs.SourceFs.MkdirAll(basepath, 777)
hugofs.SourceFs.Create(configPath)
hugofs.InitMemFs()
hugofs.Source().MkdirAll(basepath, 777)
hugofs.Source().Create(configPath)
err := doNewSite(basepath, true)
assert.NotNil(t, err)
}

6
commands/server.go
View File

@@ -1,4 +1,4 @@
// Copyright 2015 The Hugo Authors. All rights reserved.
// Copyright 2016 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -152,7 +152,7 @@ func server(cmd *cobra.Command, args []string) error {
// Hugo writes the output to memory instead of the disk
if !renderToDisk {
hugofs.DestinationFS = new(afero.MemMapFs)
hugofs.SetDestination(new(afero.MemMapFs))
// Rendering to memoryFS, publish to Root regardless of publishDir.
viper.Set("PublishDir", "/")
}
@@ -191,7 +191,7 @@ func serve(port int) {
jww.FEEDBACK.Println("Serving pages from memory")
}
httpFs := afero.NewHttpFs(hugofs.DestinationFS)
httpFs := afero.NewHttpFs(hugofs.Destination())
fs := filesOnlyFs{httpFs.Dir(helpers.AbsPathify(viper.GetString("PublishDir")))}
fileserver := http.FileServer(fs)