hugofs: Make FileMeta a struct

This commit started out investigating a `concurrent map read write` issue, ending by replacing the map with a struct.

This is easier to reason about, and it's more effective:

```
name                                  old time/op    new time/op    delta
SiteNew/Regular_Deep_content_tree-16    71.5ms ± 3%    69.4ms ± 5%    ~     (p=0.200 n=4+4)

name                                  old alloc/op   new alloc/op   delta
SiteNew/Regular_Deep_content_tree-16    29.7MB ± 0%    27.9MB ± 0%  -5.82%  (p=0.029 n=4+4)

name                                  old allocs/op  new allocs/op  delta
SiteNew/Regular_Deep_content_tree-16      313k ± 0%      303k ± 0%  -3.35%  (p=0.029 n=4+4)
```

See #8749
This commit is contained in:
Bjørn Erik Pedersen
2021-07-13 11:41:02 +02:00
parent f27e542442
commit 022c479551
44 changed files with 434 additions and 451 deletions

View File

@@ -23,7 +23,7 @@ import (
"github.com/spf13/afero"
)
func decorateDirs(fs afero.Fs, meta FileMeta) afero.Fs {
func decorateDirs(fs afero.Fs, meta *FileMeta) afero.Fs {
ffs := &baseFileDecoratorFs{Fs: fs}
decorator := func(fi os.FileInfo, name string) (os.FileInfo, error) {
@@ -82,9 +82,11 @@ func NewBaseFileDecorator(fs afero.Fs, callbacks ...func(fi FileMetaInfo)) afero
decorator := func(fi os.FileInfo, filename string) (os.FileInfo, error) {
// Store away the original in case it's a symlink.
meta := FileMeta{metaKeyName: fi.Name()}
meta := NewFileMeta()
meta.Name = fi.Name()
if fi.IsDir() {
meta[metaKeyJoinStat] = func(name string) (FileMetaInfo, error) {
meta.JoinStatFunc = func(name string) (FileMetaInfo, error) {
joinedFilename := filepath.Join(filename, name)
fi, _, err := lstatIfPossible(fs, joinedFilename)
if err != nil {
@@ -102,7 +104,7 @@ func NewBaseFileDecorator(fs afero.Fs, callbacks ...func(fi FileMetaInfo)) afero
isSymlink := isSymlink(fi)
if isSymlink {
meta[metaKeyOriginalFilename] = filename
meta.OriginalFilename = filename
var link string
var err error
link, fi, err = evalSymlinks(fs, filename)
@@ -110,7 +112,7 @@ func NewBaseFileDecorator(fs afero.Fs, callbacks ...func(fi FileMetaInfo)) afero
return nil, err
}
filename = link
meta[metaKeyIsSymlink] = true
meta.IsSymlink = true
}
opener := func() (afero.File, error) {

View File

@@ -17,6 +17,7 @@ package hugofs
import (
"os"
"path/filepath"
"reflect"
"runtime"
"sort"
"strings"
@@ -27,242 +28,128 @@ import (
"github.com/pkg/errors"
"github.com/spf13/cast"
"github.com/gohugoio/hugo/common/hreflect"
"github.com/spf13/afero"
)
const (
metaKeyFilename = "filename"
metaKeySourceRoot = "sourceRoot"
metaKeyBaseDir = "baseDir" // Abs base directory of source file.
metaKeyMountRoot = "mountRoot"
metaKeyModule = "module"
metaKeyOriginalFilename = "originalFilename"
metaKeyName = "name"
metaKeyPath = "path"
metaKeyPathWalk = "pathWalk"
metaKeyLang = "lang"
metaKeyWeight = "weight"
metaKeyOrdinal = "ordinal"
metaKeyFs = "fs"
metaKeyOpener = "opener"
metaKeyIsOrdered = "isOrdered"
metaKeyIsSymlink = "isSymlink"
metaKeyJoinStat = "joinStat"
metaKeySkipDir = "skipDir"
metaKeyClassifier = "classifier"
metaKeyTranslationBaseName = "translationBaseName"
metaKeyTranslationBaseNameWithExt = "translationBaseNameWithExt"
metaKeyTranslations = "translations"
metaKeyDecoraterPath = "decoratorPath"
)
type FileMeta map[string]interface{}
func (f FileMeta) GetInt(key string) int {
return cast.ToInt(f[key])
}
func (f FileMeta) GetString(key string) string {
return cast.ToString(f[key])
}
func (f FileMeta) GetBool(key string) bool {
return cast.ToBool(f[key])
}
func (f FileMeta) Filename() string {
return f.stringV(metaKeyFilename)
}
func (f FileMeta) OriginalFilename() string {
return f.stringV(metaKeyOriginalFilename)
}
func (f FileMeta) SkipDir() bool {
return f.GetBool(metaKeySkipDir)
}
func (f FileMeta) TranslationBaseName() string {
return f.stringV(metaKeyTranslationBaseName)
}
func (f FileMeta) TranslationBaseNameWithExt() string {
return f.stringV(metaKeyTranslationBaseNameWithExt)
}
func (f FileMeta) Translations() []string {
return cast.ToStringSlice(f[metaKeyTranslations])
}
func (f FileMeta) Name() string {
return f.stringV(metaKeyName)
}
func (f FileMeta) Classifier() files.ContentClass {
c, found := f[metaKeyClassifier]
if found {
return c.(files.ContentClass)
}
return files.ContentClassFile // For sorting
}
func (f FileMeta) Lang() string {
return f.stringV(metaKeyLang)
}
// Path returns the relative file path to where this file is mounted.
func (f FileMeta) Path() string {
return f.stringV(metaKeyPath)
func NewFileMeta() *FileMeta {
return &FileMeta{}
}
// PathFile returns the relative file path for the file source.
func (f FileMeta) PathFile() string {
base := f.stringV(metaKeyBaseDir)
if base == "" {
func (f *FileMeta) PathFile() string {
if f.BaseDir == "" {
return ""
}
return strings.TrimPrefix(strings.TrimPrefix(f.Filename(), base), filepathSeparator)
return strings.TrimPrefix(strings.TrimPrefix(f.Filename, f.BaseDir), filepathSeparator)
}
func (f FileMeta) SourceRoot() string {
return f.stringV(metaKeySourceRoot)
type FileMeta struct {
Name string
Filename string
Path string
PathWalk string
OriginalFilename string
BaseDir string
SourceRoot string
MountRoot string
Module string
Weight int
Ordinal int
IsOrdered bool
IsSymlink bool
IsRootFile bool
Watch bool
Classifier files.ContentClass
SkipDir bool
Lang string
TranslationBaseName string
TranslationBaseNameWithExt string
Translations []string
Fs afero.Fs
OpenFunc func() (afero.File, error)
JoinStatFunc func(name string) (FileMetaInfo, error)
}
func (f FileMeta) MountRoot() string {
return f.stringV(metaKeyMountRoot)
}
func (f FileMeta) Module() string {
return f.stringV(metaKeyModule)
}
func (f FileMeta) Weight() int {
return f.GetInt(metaKeyWeight)
}
func (f FileMeta) Ordinal() int {
return f.GetInt(metaKeyOrdinal)
}
func (f FileMeta) IsOrdered() bool {
return f.GetBool(metaKeyIsOrdered)
}
// IsSymlink returns whether this comes from a symlinked file or directory.
func (f FileMeta) IsSymlink() bool {
return f.GetBool(metaKeyIsSymlink)
}
func (f FileMeta) Watch() bool {
if v, found := f["watch"]; found {
return v.(bool)
func (m *FileMeta) Copy() *FileMeta {
if m == nil {
return NewFileMeta()
}
return false
c := *m
return &c
}
func (f FileMeta) Fs() afero.Fs {
if v, found := f[metaKeyFs]; found {
return v.(afero.Fs)
}
return nil
}
func (f FileMeta) GetOpener() func() (afero.File, error) {
o, found := f[metaKeyOpener]
if !found {
return nil
}
return o.(func() (afero.File, error))
}
func (f FileMeta) Open() (afero.File, error) {
v, found := f[metaKeyOpener]
if !found {
return nil, errors.New("file opener not found")
}
return v.(func() (afero.File, error))()
}
func (f FileMeta) JoinStat(name string) (FileMetaInfo, error) {
v, found := f[metaKeyJoinStat]
if !found {
return nil, os.ErrNotExist
}
return v.(func(name string) (FileMetaInfo, error))(name)
}
func (f FileMeta) stringV(key string) string {
if v, found := f[key]; found {
return v.(string)
}
return ""
}
func (f FileMeta) setIfNotZero(key string, val interface{}) {
if !hreflect.IsTruthful(val) {
func (m *FileMeta) Merge(from *FileMeta) {
if m == nil || from == nil {
return
}
f[key] = val
dstv := reflect.Indirect(reflect.ValueOf(m))
srcv := reflect.Indirect(reflect.ValueOf(from))
for i := 0; i < dstv.NumField(); i++ {
v := dstv.Field(i)
if !hreflect.IsTruthfulValue(v) {
v.Set(srcv.Field(i))
}
}
}
func (f *FileMeta) Open() (afero.File, error) {
if f.OpenFunc == nil {
return nil, errors.New("OpenFunc not set")
}
return f.OpenFunc()
}
func (f *FileMeta) JoinStat(name string) (FileMetaInfo, error) {
if f.JoinStatFunc == nil {
return nil, os.ErrNotExist
}
return f.JoinStatFunc(name)
}
type FileMetaInfo interface {
os.FileInfo
Meta() FileMeta
Meta() *FileMeta
}
type fileInfoMeta struct {
os.FileInfo
m FileMeta
m *FileMeta
}
// Name returns the file's name. Note that we follow symlinks,
// if supported by the file system, and the Name given here will be the
// name of the symlink, which is what Hugo needs in all situations.
func (fi *fileInfoMeta) Name() string {
if name := fi.m.Name(); name != "" {
if name := fi.m.Name; name != "" {
return name
}
return fi.FileInfo.Name()
}
func (fi *fileInfoMeta) Meta() FileMeta {
func (fi *fileInfoMeta) Meta() *FileMeta {
return fi.m
}
func NewFileMetaInfo(fi os.FileInfo, m FileMeta) FileMetaInfo {
func NewFileMetaInfo(fi os.FileInfo, m *FileMeta) FileMetaInfo {
if m == nil {
panic("FileMeta must be set")
}
if fim, ok := fi.(FileMetaInfo); ok {
mergeFileMeta(fim.Meta(), m)
m.Merge(fim.Meta())
}
return &fileInfoMeta{FileInfo: fi, m: m}
}
func copyFileMeta(m FileMeta) FileMeta {
c := make(FileMeta)
for k, v := range m {
c[k] = v
}
return c
}
// Merge metadata, last entry wins.
func mergeFileMeta(from, to FileMeta) {
if from == nil {
return
}
for k, v := range from {
if _, found := to[k]; !found {
to[k] = v
}
}
}
type dirNameOnlyFileInfo struct {
name string
modTime time.Time
@@ -292,16 +179,16 @@ func (fi *dirNameOnlyFileInfo) Sys() interface{} {
return nil
}
func newDirNameOnlyFileInfo(name string, meta FileMeta, fileOpener func() (afero.File, error)) FileMetaInfo {
func newDirNameOnlyFileInfo(name string, meta *FileMeta, fileOpener func() (afero.File, error)) FileMetaInfo {
name = normalizeFilename(name)
_, base := filepath.Split(name)
m := copyFileMeta(meta)
if _, found := m[metaKeyFilename]; !found {
m.setIfNotZero(metaKeyFilename, name)
m := meta.Copy()
if m.Filename == "" {
m.Filename = name
}
m[metaKeyOpener] = fileOpener
m[metaKeyIsOrdered] = false
m.OpenFunc = fileOpener
m.IsOrdered = false
return NewFileMetaInfo(
&dirNameOnlyFileInfo{name: base, modTime: time.Now()},
@@ -312,8 +199,8 @@ func newDirNameOnlyFileInfo(name string, meta FileMeta, fileOpener func() (afero
func decorateFileInfo(
fi os.FileInfo,
fs afero.Fs, opener func() (afero.File, error),
filename, filepath string, inMeta FileMeta) FileMetaInfo {
var meta FileMeta
filename, filepath string, inMeta *FileMeta) FileMetaInfo {
var meta *FileMeta
var fim FileMetaInfo
filepath = strings.TrimPrefix(filepath, filepathSeparator)
@@ -322,16 +209,26 @@ func decorateFileInfo(
if fim, ok = fi.(FileMetaInfo); ok {
meta = fim.Meta()
} else {
meta = make(FileMeta)
meta = NewFileMeta()
fim = NewFileMetaInfo(fi, meta)
}
meta.setIfNotZero(metaKeyOpener, opener)
meta.setIfNotZero(metaKeyFs, fs)
meta.setIfNotZero(metaKeyPath, normalizeFilename(filepath))
meta.setIfNotZero(metaKeyFilename, normalizeFilename(filename))
if opener != nil {
meta.OpenFunc = opener
}
if fs != nil {
meta.Fs = fs
}
nfilepath := normalizeFilename(filepath)
nfilename := normalizeFilename(filename)
if nfilepath != "" {
meta.Path = nfilepath
}
if nfilename != "" {
meta.Filename = nfilename
}
mergeFileMeta(inMeta, meta)
meta.Merge(inMeta)
return fim
}
@@ -377,6 +274,6 @@ func fromSlash(filenames []string) []string {
func sortFileInfos(fis []os.FileInfo) {
sort.Slice(fis, func(i, j int) bool {
fimi, fimj := fis[i].(FileMetaInfo), fis[j].(FileMetaInfo)
return fimi.Meta().Filename() < fimj.Meta().Filename()
return fimi.Meta().Filename < fimj.Meta().Filename
})
}

51
hugofs/fileinfo_test.go Normal file
View File

@@ -0,0 +1,51 @@
// Copyright 2021 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hugofs
import (
"testing"
qt "github.com/frankban/quicktest"
)
func TestFileMeta(t *testing.T) {
c := qt.New(t)
c.Run("Merge", func(c *qt.C) {
src := &FileMeta{
Filename: "fs1",
Path: "ps1",
}
dst := &FileMeta{
Filename: "fd1",
}
dst.Merge(src)
c.Assert(dst.Path, qt.Equals, "ps1")
c.Assert(dst.Filename, qt.Equals, "fd1")
})
c.Run("Copy", func(c *qt.C) {
src := &FileMeta{
Filename: "fs1",
Path: "ps1",
}
dst := src.Copy()
c.Assert(dst, qt.Not(qt.Equals), src)
c.Assert(dst, qt.DeepEquals, src)
})
}

View File

@@ -44,7 +44,7 @@ func NewLanguageFs(langs map[string]int, fs afero.Fs) (afero.Fs, error) {
}
meta := fi.(FileMetaInfo).Meta()
lang := meta.Lang()
lang := meta.Lang
fileLang, translationBaseName, translationBaseNameWithExt := langInfoFrom(langs, fi.Name())
weight := 0
@@ -58,14 +58,16 @@ func NewLanguageFs(langs map[string]int, fs afero.Fs) (afero.Fs, error) {
lang = fileLang
}
fim := NewFileMetaInfo(fi, FileMeta{
metaKeyLang: lang,
metaKeyWeight: weight,
metaKeyOrdinal: langs[lang],
metaKeyTranslationBaseName: translationBaseName,
metaKeyTranslationBaseNameWithExt: translationBaseNameWithExt,
metaKeyClassifier: files.ClassifyContentFile(fi.Name(), meta.GetOpener()),
})
fim := NewFileMetaInfo(
fi,
&FileMeta{
Lang: lang,
Weight: weight,
Ordinal: langs[lang],
TranslationBaseName: translationBaseName,
TranslationBaseNameWithExt: translationBaseNameWithExt,
Classifier: files.ClassifyContentFile(fi.Name(), meta.OpenFunc),
})
fis[i] = fim
}
@@ -74,9 +76,9 @@ func NewLanguageFs(langs map[string]int, fs afero.Fs) (afero.Fs, error) {
all := func(fis []os.FileInfo) {
// Maps translation base name to a list of language codes.
translations := make(map[string][]string)
trackTranslation := func(meta FileMeta) {
name := meta.TranslationBaseNameWithExt()
translations[name] = append(translations[name], meta.Lang())
trackTranslation := func(meta *FileMeta) {
name := meta.TranslationBaseNameWithExt
translations[name] = append(translations[name], meta.Lang)
}
for _, fi := range fis {
if fi.IsDir() {
@@ -90,9 +92,9 @@ func NewLanguageFs(langs map[string]int, fs afero.Fs) (afero.Fs, error) {
for _, fi := range fis {
fim := fi.(FileMetaInfo)
langs := translations[fim.Meta().TranslationBaseNameWithExt()]
langs := translations[fim.Meta().TranslationBaseNameWithExt]
if len(langs) > 0 {
fim.Meta()["translations"] = sortAndremoveStringDuplicates(langs)
fim.Meta().Translations = sortAndremoveStringDuplicates(langs)
}
}
}
@@ -108,7 +110,7 @@ func NewFilterFs(fs afero.Fs) (afero.Fs, error) {
applyMeta := func(fs *FilterFs, name string, fis []os.FileInfo) {
for i, fi := range fis {
if fi.IsDir() {
fis[i] = decorateFileInfo(fi, fs, fs.getOpener(fi.(FileMetaInfo).Meta().Filename()), "", "", nil)
fis[i] = decorateFileInfo(fi, fs, fs.getOpener(fi.(FileMetaInfo).Meta().Filename), "", "", nil)
}
}
}

View File

@@ -35,7 +35,7 @@ func TestGlob(t *testing.T) {
collect := func(pattern string) []string {
var paths []string
h := func(fi FileMetaInfo) (bool, error) {
paths = append(paths, fi.Meta().Path())
paths = append(paths, fi.Meta().Path)
return false, nil
}
err := Glob(fs, pattern, h)

View File

@@ -59,7 +59,7 @@ var LanguageDirsMerger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) {
m := make(map[string]FileMetaInfo)
getKey := func(fim FileMetaInfo) string {
return path.Join(fim.Meta().Lang(), fim.Name())
return path.Join(fim.Meta().Lang, fim.Name())
}
for _, fi := range lofi {

View File

@@ -103,7 +103,7 @@ func (fs *noSymlinkFs) checkSymlinkStatus(name string, fi os.FileInfo) (os.FileI
if fim, ok := fi.(FileMetaInfo); ok {
meta := fim.Meta()
metaIsSymlink = meta.IsSymlink()
metaIsSymlink = meta.IsSymlink
}
if metaIsSymlink {

View File

@@ -55,19 +55,19 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {
// Extract "blog" from "content/blog"
rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator)
if rm.Meta == nil {
rm.Meta = make(FileMeta)
rm.Meta = NewFileMeta()
}
rm.Meta[metaKeySourceRoot] = rm.To
rm.Meta[metaKeyBaseDir] = rm.ToBasedir
rm.Meta[metaKeyMountRoot] = rm.path
rm.Meta[metaKeyModule] = rm.Module
rm.Meta.SourceRoot = rm.To
rm.Meta.BaseDir = rm.ToBasedir
rm.Meta.MountRoot = rm.path
rm.Meta.Module = rm.Module
meta := copyFileMeta(rm.Meta)
meta := rm.Meta.Copy()
if !fi.IsDir() {
_, name := filepath.Split(rm.From)
meta[metaKeyName] = name
meta.Name = name
}
rm.fi = NewFileMetaInfo(fi, meta)
@@ -114,11 +114,11 @@ func newRootMappingFsFromFromTo(
// RootMapping describes a virtual file or directory mount.
type RootMapping struct {
From string // The virtual mount.
To string // The source directory or file.
ToBasedir string // The base of To. May be empty if an absolute path was provided.
Module string // The module path/ID.
Meta FileMeta // File metadata (lang etc.)
From string // The virtual mount.
To string // The source directory or file.
ToBasedir string // The base of To. May be empty if an absolute path was provided.
Module string // The module path/ID.
Meta *FileMeta // File metadata (lang etc.)
fi FileMetaInfo
path string // The virtual mount point, e.g. "blog".
@@ -177,7 +177,7 @@ func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) {
}
if !fi.IsDir() {
mergeFileMeta(r.Meta, fi.(FileMetaInfo).Meta())
fi.(FileMetaInfo).Meta().Merge(r.Meta)
}
fss[i] = fi.(FileMetaInfo)
@@ -304,7 +304,7 @@ func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) {
return f, nil
}
rf := &rootMappingFile{File: f, fs: fs, name: meta.Name(), meta: meta}
rf := &rootMappingFile{File: f, fs: fs, name: meta.Name, meta: meta}
if len(fis) == 1 {
return rf, err
}
@@ -367,7 +367,7 @@ func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error)
for _, fi := range direntries {
meta := fi.(FileMetaInfo).Meta()
mergeFileMeta(rm.Meta, meta)
meta.Merge(rm.Meta)
if fi.IsDir() {
name := fi.Name()
if seen[name] {
@@ -556,7 +556,7 @@ func (fs *RootMappingFs) virtualDirOpener(name string) func() (afero.File, error
return func() (afero.File, error) { return &rootMappingFile{name: name, fs: fs}, nil }
}
func (fs *RootMappingFs) realDirOpener(name string, meta FileMeta) func() (afero.File, error) {
func (fs *RootMappingFs) realDirOpener(name string, meta *FileMeta) func() (afero.File, error) {
return func() (afero.File, error) {
f, err := fs.Fs.Open(name)
if err != nil {
@@ -570,7 +570,7 @@ type rootMappingFile struct {
afero.File
fs *RootMappingFs
name string
meta FileMeta
meta *FileMeta
}
func (f *rootMappingFile) Close() error {

View File

@@ -49,27 +49,27 @@ func TestLanguageRootMapping(t *testing.T) {
RootMapping{
From: "content/blog", // Virtual path, first element is one of content, static, layouts etc.
To: "themes/a/mysvblogcontent", // Real path
Meta: FileMeta{"lang": "sv"},
Meta: &FileMeta{Lang: "sv"},
},
RootMapping{
From: "content/blog",
To: "themes/a/myenblogcontent",
Meta: FileMeta{"lang": "en"},
Meta: &FileMeta{Lang: "en"},
},
RootMapping{
From: "content/blog",
To: "content/sv",
Meta: FileMeta{"lang": "sv"},
Meta: &FileMeta{Lang: "sv"},
},
RootMapping{
From: "content/blog",
To: "themes/a/myotherenblogcontent",
Meta: FileMeta{"lang": "en"},
Meta: &FileMeta{Lang: "en"},
},
RootMapping{
From: "content/docs",
To: "themes/a/mysvdocs",
Meta: FileMeta{"lang": "sv"},
Meta: &FileMeta{Lang: "sv"},
},
)
@@ -122,13 +122,13 @@ func TestLanguageRootMapping(t *testing.T) {
}
rfsEn := rfs.Filter(func(rm RootMapping) bool {
return rm.Meta.Lang() == "en"
return rm.Meta.Lang == "en"
})
c.Assert(getDirnames("content/blog", rfsEn), qt.DeepEquals, []string{"d1", "en-f.txt", "en-f2.txt"})
rfsSv := rfs.Filter(func(rm RootMapping) bool {
return rm.Meta.Lang() == "sv"
return rm.Meta.Lang == "sv"
})
c.Assert(getDirnames("content/blog", rfsSv), qt.DeepEquals, []string{"d1", "sv-f.txt", "svdir"})
@@ -157,7 +157,7 @@ func TestRootMappingFsDirnames(t *testing.T) {
c.Assert(err, qt.IsNil)
c.Assert(fif.Name(), qt.Equals, "myfile.txt")
fifm := fif.(FileMetaInfo).Meta()
c.Assert(fifm.Filename(), qt.Equals, filepath.FromSlash("f2t/myfile.txt"))
c.Assert(fifm.Filename, qt.Equals, filepath.FromSlash("f2t/myfile.txt"))
root, err := rfs.Open("static")
c.Assert(err, qt.IsNil)
@@ -185,7 +185,7 @@ func TestRootMappingFsFilename(t *testing.T) {
fi, err := rfs.Stat(filepath.FromSlash("static/f1/foo/file.txt"))
c.Assert(err, qt.IsNil)
fim := fi.(FileMetaInfo)
c.Assert(fim.Meta().Filename(), qt.Equals, testfilename)
c.Assert(fim.Meta().Filename, qt.Equals, testfilename)
_, err = rfs.Stat(filepath.FromSlash("static/f1"))
c.Assert(err, qt.IsNil)
}
@@ -209,30 +209,30 @@ func TestRootMappingFsMount(t *testing.T) {
{
From: "content/blog",
To: "mynoblogcontent",
Meta: FileMeta{"lang": "no"},
Meta: &FileMeta{Lang: "no"},
},
{
From: "content/blog",
To: "myenblogcontent",
Meta: FileMeta{"lang": "en"},
Meta: &FileMeta{Lang: "en"},
},
{
From: "content/blog",
To: "mysvblogcontent",
Meta: FileMeta{"lang": "sv"},
Meta: &FileMeta{Lang: "sv"},
},
// Files
{
From: "content/singles/p1.md",
To: "singlefiles/no.txt",
ToBasedir: "singlefiles",
Meta: FileMeta{"lang": "no"},
Meta: &FileMeta{Lang: "no"},
},
{
From: "content/singles/p1.md",
To: "singlefiles/sv.txt",
ToBasedir: "singlefiles",
Meta: FileMeta{"lang": "sv"},
Meta: &FileMeta{Lang: "sv"},
},
}
@@ -243,7 +243,7 @@ func TestRootMappingFsMount(t *testing.T) {
c.Assert(err, qt.IsNil)
c.Assert(blog.IsDir(), qt.Equals, true)
blogm := blog.(FileMetaInfo).Meta()
c.Assert(blogm.Lang(), qt.Equals, "no") // First match
c.Assert(blogm.Lang, qt.Equals, "no") // First match
f, err := blogm.Open()
c.Assert(err, qt.IsNil)
@@ -261,7 +261,7 @@ func TestRootMappingFsMount(t *testing.T) {
c.Assert(testfilefi.Name(), qt.Equals, testfile)
testfilem := testfilefi.(FileMetaInfo).Meta()
c.Assert(testfilem.Filename(), qt.Equals, filepath.FromSlash("themes/a/mynoblogcontent/test.txt"))
c.Assert(testfilem.Filename, qt.Equals, filepath.FromSlash("themes/a/mynoblogcontent/test.txt"))
tf, err := testfilem.Open()
c.Assert(err, qt.IsNil)
@@ -283,7 +283,7 @@ func TestRootMappingFsMount(t *testing.T) {
for i, lang := range []string{"no", "sv"} {
fi := singles[i].(FileMetaInfo)
c.Assert(fi.Meta().PathFile(), qt.Equals, filepath.FromSlash("themes/a/singlefiles/"+lang+".txt"))
c.Assert(fi.Meta().Lang(), qt.Equals, lang)
c.Assert(fi.Meta().Lang, qt.Equals, lang)
c.Assert(fi.Name(), qt.Equals, "p1.md")
}
}
@@ -431,7 +431,7 @@ func TestRootMappingFsOs(t *testing.T) {
}
i++
meta := fi.(FileMetaInfo).Meta()
c.Assert(meta.Filename(), qt.Equals, filepath.Join(d, fmt.Sprintf("/d1/d2/d3/f-%d.txt", i)))
c.Assert(meta.Filename, qt.Equals, filepath.Join(d, fmt.Sprintf("/d1/d2/d3/f-%d.txt", i)))
c.Assert(meta.PathFile(), qt.Equals, filepath.FromSlash(fmt.Sprintf("d1/d2/d3/f-%d.txt", i)))
}

View File

@@ -144,7 +144,7 @@ func (fs *SliceFs) getOpener(name string) func() (afero.File, error) {
func (fs *SliceFs) pickFirst(name string) (os.FileInfo, int, error) {
for i, mfs := range fs.dirs {
meta := mfs.Meta()
fs := meta.Fs()
fs := meta.Fs
fi, _, err := lstatIfPossible(fs, name)
if err == nil {
// Gotta match!
@@ -162,8 +162,8 @@ func (fs *SliceFs) pickFirst(name string) (os.FileInfo, int, error) {
}
func (fs *SliceFs) readDirs(name string, startIdx, count int) ([]os.FileInfo, error) {
collect := func(lfs FileMeta) ([]os.FileInfo, error) {
d, err := lfs.Fs().Open(name)
collect := func(lfs *FileMeta) ([]os.FileInfo, error) {
d, err := lfs.Fs.Open(name)
if err != nil {
if !os.IsNotExist(err) {
return nil, err
@@ -204,7 +204,7 @@ func (fs *SliceFs) readDirs(name string, startIdx, count int) ([]os.FileInfo, er
duplicates = append(duplicates, i)
} else {
// Make sure it's opened by this filesystem.
dirs[i] = decorateFileInfo(fi, fs, fs.getOpener(fi.(FileMetaInfo).Meta().Filename()), "", "", nil)
dirs[i] = decorateFileInfo(fi, fs, fs.getOpener(fi.(FileMetaInfo).Meta().Filename), "", "", nil)
seen[fi.Name()] = true
}
}

View File

@@ -73,7 +73,7 @@ type WalkwayConfig struct {
func NewWalkway(cfg WalkwayConfig) *Walkway {
var fs afero.Fs
if cfg.Info != nil {
fs = cfg.Info.Meta().Fs()
fs = cfg.Info.Meta().Fs
} else {
fs = cfg.Fs
}
@@ -184,7 +184,7 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo
}
meta := info.Meta()
filename := meta.Filename()
filename := meta.Filename
if dirEntries == nil {
f, err := w.fs.Open(path)
@@ -206,7 +206,7 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo
dirEntries = fileInfosToFileMetaInfos(fis)
if !meta.IsOrdered() {
if !meta.IsOrdered {
sort.Slice(dirEntries, func(i, j int) bool {
fii := dirEntries[i]
fij := dirEntries[j]
@@ -214,7 +214,7 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo
fim, fjm := fii.Meta(), fij.Meta()
// Pull bundle headers to the top.
ficlass, fjclass := fim.Classifier(), fjm.Classifier()
ficlass, fjclass := fim.Classifier, fjm.Classifier
if ficlass != fjclass {
return ficlass < fjclass
}
@@ -222,20 +222,20 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo
// With multiple content dirs with different languages,
// there can be duplicate files, and a weight will be added
// to the closest one.
fiw, fjw := fim.Weight(), fjm.Weight()
fiw, fjw := fim.Weight, fjm.Weight
if fiw != fjw {
return fiw > fjw
}
// Explicit order set.
fio, fjo := fim.Ordinal(), fjm.Ordinal()
fio, fjo := fim.Ordinal, fjm.Ordinal
if fio != fjo {
return fio < fjo
}
// When we walk into a symlink, we keep the reference to
// the original name.
fin, fjn := fim.Name(), fjm.Name()
fin, fjn := fim.Name, fjm.Name
if fin != "" && fjn != "" {
return fin < fjn
}
@@ -252,7 +252,7 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo
meta := fim.Meta()
// Note that we use the original Name even if it's a symlink.
name := meta.Name()
name := meta.Name
if name == "" {
name = fim.Name()
}
@@ -267,13 +267,13 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo
pathMeta = strings.TrimPrefix(pathn, w.basePath)
}
meta[metaKeyPath] = normalizeFilename(pathMeta)
meta[metaKeyPathWalk] = pathn
meta.Path = normalizeFilename(pathMeta)
meta.PathWalk = pathn
if fim.IsDir() && w.isSeen(meta.Filename()) {
if fim.IsDir() && w.isSeen(meta.Filename) {
// Prevent infinite recursion
// Possible cyclic reference
meta[metaKeySkipDir] = true
meta.SkipDir = true
}
}
@@ -291,11 +291,11 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo
fim := fi.(FileMetaInfo)
meta := fim.Meta()
if meta.SkipDir() {
if meta.SkipDir {
continue
}
err := w.walk(meta.GetString(metaKeyPathWalk), fim, nil, walkFn)
err := w.walk(meta.PathWalk, fim, nil, walkFn)
if err != nil {
if !fi.IsDir() || err != filepath.SkipDir {
return err

View File

@@ -14,6 +14,7 @@
package hugofs
import (
"context"
"fmt"
"os"
"path/filepath"
@@ -23,6 +24,7 @@ import (
"github.com/pkg/errors"
"github.com/gohugoio/hugo/common/para"
"github.com/gohugoio/hugo/htesting"
"github.com/spf13/afero"
@@ -47,38 +49,76 @@ func TestWalk(t *testing.T) {
func TestWalkRootMappingFs(t *testing.T) {
c := qt.New(t)
fs := NewBaseFileDecorator(afero.NewMemMapFs())
testfile := "test.txt"
prepare := func(c *qt.C) afero.Fs {
fs := NewBaseFileDecorator(afero.NewMemMapFs())
c.Assert(afero.WriteFile(fs, filepath.Join("a/b", testfile), []byte("some content"), 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join("c/d", testfile), []byte("some content"), 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join("e/f", testfile), []byte("some content"), 0755), qt.IsNil)
testfile := "test.txt"
rm := []RootMapping{
{
From: "static/b",
To: "e/f",
},
{
From: "static/a",
To: "c/d",
},
c.Assert(afero.WriteFile(fs, filepath.Join("a/b", testfile), []byte("some content"), 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join("c/d", testfile), []byte("some content"), 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join("e/f", testfile), []byte("some content"), 0755), qt.IsNil)
{
From: "static/c",
To: "a/b",
},
rm := []RootMapping{
{
From: "static/b",
To: "e/f",
},
{
From: "static/a",
To: "c/d",
},
{
From: "static/c",
To: "a/b",
},
}
rfs, err := NewRootMappingFs(fs, rm...)
c.Assert(err, qt.IsNil)
return afero.NewBasePathFs(rfs, "static")
}
rfs, err := NewRootMappingFs(fs, rm...)
c.Assert(err, qt.IsNil)
bfs := afero.NewBasePathFs(rfs, "static")
c.Run("Basic", func(c *qt.C) {
names, err := collectFilenames(bfs, "", "")
bfs := prepare(c)
c.Assert(err, qt.IsNil)
c.Assert(names, qt.DeepEquals, []string{"a/test.txt", "b/test.txt", "c/test.txt"})
names, err := collectFilenames(bfs, "", "")
c.Assert(err, qt.IsNil)
c.Assert(names, qt.DeepEquals, []string{"a/test.txt", "b/test.txt", "c/test.txt"})
})
c.Run("Para", func(c *qt.C) {
bfs := prepare(c)
p := para.New(4)
r, _ := p.Start(context.Background())
for i := 0; i < 8; i++ {
r.Run(func() error {
_, err := collectFilenames(bfs, "", "")
if err != nil {
return err
}
fi, err := bfs.Stat("b/test.txt")
if err != nil {
return err
}
meta := fi.(FileMetaInfo).Meta()
if meta.Filename == "" {
return errors.New("fail")
}
return nil
})
}
c.Assert(r.Wait(), qt.IsNil)
})
}
func skipSymlink() bool {
@@ -157,7 +197,7 @@ func collectFilenames(fs afero.Fs, base, root string) ([]string, error) {
return nil
}
filename := info.Meta().Path()
filename := info.Meta().Path
filename = filepath.ToSlash(filename)
names = append(names, filename)
@@ -221,7 +261,7 @@ func BenchmarkWalk(b *testing.B) {
return nil
}
filename := info.Meta().Filename()
filename := info.Meta().Filename
if !strings.HasPrefix(filename, "root") {
return errors.New(filename)
}