hugofs: Make FileMeta a struct

This commit started out investigating a `concurrent map read write` issue, ending by replacing the map with a struct.

This is easier to reason about, and it's more effective:

```
name                                  old time/op    new time/op    delta
SiteNew/Regular_Deep_content_tree-16    71.5ms ± 3%    69.4ms ± 5%    ~     (p=0.200 n=4+4)

name                                  old alloc/op   new alloc/op   delta
SiteNew/Regular_Deep_content_tree-16    29.7MB ± 0%    27.9MB ± 0%  -5.82%  (p=0.029 n=4+4)

name                                  old allocs/op  new allocs/op  delta
SiteNew/Regular_Deep_content_tree-16      313k ± 0%      303k ± 0%  -3.35%  (p=0.029 n=4+4)
```

See #8749
This commit is contained in:
Bjørn Erik Pedersen
2021-07-13 11:41:02 +02:00
parent f27e542442
commit 022c479551
44 changed files with 434 additions and 451 deletions

View File

@@ -14,6 +14,7 @@
package hugofs
import (
"context"
"fmt"
"os"
"path/filepath"
@@ -23,6 +24,7 @@ import (
"github.com/pkg/errors"
"github.com/gohugoio/hugo/common/para"
"github.com/gohugoio/hugo/htesting"
"github.com/spf13/afero"
@@ -47,38 +49,76 @@ func TestWalk(t *testing.T) {
func TestWalkRootMappingFs(t *testing.T) {
c := qt.New(t)
fs := NewBaseFileDecorator(afero.NewMemMapFs())
testfile := "test.txt"
prepare := func(c *qt.C) afero.Fs {
fs := NewBaseFileDecorator(afero.NewMemMapFs())
c.Assert(afero.WriteFile(fs, filepath.Join("a/b", testfile), []byte("some content"), 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join("c/d", testfile), []byte("some content"), 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join("e/f", testfile), []byte("some content"), 0755), qt.IsNil)
testfile := "test.txt"
rm := []RootMapping{
{
From: "static/b",
To: "e/f",
},
{
From: "static/a",
To: "c/d",
},
c.Assert(afero.WriteFile(fs, filepath.Join("a/b", testfile), []byte("some content"), 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join("c/d", testfile), []byte("some content"), 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join("e/f", testfile), []byte("some content"), 0755), qt.IsNil)
{
From: "static/c",
To: "a/b",
},
rm := []RootMapping{
{
From: "static/b",
To: "e/f",
},
{
From: "static/a",
To: "c/d",
},
{
From: "static/c",
To: "a/b",
},
}
rfs, err := NewRootMappingFs(fs, rm...)
c.Assert(err, qt.IsNil)
return afero.NewBasePathFs(rfs, "static")
}
rfs, err := NewRootMappingFs(fs, rm...)
c.Assert(err, qt.IsNil)
bfs := afero.NewBasePathFs(rfs, "static")
c.Run("Basic", func(c *qt.C) {
names, err := collectFilenames(bfs, "", "")
bfs := prepare(c)
c.Assert(err, qt.IsNil)
c.Assert(names, qt.DeepEquals, []string{"a/test.txt", "b/test.txt", "c/test.txt"})
names, err := collectFilenames(bfs, "", "")
c.Assert(err, qt.IsNil)
c.Assert(names, qt.DeepEquals, []string{"a/test.txt", "b/test.txt", "c/test.txt"})
})
c.Run("Para", func(c *qt.C) {
bfs := prepare(c)
p := para.New(4)
r, _ := p.Start(context.Background())
for i := 0; i < 8; i++ {
r.Run(func() error {
_, err := collectFilenames(bfs, "", "")
if err != nil {
return err
}
fi, err := bfs.Stat("b/test.txt")
if err != nil {
return err
}
meta := fi.(FileMetaInfo).Meta()
if meta.Filename == "" {
return errors.New("fail")
}
return nil
})
}
c.Assert(r.Wait(), qt.IsNil)
})
}
func skipSymlink() bool {
@@ -157,7 +197,7 @@ func collectFilenames(fs afero.Fs, base, root string) ([]string, error) {
return nil
}
filename := info.Meta().Path()
filename := info.Meta().Path
filename = filepath.ToSlash(filename)
names = append(names, filename)
@@ -221,7 +261,7 @@ func BenchmarkWalk(b *testing.B) {
return nil
}
filename := info.Meta().Filename()
filename := info.Meta().Filename
if !strings.HasPrefix(filename, "root") {
return errors.New(filename)
}