mirror/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2025-08-12 20:13:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

deploy: walkLocal worker pool for performance

Browse Source
This commit is contained in:
David Jones
2025-06-29 15:41:56 +01:00
committed by GitHub
parent 762417617c
commit dd6e2c8724
2 changed files with 56 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
deploy/deploy.go
View File

@@ -36,6 +36,7 @@ import (
"github.com/dustin/go-humanize"
"github.com/gobwas/glob"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/para"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/deploy/deployconfig"
"github.com/gohugoio/hugo/media"
@@ -487,7 +488,12 @@ func knownHiddenDirectory(name string) bool {
// walkLocal walks the source directory and returns a flat list of files,
// using localFile.SlashPath as the map keys.
func (d *Deployer) walkLocal(fs afero.Fs, matchers []*deployconfig.Matcher, include, exclude glob.Glob, mediaTypes media.Types, mappath func(string) string) (map[string]*localFile, error) {
retval := map[string]*localFile{}
retval := make(map[string]*localFile)
var mu sync.Mutex
workers := para.New(d.cfg.Workers)
g, _ := workers.Start(context.Background())
err := afero.Walk(fs, "", func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
@@ -508,6 +514,8 @@ func (d *Deployer) walkLocal(fs afero.Fs, matchers []*deployconfig.Matcher, incl
return nil
}
// Process each file in a worker
g.Run(func() error {
// When a file system is HFS+, its filepath is in NFD form.
if runtime.GOOS == "darwin" {
path = norm.NFC.String(path)
@@ -541,12 +549,19 @@ func (d *Deployer) walkLocal(fs afero.Fs, matchers []*deployconfig.Matcher, incl
if err != nil {
return err
}
mu.Lock()
retval[lf.SlashPath] = lf
mu.Unlock()
return nil
})
return nil
})
if err != nil {
return nil, err
}
if err := g.Wait(); err != nil {
return nil, err
}
return retval, nil
}

13
deploy/deploy_test.go
View File

@@ -623,7 +623,7 @@ func TestEndToEndSync(t *testing.T) {
localFs: test.fs,
bucket: test.bucket,
mediaTypes: media.DefaultTypes,
cfg: deployconfig.DeployConfig{MaxDeletes: -1},
cfg: deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1},
}
// Initial deployment should sync remote with local.
@@ -706,7 +706,7 @@ func TestMaxDeletes(t *testing.T) {
localFs: test.fs,
bucket: test.bucket,
mediaTypes: media.DefaultTypes,
cfg: deployconfig.DeployConfig{MaxDeletes: -1},
cfg: deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1},
}
// Sync remote with local.
@@ -836,7 +836,7 @@ func TestIncludeExclude(t *testing.T) {
}
deployer := &Deployer{
localFs: fsTest.fs,
cfg: deployconfig.DeployConfig{MaxDeletes: -1}, bucket: fsTest.bucket,
cfg: deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1}, bucket: fsTest.bucket,
target: tgt,
mediaTypes: media.DefaultTypes,
}
@@ -893,7 +893,7 @@ func TestIncludeExcludeRemoteDelete(t *testing.T) {
}
deployer := &Deployer{
localFs: fsTest.fs,
cfg: deployconfig.DeployConfig{MaxDeletes: -1}, bucket: fsTest.bucket,
cfg: deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1}, bucket: fsTest.bucket,
mediaTypes: media.DefaultTypes,
}
@@ -945,7 +945,7 @@ func TestCompression(t *testing.T) {
deployer := &Deployer{
localFs: test.fs,
bucket: test.bucket,
cfg: deployconfig.DeployConfig{MaxDeletes: -1, Matchers: []*deployconfig.Matcher{{Pattern: ".*", Gzip: true, Re: regexp.MustCompile(".*")}}},
cfg: deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1, Matchers: []*deployconfig.Matcher{{Pattern: ".*", Gzip: true, Re: regexp.MustCompile(".*")}}},
mediaTypes: media.DefaultTypes,
}
@@ -1000,7 +1000,7 @@ func TestMatching(t *testing.T) {
deployer := &Deployer{
localFs: test.fs,
bucket: test.bucket,
cfg: deployconfig.DeployConfig{MaxDeletes: -1, Matchers: []*deployconfig.Matcher{{Pattern: "^subdir/aaa$", Force: true, Re: regexp.MustCompile("^subdir/aaa$")}}},
cfg: deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1, Matchers: []*deployconfig.Matcher{{Pattern: "^subdir/aaa$", Force: true, Re: regexp.MustCompile("^subdir/aaa$")}}},
mediaTypes: media.DefaultTypes,
}
@@ -1097,5 +1097,6 @@ func verifyRemote(ctx context.Context, bucket *blob.Bucket, local []*fileData) (
func newDeployer() *Deployer {
return &Deployer{
logger: loggers.NewDefault(),
cfg: deployconfig.DeployConfig{Workers: 2},
}
}