Make js.Build fully support modules

Fixes #7816
Fixes #7777
Fixes #7916
This commit is contained in:
Bjørn Erik Pedersen
2020-10-05 13:34:14 +02:00
parent 3089fc0ba1
commit 85e4dd7370
22 changed files with 949 additions and 988 deletions

View File

@@ -22,6 +22,8 @@ import (
"sync"
"sync/atomic"
"github.com/fsnotify/fsnotify"
"github.com/gohugoio/hugo/identity"
radix "github.com/armon/go-radix"
@@ -85,6 +87,10 @@ type HugoSites struct {
// Keeps track of bundle directories and symlinks to enable partial rebuilding.
ContentChanges *contentChangeMap
// File change events with filename stored in this map will be skipped.
skipRebuildForFilenamesMu sync.Mutex
skipRebuildForFilenames map[string]bool
init *hugoSitesInit
workers *para.Workers
@@ -94,6 +100,14 @@ type HugoSites struct {
*testCounters
}
// ShouldSkipFileChangeEvent allows skipping filesystem event early before
// the build is started.
func (h *HugoSites) ShouldSkipFileChangeEvent(ev fsnotify.Event) bool {
h.skipRebuildForFilenamesMu.Lock()
defer h.skipRebuildForFilenamesMu.Unlock()
return h.skipRebuildForFilenames[ev.Name]
}
func (h *HugoSites) getContentMaps() *pageMaps {
h.contentInit.Do(func() {
h.content = newPageMaps(h)
@@ -304,12 +318,13 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
}
h := &HugoSites{
running: cfg.Running,
multilingual: langConfig,
multihost: cfg.Cfg.GetBool("multihost"),
Sites: sites,
workers: workers,
numWorkers: numWorkers,
running: cfg.Running,
multilingual: langConfig,
multihost: cfg.Cfg.GetBool("multihost"),
Sites: sites,
workers: workers,
numWorkers: numWorkers,
skipRebuildForFilenames: make(map[string]bool),
init: &hugoSitesInit{
data: lazy.New(),
layouts: lazy.New(),

View File

@@ -33,8 +33,6 @@ import (
"github.com/spf13/afero"
"github.com/gohugoio/hugo/resources/resource"
"github.com/gohugoio/hugo/output"
"github.com/pkg/errors"
@@ -351,14 +349,45 @@ func (h *HugoSites) postProcess() error {
return err
}
var toPostProcess []resource.OriginProvider
for _, s := range h.Sites {
for _, v := range s.ResourceSpec.PostProcessResources {
toPostProcess = append(toPostProcess, v)
// This will only be set when js.Build have been triggered with
// imports that resolves to the project or a module.
// Write a jsconfig.json file to the project's /asset directory
// to help JS intellisense in VS Code etc.
if !h.ResourceSpec.BuildConfig.NoJSConfigInAssets && h.BaseFs.Assets.Dirs != nil {
m := h.BaseFs.Assets.Dirs[0].Meta()
assetsDir := m.Filename()
if strings.HasPrefix(assetsDir, h.ResourceSpec.WorkingDir) {
if jsConfig := h.ResourceSpec.JSConfigBuilder.Build(assetsDir); jsConfig != nil {
b, err := json.MarshalIndent(jsConfig, "", " ")
if err != nil {
h.Log.Warnf("Failed to create jsconfig.json: %s", err)
} else {
filename := filepath.Join(assetsDir, "jsconfig.json")
if h.running {
h.skipRebuildForFilenamesMu.Lock()
h.skipRebuildForFilenames[filename] = true
h.skipRebuildForFilenamesMu.Unlock()
}
// Make sure it's written to the OS fs as this is used by
// editors.
if err := afero.WriteFile(hugofs.Os, filename, b, 0666); err != nil {
h.Log.Warnf("Failed to write jsconfig.json: %s", err)
}
}
}
}
}
var toPostProcess []postpub.PostPublishedResource
for _, r := range h.ResourceSpec.PostProcessResources {
toPostProcess = append(toPostProcess, r)
}
if len(toPostProcess) == 0 {
// Nothing more to do.
return nil
}

View File

@@ -14,6 +14,7 @@
package hugolib
import (
"fmt"
"os"
"os/exec"
"path/filepath"
@@ -22,7 +23,6 @@ import (
"github.com/gohugoio/hugo/htesting"
"github.com/spf13/afero"
"github.com/spf13/viper"
qt "github.com/frankban/quicktest"
@@ -82,9 +82,7 @@ document.body.textContent = greeter(user);`
"scripts": {},
"dependencies": {
"to-camel-case": "1.0.0",
"react": "^16",
"react-dom": "^16"
"to-camel-case": "1.0.0"
}
}
`
@@ -153,333 +151,46 @@ func TestJSBuild(t *testing.T) {
c := qt.New(t)
mainJS := `
import "./included";
console.log("main");
`
includedJS := `
console.log("included");
`
workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-test-js")
workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-test-js-mod")
c.Assert(err, qt.IsNil)
defer clean()
v := viper.New()
v.Set("workingDir", workDir)
v.Set("disableKinds", []string{"taxonomy", "term", "page"})
b := newTestSitesBuilder(t).WithLogger(loggers.NewWarningLogger())
config := fmt.Sprintf(`
baseURL = "https://example.org"
workingDir = %q
b.Fs = hugofs.NewDefault(v)
b.WithWorkingDir(workDir)
b.WithViper(v)
b.WithContent("p1.md", "")
disableKinds = ["page", "section", "term", "taxonomy"]
b.WithTemplates("index.html", `
{{ $js := resources.Get "js/main.js" | js.Build }}
JS: {{ template "print" $js }}
[module]
[[module.imports]]
path="github.com/gohugoio/hugoTestProjectJSModImports"
{{ define "print" }}RelPermalink: {{.RelPermalink}}|MIME: {{ .MediaType }}|Content: {{ .Content | safeJS }}{{ end }}
`)
jsDir := filepath.Join(workDir, "assets", "js")
b.Assert(os.MkdirAll(jsDir, 0777), qt.IsNil)
b.Assert(os.Chdir(workDir), qt.IsNil)
b.WithSourceFile("assets/js/main.js", mainJS)
b.WithSourceFile("assets/js/included.js", includedJS)
b.Build(BuildCfg{})
b.AssertFileContent("public/index.html", `
console.log("included");
`)
}
func TestJSBuildGlobals(t *testing.T) {
if !isCI() {
t.Skip("skip (relative) long running modules test when running locally")
}
wd, _ := os.Getwd()
defer func() {
os.Chdir(wd)
}()
c := qt.New(t)
workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-test-js")
c.Assert(err, qt.IsNil)
defer clean()
v := viper.New()
v.Set("workingDir", workDir)
v.Set("disableKinds", []string{"taxonomy", "term", "page"})
b := newTestSitesBuilder(t).WithLogger(loggers.NewWarningLogger())
b.Fs = hugofs.NewDefault(v)
b.WithWorkingDir(workDir)
b.WithViper(v)
b.WithContent("p1.md", "")
jsDir := filepath.Join(workDir, "assets", "js")
b.Assert(os.MkdirAll(jsDir, 0777), qt.IsNil)
b.Assert(os.Chdir(workDir), qt.IsNil)
b.WithTemplates("index.html", `
{{- $js := resources.Get "js/main-project.js" | js.Build -}}
{{ template "print" (dict "js" $js "name" "root") }}
{{- define "print" -}}
{{ printf "rellink-%s-%s" .name .js.RelPermalink | safeHTML }}
{{ printf "mime-%s-%s" .name .js.MediaType | safeHTML }}
{{ printf "content-%s-%s" .name .js.Content | safeHTML }}
{{- end -}}
`)
b.WithSourceFile("assets/js/normal.js", `
const name = "root-normal";
export default name;
`)
b.WithSourceFile("assets/js/main-project.js", `
import normal from "@js/normal";
window.normal = normal; // make sure not to tree-shake
`)
b.Build(BuildCfg{})
b.AssertFileContent("public/index.html", `
const name = "root-normal";
`)
}
func TestJSBuildOverride(t *testing.T) {
if !isCI() {
t.Skip("skip (relative) long running modules test when running locally")
}
wd, _ := os.Getwd()
defer func() {
os.Chdir(wd)
}()
c := qt.New(t)
workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-test-js2")
c.Assert(err, qt.IsNil)
defer clean()
// workDir := "/tmp/hugo-test-js2"
c.Assert(os.Chdir(workDir), qt.IsNil)
cfg := viper.New()
cfg.Set("workingDir", workDir)
fs := hugofs.NewFrom(afero.NewOsFs(), cfg)
`, workDir)
b := newTestSitesBuilder(t)
b.Fs = fs
b.WithLogger(loggers.NewWarningLogger())
b.Fs = hugofs.NewDefault(viper.New())
b.WithWorkingDir(workDir).WithConfigFile("toml", config).WithLogger(loggers.NewInfoLogger())
b.WithSourceFile("go.mod", `module github.com/gohugoio/tests/testHugoModules
go 1.15
require github.com/gohugoio/hugoTestProjectJSModImports v0.3.0 // indirect
realWrite := func(name string, content string) {
realLocation := filepath.Join(workDir, name)
realDir := filepath.Dir(realLocation)
if _, err := os.Stat(realDir); err != nil {
os.MkdirAll(realDir, 0777)
}
bytesContent := []byte(content)
// c.Assert(ioutil.WriteFile(realLocation, bytesContent, 0777), qt.IsNil)
c.Assert(afero.WriteFile(b.Fs.Source, realLocation, bytesContent, 0777), qt.IsNil)
}
realWrite("config.toml", `
baseURL="https://example.org"
[module]
[[module.imports]]
path="mod2"
[[module.imports.mounts]]
source="assets"
target="assets"
[[module.imports.mounts]]
source="layouts"
target="layouts"
[[module.imports]]
path="mod1"
[[module.imports.mounts]]
source="assets"
target="assets"
[[module.imports.mounts]]
source="layouts"
target="layouts"
`)
realWrite("content/p1.md", `---
layout: sample
---
`)
realWrite("themes/mod1/layouts/_default/sample.html", `
{{- $js := resources.Get "js/main-project.js" | js.Build -}}
{{ template "print" (dict "js" $js "name" "root") }}
{{- $js = resources.Get "js/main-mod1.js" | js.Build -}}
{{ template "print" (dict "js" $js "name" "mod1") }}
{{- $js = resources.Get "js/main-mod2.js" | js.Build (dict "data" .Site.Params) -}}
{{ template "print" (dict "js" $js "name" "mod2") }}
{{- $js = resources.Get "js/main-mod2.js" | js.Build (dict "data" .Site.Params "sourceMap" "inline" "targetPath" "js/main-mod2-inline.js") -}}
{{ template "print" (dict "js" $js "name" "mod2") }}
{{- $js = resources.Get "js/main-mod2.js" | js.Build (dict "data" .Site.Params "sourceMap" "external" "targetPath" "js/main-mod2-external.js") -}}
{{ template "print" (dict "js" $js "name" "mod2") }}
{{- define "print" -}}
{{ printf "rellink-%s-%s" .name .js.RelPermalink | safeHTML }}
{{ printf "mime-%s-%s" .name .js.MediaType | safeHTML }}
{{ printf "content-%s-%s" .name .js.Content | safeHTML }}
{{- end -}}
`)
// Override project included file
// This file will override the one in mod1 and mod2
realWrite("assets/js/override.js", `
const name = "root-override";
export default name;
`)
// Add empty theme mod config files
realWrite("themes/mod1/config.yml", ``)
realWrite("themes/mod2/config.yml", ``)
// This is the main project js file.
// try to include @js/override which is overridden inside of project
// try to include @js/override-mod which is overridden in mod2
realWrite("assets/js/main-project.js", `
import override from "@js/override";
import overrideMod from "@js/override-mod";
window.override = override; // make sure to prevent tree-shake
window.overrideMod = overrideMod; // make sure to prevent tree-shake
`)
// This is the mod1 js file
// try to include @js/override which is overridden inside of the project
// try to include @js/override-mod which is overridden in mod2
realWrite("themes/mod1/assets/js/main-mod1.js", `
import override from "@js/override";
import overrideMod from "@js/override-mod";
window.mod = "mod1";
window.override = override; // make sure to prevent tree-shake
window.overrideMod = overrideMod; // make sure to prevent tree-shake
`)
// This is the mod1 js file (overridden in mod2)
// try to include @js/override which is overridden inside of the project
// try to include @js/override-mod which is overridden in mod2
realWrite("themes/mod2/assets/js/main-mod1.js", `
import override from "@js/override";
import overrideMod from "@js/override-mod";
window.mod = "mod2";
window.override = override; // make sure to prevent tree-shake
window.overrideMod = overrideMod; // make sure to prevent tree-shake
`)
// This is mod2 js file
// try to include @js/override which is overridden inside of the project
// try to include @js/override-mod which is overridden in mod2
// try to include @config which is declared in a local jsconfig.json file
// try to include @data which was passed as "data" into js.Build
realWrite("themes/mod2/assets/js/main-mod2.js", `
import override from "@js/override";
import overrideMod from "@js/override-mod";
import config from "@config";
import data from "@data";
window.data = data;
window.override = override; // make sure to prevent tree-shake
window.overrideMod = overrideMod; // make sure to prevent tree-shake
window.config = config;
`)
realWrite("themes/mod2/assets/js/jsconfig.json", `
{
"compilerOptions": {
"baseUrl": ".",
"paths": {
"@config": ["./config.json"]
}
}
}
`)
realWrite("themes/mod2/assets/js/config.json", `
{
"data": {
"sample": "sample"
}
}
`)
realWrite("themes/mod1/assets/js/override.js", `
const name = "mod1-override";
export default name;
`)
realWrite("themes/mod2/assets/js/override.js", `
const name = "mod2-override";
export default name;
`)
realWrite("themes/mod1/assets/js/override-mod.js", `
const nameMod = "mod1-override";
export default nameMod;
`)
realWrite("themes/mod2/assets/js/override-mod.js", `
const nameMod = "mod2-override";
export default nameMod;
`)
b.WithConfigFile("toml", `
baseURL="https://example.org"
themesDir="./themes"
[module]
[[module.imports]]
path="mod2"
[[module.imports.mounts]]
source="assets"
target="assets"
[[module.imports.mounts]]
source="layouts"
target="layouts"
[[module.imports]]
path="mod1"
[[module.imports.mounts]]
source="assets"
target="assets"
[[module.imports.mounts]]
source="layouts"
target="layouts"
`)
b.WithWorkingDir(workDir)
b.LoadConfig()
b.WithContent("p1.md", "").WithNothingAdded()
b.Build(BuildCfg{})
b.AssertFileContent("public/js/main-mod1.js", `
name = "root-override";
nameMod = "mod2-override";
window.mod = "mod2";
`)
b.AssertFileContent("public/js/main-mod2.js", `
name = "root-override";
nameMod = "mod2-override";
sample: "sample"
"sect"
`)
b.AssertFileContent("public/js/main-project.js", `
name = "root-override";
nameMod = "mod2-override";
`)
b.AssertFileContent("public/js/main-mod2-external.js.map", `
const nameMod = \"mod2-override\";\nexport default nameMod;\n
"\nimport override from \"@js/override\";\nimport overrideMod from \"@js/override-mod\";\nimport config from \"@config\";\nimport data from \"@data\";\nwindow.data = data;\nwindow.override = override; // make sure to prevent tree-shake\nwindow.overrideMod = overrideMod; // make sure to prevent tree-shake\nwindow.config = config;\n"
`)
b.AssertFileContent("public/js/main-mod2-inline.js", `
sourceMappingURL=data:application/json;base64,ewogICJ2ZXJzaW9uIjogMywKICAic291cmNlcyI6IFsiYXNzZXRzL2pzL292ZXJyaWRlLmpzIiwgInRoZW
`)
b.AssertFileContent("public/js/main.js", `
greeting: "greeting configured in mod2"
Hello1 from mod1: $
return "Hello2 from mod1";
var Hugo = "Rocks!";
return "Hello3 from mod2";
return "Hello from lib in the main project";
var myparam = "Hugo Rocks!";`)
}

View File

@@ -997,6 +997,16 @@ func (s *Site) translateFileEvents(events []fsnotify.Event) []fsnotify.Event {
return filtered
}
var (
// These are only used for cache busting, so false positives are fine.
// We also deliberately do not match for file suffixes to also catch
// directory names.
// TODO(bep) consider this when completing the relevant PR rewrite on this.
cssFileRe = regexp.MustCompile("(css|sass|scss)")
cssConfigRe = regexp.MustCompile(`(postcss|tailwind)\.config\.js`)
jsFileRe = regexp.MustCompile("(js|ts|jsx|tsx)")
)
// reBuild partially rebuilds a site given the filesystem events.
// It returns whetever the content source was changed.
// TODO(bep) clean up/rewrite this method.
@@ -1028,19 +1038,24 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro
logger = helpers.NewDistinctFeedbackLogger()
)
var isCSSConfigRe = regexp.MustCompile(`(postcss|tailwind)\.config\.js`)
var isCSSFileRe = regexp.MustCompile(`\.(css|scss|sass)`)
var cachePartitions []string
// Special case
// TODO(bep) I have a ongoing branch where I have redone the cache. Consider this there.
var isCSSChange bool
var (
evictCSSRe *regexp.Regexp
evictJSRe *regexp.Regexp
)
for _, ev := range events {
if assetsFilename := s.BaseFs.Assets.MakePathRelative(ev.Name); assetsFilename != "" {
cachePartitions = append(cachePartitions, resources.ResourceKeyPartitions(assetsFilename)...)
if !isCSSChange {
isCSSChange = isCSSFileRe.MatchString(assetsFilename) || isCSSConfigRe.MatchString(assetsFilename)
if evictCSSRe == nil {
if cssFileRe.MatchString(assetsFilename) || cssConfigRe.MatchString(assetsFilename) {
evictCSSRe = cssFileRe
}
}
if evictJSRe == nil && jsFileRe.MatchString(assetsFilename) {
evictJSRe = jsFileRe
}
}
@@ -1088,8 +1103,11 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro
// These in memory resource caches will be rebuilt on demand.
for _, s := range s.h.Sites {
s.ResourceSpec.ResourceCache.DeletePartitions(cachePartitions...)
if isCSSChange {
s.ResourceSpec.ResourceCache.DeleteContains("css", "scss", "sass")
if evictCSSRe != nil {
s.ResourceSpec.ResourceCache.DeleteMatches(evictCSSRe)
}
if evictJSRe != nil {
s.ResourceSpec.ResourceCache.DeleteMatches(evictJSRe)
}
}