mirror/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2025-08-22 21:42:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add Hugo Modules

Browse Source 
This commit implements Hugo Modules.

This is a broad subject, but some keywords include:

* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`,  `hugo mod get`,  `hugo mod graph`,  `hugo mod tidy`, and  `hugo mod vendor`.

All of the above is backed by Go Modules.

Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092
This commit is contained in:
Bjørn Erik Pedersen
2019-05-03 09:16:58 +02:00
parent 47953148b6
commit 9f5a92078a
158 changed files with 9895 additions and 5433 deletions
Download Patch File Download Diff File Expand all files Collapse all files

326
commands/hugo.go
View File

@@ -16,19 +16,18 @@
package commands
import (
"context"
"fmt"
"io/ioutil"
"os/signal"
"runtime/pprof"
"runtime/trace"
"sort"
"sync/atomic"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/common/hugo"
"github.com/pkg/errors"
"github.com/gohugoio/hugo/common/herrors"
@@ -49,7 +48,6 @@ import (
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/parser/metadecoders"
flag "github.com/spf13/pflag"
"github.com/fsnotify/fsnotify"
@@ -196,6 +194,7 @@ func initializeFlags(cmd *cobra.Command, cfg config.Provider) {
"forceSyncStatic",
"noTimes",
"noChmod",
"ignoreVendor",
"templateMetrics",
"templateMetricsHints",
@@ -291,6 +290,7 @@ func ifTerminal(s string) string {
}
func (c *commandeer) fullBuild() error {
var (
g errgroup.Group
langCount map[string]uint64
@@ -309,13 +309,9 @@ func (c *commandeer) fullBuild() error {
cnt, err := c.copyStatic()
if err != nil {
if !os.IsNotExist(err) {
return errors.Wrap(err, "Error copying static files")
}
c.logger.INFO.Println("No Static directory found")
return errors.Wrap(err, "Error copying static files")
}
langCount = cnt
langCount = cnt
return nil
}
buildSitesFunc := func() error {
@@ -503,7 +499,11 @@ func (c *commandeer) build() error {
if err != nil {
return err
}
c.logger.FEEDBACK.Println("Watching for changes in", c.hugo.PathSpec.AbsPathify(c.Cfg.GetString("contentDir")))
baseWatchDir := c.Cfg.GetString("workingDir")
rootWatchDirs := getRootWatchDirsStr(baseWatchDir, watchDirs)
c.logger.FEEDBACK.Printf("Watching for changes in %s%s{%s}\n", baseWatchDir, helpers.FilePathSeparator, rootWatchDirs)
c.logger.FEEDBACK.Println("Press Ctrl+C to stop")
watcher, err := c.newWatcher(watchDirs...)
checkErr(c.Logger, err)
@@ -547,7 +547,11 @@ func (c *commandeer) serverBuild() error {
}
func (c *commandeer) copyStatic() (map[string]uint64, error) {
return c.doWithPublishDirs(c.copyStaticTo)
m, err := c.doWithPublishDirs(c.copyStaticTo)
if err == nil || os.IsNotExist(err) {
return m, nil
}
return m, err
}
func (c *commandeer) doWithPublishDirs(f func(sourceFs *filesystems.SourceFilesystem) (uint64, error)) (map[string]uint64, error) {
@@ -566,6 +570,7 @@ func (c *commandeer) doWithPublishDirs(f func(sourceFs *filesystems.SourceFilesy
if err != nil {
return langCount, err
}
if lang == "" {
// Not multihost
for _, l := range c.languages {
@@ -594,6 +599,16 @@ func (fs *countingStatFs) Stat(name string) (os.FileInfo, error) {
return f, err
}
func chmodFilter(dst, src os.FileInfo) bool {
// Hugo publishes data from multiple sources, potentially
// with overlapping directory structures. We cannot sync permissions
// for directories as that would mean that we might end up with write-protected
// directories inside /public.
// One example of this would be syncing from the Go Module cache,
// which have 0555 directories.
return src.IsDir()
}
func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint64, error) {
publishDir := c.hugo.PathSpec.PublishDir
// If root, remove the second '/'
@@ -610,6 +625,7 @@ func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint6
syncer := fsync.NewSyncer()
syncer.NoTimes = c.Cfg.GetBool("noTimes")
syncer.NoChmod = c.Cfg.GetBool("noChmod")
syncer.ChmodFilter = chmodFilter
syncer.SrcFs = fs
syncer.DestFs = c.Fs.Destination
// Now that we are using a unionFs for the static directories
@@ -652,120 +668,39 @@ func (c *commandeer) timeTrack(start time.Time, name string) {
// getDirList provides NewWatcher() with a list of directories to watch for changes.
func (c *commandeer) getDirList() ([]string, error) {
var a []string
var dirnames []string
// To handle nested symlinked content dirs
var seen = make(map[string]bool)
var nested []string
newWalker := func(allowSymbolicDirs bool) func(path string, fi os.FileInfo, err error) error {
return func(path string, fi os.FileInfo, err error) error {
if err != nil {
if os.IsNotExist(err) {
return nil
}
c.logger.ERROR.Println("Walker: ", err)
return nil
}
// Skip .git directories.
// Related to https://github.com/gohugoio/hugo/issues/3468.
if fi.Name() == ".git" {
return nil
}
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
link, err := filepath.EvalSymlinks(path)
if err != nil {
c.logger.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", path, err)
return nil
}
linkfi, err := helpers.LstatIfPossible(c.Fs.Source, link)
if err != nil {
c.logger.ERROR.Printf("Cannot stat %q: %s", link, err)
return nil
}
if !allowSymbolicDirs && !linkfi.Mode().IsRegular() {
c.logger.ERROR.Printf("Symbolic links for directories not supported, skipping %q", path)
return nil
}
if allowSymbolicDirs && linkfi.IsDir() {
// afero.Walk will not walk symbolic links, so wee need to do it.
if !seen[path] {
seen[path] = true
nested = append(nested, path)
}
return nil
}
fi = linkfi
}
if fi.IsDir() {
if fi.Name() == ".git" ||
fi.Name() == "node_modules" || fi.Name() == "bower_components" {
return filepath.SkipDir
}
a = append(a, path)
}
walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error {
if err != nil {
c.logger.ERROR.Println("walker: ", err)
return nil
}
if fi.IsDir() {
if fi.Name() == ".git" ||
fi.Name() == "node_modules" || fi.Name() == "bower_components" {
return filepath.SkipDir
}
dirnames = append(dirnames, fi.Meta().Filename())
}
return nil
}
symLinkWalker := newWalker(true)
regularWalker := newWalker(false)
watchDirs := c.hugo.PathSpec.BaseFs.WatchDirs()
for _, watchDir := range watchDirs {
// SymbolicWalk will log anny ERRORs
// Also note that the Dirnames fetched below will contain any relevant theme
// directories.
for _, contentDir := range c.hugo.PathSpec.BaseFs.Content.Dirnames {
_ = helpers.SymbolicWalk(c.Fs.Source, contentDir, symLinkWalker)
}
for _, staticDir := range c.hugo.PathSpec.BaseFs.Data.Dirnames {
_ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker)
}
for _, staticDir := range c.hugo.PathSpec.BaseFs.I18n.Dirnames {
_ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker)
}
for _, staticDir := range c.hugo.PathSpec.BaseFs.Layouts.Dirnames {
_ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker)
}
for _, staticFilesystem := range c.hugo.PathSpec.BaseFs.Static {
for _, staticDir := range staticFilesystem.Dirnames {
_ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker)
w := hugofs.NewWalkway(hugofs.WalkwayConfig{Logger: c.logger, Info: watchDir, WalkFn: walkFn})
if err := w.Walk(); err != nil {
c.logger.ERROR.Println("walker: ", err)
}
}
for _, assetDir := range c.hugo.PathSpec.BaseFs.Assets.Dirnames {
_ = helpers.SymbolicWalk(c.Fs.Source, assetDir, regularWalker)
}
dirnames = helpers.UniqueStringsSorted(dirnames)
if len(nested) > 0 {
for {
toWalk := nested
nested = nested[:0]
for _, d := range toWalk {
_ = helpers.SymbolicWalk(c.Fs.Source, d, symLinkWalker)
}
if len(nested) == 0 {
break
}
}
}
a = helpers.UniqueStrings(a)
sort.Strings(a)
return a, nil
return dirnames, nil
}
func (c *commandeer) buildSites() (err error) {
@@ -812,26 +747,60 @@ func (c *commandeer) partialReRender(urls ...string) error {
return c.hugo.Build(hugolib.BuildCfg{RecentlyVisited: visited, PartialReRender: true})
}
func (c *commandeer) fullRebuild() {
c.commandeerHugoState = &commandeerHugoState{}
err := c.loadConfig(true, true)
if err != nil {
// Set the processing on pause until the state is recovered.
c.paused = true
c.handleBuildErr(err, "Failed to reload config")
} else {
c.paused = false
}
if !c.paused {
err := c.buildSites()
if err != nil {
c.logger.ERROR.Println(err)
} else if !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") {
livereload.ForceRefresh()
func (c *commandeer) fullRebuild(changeType string) {
if changeType == configChangeGoMod {
// go.mod may be changed during the build itself, and
// we really want to prevent superfluous builds.
if !c.fullRebuildSem.TryAcquire(1) {
return
}
c.fullRebuildSem.Release(1)
}
c.fullRebuildSem.Acquire(context.Background(), 1)
go func() {
defer c.fullRebuildSem.Release(1)
c.printChangeDetected(changeType)
defer func() {
// Allow any file system events to arrive back.
// This will block any rebuild on config changes for the
// duration of the sleep.
time.Sleep(2 * time.Second)
}()
defer c.timeTrack(time.Now(), "Total")
c.commandeerHugoState = &commandeerHugoState{}
err := c.loadConfig(true, true)
if err != nil {
// Set the processing on pause until the state is recovered.
c.paused = true
c.handleBuildErr(err, "Failed to reload config")
} else {
c.paused = false
}
if !c.paused {
_, err := c.copyStatic()
if err != nil {
c.logger.ERROR.Println(err)
return
}
err = c.buildSites()
if err != nil {
c.logger.ERROR.Println(err)
} else if !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") {
livereload.ForceRefresh()
}
}
}()
}
// newWatcher creates a new watcher to watch filesystem events.
@@ -886,26 +855,53 @@ func (c *commandeer) newWatcher(dirList ...string) (*watcher.Batcher, error) {
return watcher, nil
}
func (c *commandeer) printChangeDetected(typ string) {
msg := "\nChange"
if typ != "" {
msg += " of " + typ
}
msg += " detected, rebuilding site."
c.logger.FEEDBACK.Println(msg)
const layout = "2006-01-02 15:04:05.000 -0700"
c.logger.FEEDBACK.Println(time.Now().Format(layout))
}
const (
configChangeConfig = "config file"
configChangeGoMod = "go.mod file"
)
func (c *commandeer) handleEvents(watcher *watcher.Batcher,
staticSyncer *staticSyncer,
evs []fsnotify.Event,
configSet map[string]bool) {
var isHandled bool
for _, ev := range evs {
isConfig := configSet[ev.Name]
configChangeType := configChangeConfig
if isConfig {
if strings.Contains(ev.Name, "go.mod") {
configChangeType = configChangeGoMod
}
}
if !isConfig {
// It may be one of the /config folders
dirname := filepath.Dir(ev.Name)
if dirname != "." && configSet[dirname] {
isConfig = true
}
}
if isConfig {
isHandled = true
if ev.Op&fsnotify.Chmod == fsnotify.Chmod {
continue
}
if ev.Op&fsnotify.Remove == fsnotify.Remove || ev.Op&fsnotify.Rename == fsnotify.Rename {
for _, configFile := range c.configFiles {
counter := 0
@@ -917,13 +913,20 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher,
time.Sleep(100 * time.Millisecond)
}
}
// A write event will follow.
continue
}
// Config file(s) changed. Need full rebuild.
c.fullRebuild()
break
c.fullRebuild(configChangeType)
}
}
if isHandled {
return
}
if c.paused {
// Wait for the server to get into a consistent state before
// we continue with processing.
@@ -933,7 +936,9 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher,
if len(evs) > 50 {
// This is probably a mass edit of the content dir.
// Schedule a full rebuild for when it slows down.
c.debounce(c.fullRebuild)
c.debounce(func() {
c.fullRebuild("")
})
return
}
@@ -1015,7 +1020,7 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher,
continue
}
walkAdder := func(path string, f os.FileInfo, err error) error {
walkAdder := func(path string, f hugofs.FileMetaInfo, err error) error {
if f.IsDir() {
c.logger.FEEDBACK.Println("adding created directory to watchlist", path)
if err := watcher.Add(path); err != nil {
@@ -1046,9 +1051,7 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher,
}
if len(staticEvents) > 0 {
c.logger.FEEDBACK.Println("\nStatic file changes detected")
const layout = "2006-01-02 15:04:05.000 -0700"
c.logger.FEEDBACK.Println(time.Now().Format(layout))
c.printChangeDetected("Static files")
if c.Cfg.GetBool("forceSyncStatic") {
c.logger.FEEDBACK.Printf("Syncing all static files\n")
@@ -1087,10 +1090,7 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher,
doLiveReload := !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload")
onePageName := pickOneWriteOrCreatePath(partitionedEvents.ContentEvents)
c.logger.FEEDBACK.Println("\nChange detected, rebuilding site")
const layout = "2006-01-02 15:04:05.000 -0700"
c.logger.FEEDBACK.Println(time.Now().Format(layout))
c.printChangeDetected("")
c.changeDetector.PrepareNew()
if err := c.rebuildSites(dynamicEvents); err != nil {
c.handleBuildErr(err, "Rebuild failed")
@@ -1167,41 +1167,3 @@ func pickOneWriteOrCreatePath(events []fsnotify.Event) string {
return name
}
// isThemeVsHugoVersionMismatch returns whether the current Hugo version is
// less than any of the themes' min_version.
func (c *commandeer) isThemeVsHugoVersionMismatch(fs afero.Fs) (dir string, mismatch bool, requiredMinVersion string) {
if !c.hugo.PathSpec.ThemeSet() {
return
}
for _, absThemeDir := range c.hugo.BaseFs.AbsThemeDirs {
path := filepath.Join(absThemeDir, "theme.toml")
exists, err := helpers.Exists(path, fs)
if err != nil || !exists {
continue
}
b, err := afero.ReadFile(fs, path)
if err != nil {
continue
}
tomlMeta, err := metadecoders.Default.UnmarshalToMap(b, metadecoders.TOML)
if err != nil {
continue
}
if minVersion, ok := tomlMeta["min_version"]; ok {
if hugo.CompareVersion(minVersion) > 0 {
return absThemeDir, true, fmt.Sprint(minVersion)
}
}
}
return
}