mirror/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2025-08-28 22:19:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add Hugo Modules

Browse Source 
This commit implements Hugo Modules.

This is a broad subject, but some keywords include:

* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`,  `hugo mod get`,  `hugo mod graph`,  `hugo mod tidy`, and  `hugo mod vendor`.

All of the above is backed by Go Modules.

Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092
This commit is contained in:
Bjørn Erik Pedersen
2019-05-03 09:16:58 +02:00
parent 47953148b6
commit 9f5a92078a
158 changed files with 9895 additions and 5433 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
source/fileInfo.go
View File

@@ -14,11 +14,14 @@
package source
import (
"os"
"path/filepath"
"strings"
"sync"
"github.com/gohugoio/hugo/hugofs/files"
"github.com/pkg/errors"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/hugofs"
@@ -28,8 +31,7 @@ import (
// fileInfo implements the File interface.
var (
_ File = (*FileInfo)(nil)
_ ReadableFile = (*FileInfo)(nil)
_ File = (*FileInfo)(nil)
)
// File represents a source file.
@@ -90,13 +92,7 @@ type FileWithoutOverlap interface {
// Hugo content files being one of them, considered to be unique.
UniqueID() string
FileInfo() os.FileInfo
}
// A ReadableFile is a File that is readable.
type ReadableFile interface {
File
Open() (hugio.ReadSeekCloser, error)
FileInfo() hugofs.FileMetaInfo
}
// FileInfo describes a source file.
@@ -107,7 +103,7 @@ type FileInfo struct {
sp *SourceSpec
fi os.FileInfo
fi hugofs.FileMetaInfo
// Derived from filename
ext string // Extension without any "."
@@ -179,13 +175,14 @@ func (fi *FileInfo) UniqueID() string {
}
// FileInfo returns a file's underlying os.FileInfo.
func (fi *FileInfo) FileInfo() os.FileInfo { return fi.fi }
func (fi *FileInfo) FileInfo() hugofs.FileMetaInfo { return fi.fi }
func (fi *FileInfo) String() string { return fi.BaseFileName() }
// Open implements ReadableFile.
func (fi *FileInfo) Open() (hugio.ReadSeekCloser, error) {
f, err := fi.sp.SourceFs.Open(fi.Filename())
f, err := fi.fi.Meta().Open()
return f, err
}
@@ -225,40 +222,47 @@ func NewTestFile(filename string) *FileInfo {
}
}
// NewFileInfo returns a new FileInfo structure.
func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, fi os.FileInfo) *FileInfo {
var lang, translationBaseName, relPath string
if fp, ok := fi.(hugofs.FilePather); ok {
filename = fp.Filename()
baseDir = fp.BaseDir()
relPath = fp.Path()
func (sp *SourceSpec) NewFileInfoFrom(path, filename string) (*FileInfo, error) {
meta := hugofs.FileMeta{
"filename": filename,
"path": path,
}
if fl, ok := fi.(hugofs.LanguageAnnouncer); ok {
lang = fl.Lang()
translationBaseName = fl.TranslationBaseName()
return sp.NewFileInfo(hugofs.NewFileMetaInfo(nil, meta))
}
func (sp *SourceSpec) NewFileInfo(fi hugofs.FileMetaInfo) (*FileInfo, error) {
m := fi.Meta()
filename := m.Filename()
relPath := m.Path()
isLeafBundle := m.Classifier() == files.ContentClassLeaf
if relPath == "" || strings.Contains(relPath, "TODO") {
return nil, errors.Errorf("no Path provided by %v (%T)", m, m.Fs())
}
dir, name := filepath.Split(filename)
if filename == "" || strings.Contains(filename, "TODO") {
return nil, errors.Errorf("no Filename provided by %v (%T)", m, m.Fs())
}
relDir := filepath.Dir(relPath)
if relDir == "." {
relDir = ""
}
if !strings.HasSuffix(relDir, helpers.FilePathSeparator) {
relDir = relDir + helpers.FilePathSeparator
}
lang := m.Lang()
translationBaseName := m.GetString("translationBaseName")
dir, name := filepath.Split(relPath)
if !strings.HasSuffix(dir, helpers.FilePathSeparator) {
dir = dir + helpers.FilePathSeparator
}
baseDir = strings.TrimSuffix(baseDir, helpers.FilePathSeparator)
relDir := ""
if dir != baseDir {
relDir = strings.TrimPrefix(dir, baseDir)
}
relDir = strings.TrimPrefix(relDir, helpers.FilePathSeparator)
if relPath == "" {
relPath = filepath.Join(relDir, name)
}
ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), "."))
baseName := helpers.Filename(name)
@@ -277,14 +281,14 @@ func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, f
lang: lang,
ext: ext,
dir: dir,
relDir: relDir,
relPath: relPath,
relDir: relDir, // Dir()
relPath: relPath, // Path()
name: name,
baseName: baseName,
baseName: baseName, // BaseFileName()
translationBaseName: translationBaseName,
isLeafBundle: isLeafBundle,
}
return f
return f, nil
}

57
source/fileInfo_test.go
View File

@@ -15,12 +15,9 @@ package source
import (
"path/filepath"
"strings"
"testing"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs"
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
)
@@ -55,56 +52,10 @@ func TestFileInfo(t *testing.T) {
}},
} {
f := s.NewFileInfo(this.base, this.filename, false, nil)
path := strings.TrimPrefix(this.filename, this.base)
f, err := s.NewFileInfoFrom(path, this.filename)
assert.NoError(err)
this.assert(f)
}
}
func TestFileInfoLanguage(t *testing.T) {
assert := require.New(t)
langs := map[string]bool{
"sv": true,
"en": true,
}
m := afero.NewMemMapFs()
lfs := hugofs.NewLanguageFs("sv", langs, m)
v := newTestConfig()
fs := hugofs.NewFrom(m, v)
ps, err := helpers.NewPathSpec(fs, v)
assert.NoError(err)
s := SourceSpec{SourceFs: lfs, PathSpec: ps}
s.Languages = map[string]interface{}{
"en": true,
}
err = afero.WriteFile(lfs, "page.md", []byte("abc"), 0777)
assert.NoError(err)
err = afero.WriteFile(lfs, "page.en.md", []byte("abc"), 0777)
assert.NoError(err)
sv, _ := lfs.Stat("page.md")
en, _ := lfs.Stat("page.en.md")
fiSv := s.NewFileInfo("", "page.md", false, sv)
fiEn := s.NewFileInfo("", "page.en.md", false, en)
assert.Equal("sv", fiSv.Lang())
assert.Equal("en", fiEn.Lang())
// test contentBaseName implementation
fi := s.NewFileInfo("", "2018-10-01-contentbasename.md", false, nil)
assert.Equal("2018-10-01-contentbasename", fi.ContentBaseName())
fi = s.NewFileInfo("", "2018-10-01-contentbasename.en.md", false, nil)
assert.Equal("2018-10-01-contentbasename", fi.ContentBaseName())
fi = s.NewFileInfo("", filepath.Join("2018-10-01-contentbasename", "index.en.md"), true, nil)
assert.Equal("2018-10-01-contentbasename", fi.ContentBaseName())
fi = s.NewFileInfo("", filepath.Join("2018-10-01-contentbasename", "_index.en.md"), false, nil)
assert.Equal("_index", fi.ContentBaseName())
}

106
source/filesystem.go
View File

@@ -14,29 +14,25 @@
package source
import (
"os"
"path/filepath"
"runtime"
"sync"
"github.com/gohugoio/hugo/helpers"
jww "github.com/spf13/jwalterweatherman"
"golang.org/x/text/unicode/norm"
"github.com/pkg/errors"
"github.com/gohugoio/hugo/hugofs"
)
// Filesystem represents a source filesystem.
type Filesystem struct {
files []ReadableFile
filesInit sync.Once
files []File
filesInit sync.Once
filesInitErr error
Base string
SourceSpec
}
fi hugofs.FileMetaInfo
// Input describes a source input.
type Input interface {
Files() []ReadableFile
SourceSpec
}
// NewFilesystem returns a new filesytem for a given source spec.
@@ -44,76 +40,74 @@ func (sp SourceSpec) NewFilesystem(base string) *Filesystem {
return &Filesystem{SourceSpec: sp, Base: base}
}
func (sp SourceSpec) NewFilesystemFromFileMetaInfo(fi hugofs.FileMetaInfo) *Filesystem {
return &Filesystem{SourceSpec: sp, fi: fi}
}
// Files returns a slice of readable files.
func (f *Filesystem) Files() []ReadableFile {
func (f *Filesystem) Files() ([]File, error) {
f.filesInit.Do(func() {
f.captureFiles()
err := f.captureFiles()
if err != nil {
f.filesInitErr = errors.Wrap(err, "capture files")
}
})
return f.files
return f.files, f.filesInitErr
}
// add populates a file in the Filesystem.files
func (f *Filesystem) add(name string, fi os.FileInfo) (err error) {
var file ReadableFile
func (f *Filesystem) add(name string, fi hugofs.FileMetaInfo) (err error) {
var file File
if runtime.GOOS == "darwin" {
// When a file system is HFS+, its filepath is in NFD form.
name = norm.NFC.String(name)
file, err = f.SourceSpec.NewFileInfo(fi)
if err != nil {
return err
}
file = f.SourceSpec.NewFileInfo(f.Base, name, false, fi)
f.files = append(f.files, file)
return err
}
func (f *Filesystem) captureFiles() {
walker := func(filePath string, fi os.FileInfo, err error) error {
if err != nil {
return nil
}
b, err := f.shouldRead(filePath, fi)
func (f *Filesystem) captureFiles() error {
walker := func(path string, fi hugofs.FileMetaInfo, err error) error {
if err != nil {
return err
}
if b {
f.add(filePath, fi)
if fi.IsDir() {
return nil
}
meta := fi.Meta()
filename := meta.Filename()
b, err := f.shouldRead(filename, fi)
if err != nil {
return err
}
if b {
err = f.add(filename, fi)
}
return err
}
if f.SourceFs == nil {
panic("Must have a fs")
}
err := helpers.SymbolicWalk(f.SourceFs, f.Base, walker)
w := hugofs.NewWalkway(hugofs.WalkwayConfig{
Fs: f.SourceFs,
Info: f.fi,
Root: f.Base,
WalkFn: walker,
})
if err != nil {
jww.ERROR.Println(err)
}
return w.Walk()
}
func (f *Filesystem) shouldRead(filename string, fi os.FileInfo) (bool, error) {
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
link, err := filepath.EvalSymlinks(filename)
if err != nil {
jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filename, err)
return false, nil
}
linkfi, err := f.SourceFs.Stat(link)
if err != nil {
jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err)
return false, nil
}
func (f *Filesystem) shouldRead(filename string, fi hugofs.FileMetaInfo) (bool, error) {
if !linkfi.Mode().IsRegular() {
jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filename)
}
return false, nil
}
ignore := f.SourceSpec.IgnoreFile(filename)
ignore := f.SourceSpec.IgnoreFile(fi.Meta().Filename())
if fi.IsDir() {
if ignore {

45
source/filesystem_test.go
View File

@@ -14,20 +14,31 @@
package source
import (
"os"
"fmt"
"path/filepath"
"runtime"
"testing"
"github.com/gohugoio/hugo/modules"
"github.com/gohugoio/hugo/langs"
"github.com/spf13/afero"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs"
"github.com/stretchr/testify/require"
"github.com/spf13/viper"
)
func TestEmptySourceFilesystem(t *testing.T) {
assert := require.New(t)
ss := newTestSourceSpec()
src := ss.NewFilesystem("Empty")
if len(src.Files()) != 0 {
src := ss.NewFilesystem("")
files, err := src.Files()
assert.NoError(err)
if len(files) != 0 {
t.Errorf("new filesystem should contain 0 files.")
}
}
@@ -38,6 +49,8 @@ func TestUnicodeNorm(t *testing.T) {
return
}
assert := require.New(t)
paths := []struct {
NFC string
NFD string
@@ -47,14 +60,18 @@ func TestUnicodeNorm(t *testing.T) {
}
ss := newTestSourceSpec()
var fi os.FileInfo
fi := hugofs.NewFileMetaInfo(nil, hugofs.FileMeta{})
for _, path := range paths {
src := ss.NewFilesystem("base")
for i, path := range paths {
base := fmt.Sprintf("base%d", i)
assert.NoError(afero.WriteFile(ss.Fs.Source, filepath.Join(base, path.NFD), []byte("some data"), 0777))
src := ss.NewFilesystem(base)
_ = src.add(path.NFD, fi)
f := src.Files()[0]
files, err := src.Files()
assert.NoError(err)
f := files[0]
if f.BaseFileName() != path.NFC {
t.Fatalf("file name in NFD form should be normalized (%s)", path.NFC)
t.Fatalf("file %q name in NFD form should be normalized (%s)", f.BaseFileName(), path.NFC)
}
}
@@ -70,12 +87,22 @@ func newTestConfig() *viper.Viper {
v.Set("resourceDir", "resources")
v.Set("publishDir", "public")
v.Set("assetDir", "assets")
_, err := langs.LoadLanguageSettings(v, nil)
if err != nil {
panic(err)
}
mod, err := modules.CreateProjectModule(v)
if err != nil {
panic(err)
}
v.Set("allModules", modules.Modules{mod})
return v
}
func newTestSourceSpec() *SourceSpec {
v := newTestConfig()
fs := hugofs.NewMem(v)
fs := hugofs.NewFrom(hugofs.NewBaseFileDecorator(afero.NewMemMapFs()), v)
ps, err := helpers.NewPathSpec(fs, v)
if err != nil {
panic(err)

13
source/sourceSpec.go
View File

@@ -17,6 +17,7 @@ import (
"os"
"path/filepath"
"regexp"
"runtime"
"github.com/gohugoio/hugo/langs"
"github.com/spf13/afero"
@@ -107,6 +108,18 @@ func (s *SourceSpec) IgnoreFile(filename string) bool {
}
}
if runtime.GOOS == "windows" {
// Also check the forward slash variant if different.
unixFilename := filepath.ToSlash(filename)
if unixFilename != filename {
for _, re := range s.ignoreFilesRe {
if re.MatchString(unixFilename) {
return true
}
}
}
}
return false
}