mirror/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2025-08-19 21:21:39 +02:00
Code Issues Packages Projects Releases Wiki Activity

✨ Implement Page bundling and image handling

Browse Source 
This commit is not the smallest in Hugo's history.

Some hightlights include:

* Page bundles (for complete articles, keeping images and content together etc.).
* Bundled images can be processed in as many versions/sizes as you need with the three methods `Resize`, `Fill` and `Fit`.
* Processed images are cached inside `resources/_gen/images` (default) in your project.
* Symbolic links (both files and dirs) are now allowed anywhere inside /content
* A new table based build summary
* The "Total in nn ms" now reports the total including the handling of the files inside /static. So if it now reports more than you're used to, it is just **more real** and probably faster than before (see below).

A site building  benchmark run compared to `v0.31.1` shows that this should be slightly faster and use less memory:

```bash
▶ ./benchSite.sh "TOML,num_langs=.*,num_root_sections=5,num_pages=(500|1000),tags_per_page=5,shortcodes,render"

benchmark                                                                                                         old ns/op     new ns/op     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      101785785     78067944      -23.30%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     185481057     149159919     -19.58%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      103149918     85679409      -16.94%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     203515478     169208775     -16.86%

benchmark                                                                                                         old allocs     new allocs     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      532464         391539         -26.47%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     1056549        772702         -26.87%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      555974         406630         -26.86%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     1086545        789922         -27.30%

benchmark                                                                                                         old bytes     new bytes     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      53243246      43598155      -18.12%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     105811617     86087116      -18.64%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      54558852      44545097      -18.35%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     106903858     86978413      -18.64%
```

Fixes #3651
Closes #3158
Fixes #1014
Closes #2021
Fixes #1240
Updates #3757
This commit is contained in:
Bjørn Erik Pedersen
2017-07-24 09:00:23 +02:00
parent 02f2735f68
commit 3cdf19e9b7
85 changed files with 5791 additions and 3287 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
source/filesystem.go
View File

@@ -14,73 +14,52 @@
package source
import (
"io"
"os"
"path/filepath"
"regexp"
"runtime"
"strings"
"sync"
"github.com/gohugoio/hugo/helpers"
"github.com/spf13/cast"
jww "github.com/spf13/jwalterweatherman"
"golang.org/x/text/unicode/norm"
)
type Input interface {
Files() []*File
}
type Filesystem struct {
files []*File
Base string
AvoidPaths []string
files []ReadableFile
filesInit sync.Once
Base string
SourceSpec
}
func (sp SourceSpec) NewFilesystem(base string, avoidPaths ...string) *Filesystem {
return &Filesystem{SourceSpec: sp, Base: base, AvoidPaths: avoidPaths}
type Input interface {
Files() []ReadableFile
}
func (f *Filesystem) FilesByExts(exts ...string) []*File {
var newFiles []*File
if len(exts) == 0 {
return f.Files()
}
for _, x := range f.Files() {
for _, e := range exts {
if x.Ext() == strings.TrimPrefix(e, ".") {
newFiles = append(newFiles, x)
}
}
}
return newFiles
func (sp SourceSpec) NewFilesystem(base string) *Filesystem {
return &Filesystem{SourceSpec: sp, Base: base}
}
func (f *Filesystem) Files() []*File {
if len(f.files) < 1 {
func (f *Filesystem) Files() []ReadableFile {
f.filesInit.Do(func() {
f.captureFiles()
}
})
return f.files
}
// add populates a file in the Filesystem.files
func (f *Filesystem) add(name string, reader io.Reader) (err error) {
var file *File
func (f *Filesystem) add(name string, fi os.FileInfo) (err error) {
var file ReadableFile
if runtime.GOOS == "darwin" {
// When a file system is HFS+, its filepath is in NFD form.
name = norm.NFC.String(name)
}
file, err = f.SourceSpec.NewFileFromAbs(f.Base, name, reader)
file = f.SourceSpec.NewFileInfo(f.Base, name, fi)
f.files = append(f.files, file)
if err == nil {
f.files = append(f.files, file)
}
return err
}
@@ -90,16 +69,12 @@ func (f *Filesystem) captureFiles() {
return nil
}
b, err := f.ShouldRead(filePath, fi)
b, err := f.shouldRead(filePath, fi)
if err != nil {
return err
}
if b {
rd, err := NewLazyFileReader(f.Fs.Source, filePath)
if err != nil {
return err
}
f.add(filePath, rd)
f.add(filePath, fi)
}
return err
}
@@ -118,11 +93,11 @@ func (f *Filesystem) captureFiles() {
}
func (f *Filesystem) ShouldRead(filePath string, fi os.FileInfo) (bool, error) {
func (f *Filesystem) shouldRead(filename string, fi os.FileInfo) (bool, error) {
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
link, err := filepath.EvalSymlinks(filePath)
link, err := filepath.EvalSymlinks(filename)
if err != nil {
jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filePath, err)
jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filename, err)
return false, nil
}
linkfi, err := f.Fs.Source.Stat(link)
@@ -130,52 +105,25 @@ func (f *Filesystem) ShouldRead(filePath string, fi os.FileInfo) (bool, error) {
jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err)
return false, nil
}
if !linkfi.Mode().IsRegular() {
jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filePath)
jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filename)
}
return false, nil
}
ignore := f.SourceSpec.IgnoreFile(filename)
if fi.IsDir() {
if f.avoid(filePath) || f.isNonProcessablePath(filePath) {
if ignore {
return false, filepath.SkipDir
}
return false, nil
}
if f.isNonProcessablePath(filePath) {
if ignore {
return false, nil
}
return true, nil
}
func (f *Filesystem) avoid(filePath string) bool {
for _, avoid := range f.AvoidPaths {
if avoid == filePath {
return true
}
}
return false
}
func (sp SourceSpec) isNonProcessablePath(filePath string) bool {
base := filepath.Base(filePath)
if strings.HasPrefix(base, ".") ||
strings.HasPrefix(base, "#") ||
strings.HasSuffix(base, "~") {
return true
}
ignoreFiles := cast.ToStringSlice(sp.Cfg.Get("ignoreFiles"))
if len(ignoreFiles) > 0 {
for _, ignorePattern := range ignoreFiles {
match, err := regexp.MatchString(ignorePattern, filePath)
if err != nil {
helpers.DistinctErrorLog.Printf("Invalid regexp '%s' in ignoreFiles: %s", ignorePattern, err)
return false
} else if match {
return true
}
}
}
return false
}