mirror/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2025-08-21 21:35:28 +02:00
Code Issues Packages Projects Releases Wiki Activity

✨ Implement Page bundling and image handling

Browse Source 
This commit is not the smallest in Hugo's history.

Some hightlights include:

* Page bundles (for complete articles, keeping images and content together etc.).
* Bundled images can be processed in as many versions/sizes as you need with the three methods `Resize`, `Fill` and `Fit`.
* Processed images are cached inside `resources/_gen/images` (default) in your project.
* Symbolic links (both files and dirs) are now allowed anywhere inside /content
* A new table based build summary
* The "Total in nn ms" now reports the total including the handling of the files inside /static. So if it now reports more than you're used to, it is just **more real** and probably faster than before (see below).

A site building  benchmark run compared to `v0.31.1` shows that this should be slightly faster and use less memory:

```bash
▶ ./benchSite.sh "TOML,num_langs=.*,num_root_sections=5,num_pages=(500|1000),tags_per_page=5,shortcodes,render"

benchmark                                                                                                         old ns/op     new ns/op     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      101785785     78067944      -23.30%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     185481057     149159919     -19.58%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      103149918     85679409      -16.94%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     203515478     169208775     -16.86%

benchmark                                                                                                         old allocs     new allocs     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      532464         391539         -26.47%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     1056549        772702         -26.87%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      555974         406630         -26.86%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     1086545        789922         -27.30%

benchmark                                                                                                         old bytes     new bytes     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      53243246      43598155      -18.12%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     105811617     86087116      -18.64%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      54558852      44545097      -18.35%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     106903858     86978413      -18.64%
```

Fixes #3651
Closes #3158
Fixes #1014
Closes #2021
Fixes #1240
Updates #3757
This commit is contained in:
Bjørn Erik Pedersen
2017-07-24 09:00:23 +02:00
parent 02f2735f68
commit 3cdf19e9b7
85 changed files with 5791 additions and 3287 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
helpers/content.go
View File

@@ -47,6 +47,10 @@ type ContentSpec struct {
// SummaryLength is the length of the summary that Hugo extracts from a content.
summaryLength int
BuildFuture bool
BuildExpired bool
BuildDrafts bool
Highlight func(code, lang, optsStr string) (string, error)
defatultPygmentsOpts map[string]string
@@ -62,6 +66,9 @@ func NewContentSpec(cfg config.Provider) (*ContentSpec, error) {
footnoteAnchorPrefix: cfg.GetString("footnoteAnchorPrefix"),
footnoteReturnLinkContents: cfg.GetString("footnoteReturnLinkContents"),
summaryLength: cfg.GetInt("summaryLength"),
BuildFuture: cfg.GetBool("buildFuture"),
BuildExpired: cfg.GetBool("buildExpired"),
BuildDrafts: cfg.GetBool("buildDrafts"),
cfg: cfg,
}

22
helpers/content_test.go
View File

@@ -19,9 +19,12 @@ import (
"strings"
"testing"
"github.com/spf13/viper"
"github.com/miekg/mmark"
"github.com/russross/blackfriday"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const tstHTMLContent = "<!DOCTYPE html><html><head><script src=\"http://two/foobar.js\"></script></head><body><nav><ul><li hugo-nav=\"section_0\"></li><li hugo-nav=\"section_1\"></li></ul></nav><article>content <a href=\"http://two/foobar\">foobar</a>. Follow up</article><p>This is some text.<br>And some more.</p></body></html>"
@@ -73,6 +76,25 @@ func TestBytesToHTML(t *testing.T) {
assert.Equal(t, template.HTML("dobedobedo"), BytesToHTML([]byte("dobedobedo")))
}
func TestNewContentSpec(t *testing.T) {
cfg := viper.New()
assert := require.New(t)
cfg.Set("summaryLength", 32)
cfg.Set("buildFuture", true)
cfg.Set("buildExpired", true)
cfg.Set("buildDrafts", true)
spec, err := NewContentSpec(cfg)
assert.NoError(err)
assert.Equal(32, spec.summaryLength)
assert.True(spec.BuildFuture)
assert.True(spec.BuildExpired)
assert.True(spec.BuildDrafts)
}
var benchmarkTruncateString = strings.Repeat("This is a sentence about nothing.", 20)
func BenchmarkTestTruncateWordsToWholeSentence(b *testing.B) {

4
helpers/general.go
View File

@@ -365,8 +365,8 @@ func SliceToLower(s []string) []string {
return l
}
// Md5String takes a string and returns its MD5 hash.
func Md5String(f string) string {
// MD5String takes a string and returns its MD5 hash.
func MD5String(f string) string {
h := md5.New()
h.Write([]byte(f))
return hex.EncodeToString(h.Sum([]byte{}))

22
helpers/language.go
View File

@@ -53,7 +53,16 @@ func (l *Language) String() string {
// NewLanguage creates a new language.
func NewLanguage(lang string, cfg config.Provider) *Language {
return &Language{Lang: lang, Cfg: cfg, params: make(map[string]interface{})}
params := make(map[string]interface{})
// Merge with global config.
globalParams := cfg.GetStringMap("params")
for k, v := range globalParams {
if _, ok := params[k]; !ok {
params[k] = v
}
}
l := &Language{Lang: lang, Cfg: cfg, params: params}
return l
}
// NewDefaultLanguage creates the default language for a config.Provider.
@@ -88,17 +97,6 @@ func (l Languages) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
// Params retunrs language-specific params merged with the global params.
func (l *Language) Params() map[string]interface{} {
l.paramsInit.Do(func() {
// Merge with global config.
// TODO(bep) consider making this part of a constructor func.
globalParams := l.Cfg.GetStringMap("params")
for k, v := range globalParams {
if _, ok := l.params[k]; !ok {
l.params[k] = v
}
}
})
return l.params
}

46
helpers/path.go
View File

@@ -277,6 +277,12 @@ func Ext(in string) string {
return ext
}
// FileAndExt takes a path and returns the file and extension separated,
// the extension including the delmiter, i.e. ".md".
func FileAndExt(in string) (string, string) {
return fileAndExt(in, fpb)
}
// Filename takes a path, strips out the extension,
// and returns the name of the file.
func Filename(in string) (name string) {
@@ -348,40 +354,6 @@ func GetRelativePath(path, base string) (final string, err error) {
return name, nil
}
// GuessSection returns the section given a source path.
// A section is the part between the root slash and the second slash
// or before the first slash.
func GuessSection(in string) string {
parts := strings.Split(in, FilePathSeparator)
// This will include an empty entry before and after paths with leading and trailing slashes
// eg... /sect/one/ -> ["", "sect", "one", ""]
// Needs to have at least a value and a slash
if len(parts) < 2 {
return ""
}
// If it doesn't have a leading slash and value and file or trailing slash, then return ""
if parts[0] == "" && len(parts) < 3 {
return ""
}
// strip leading slash
if parts[0] == "" {
parts = parts[1:]
}
// if first directory is "content", return second directory
if parts[0] == "content" {
if len(parts) > 2 {
return parts[1]
}
return ""
}
return parts[0]
}
// PathPrep prepares the path using the uglify setting to create paths on
// either the form /section/name/index.html or /section/name.html.
func PathPrep(ugly bool, in string) string {
@@ -504,7 +476,7 @@ func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error {
}
func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) {
fileInfo, err := lstatIfOs(fs, path)
fileInfo, err := LstatIfOs(fs, path)
realPath := path
if err != nil {
@@ -516,7 +488,7 @@ func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) {
if err != nil {
return nil, "", fmt.Errorf("Cannot read symbolic link '%s', error was: %s", path, err)
}
fileInfo, err = lstatIfOs(fs, link)
fileInfo, err = LstatIfOs(fs, link)
if err != nil {
return nil, "", fmt.Errorf("Cannot stat '%s', error was: %s", link, err)
}
@@ -539,7 +511,7 @@ func GetRealPath(fs afero.Fs, path string) (string, error) {
// Code copied from Afero's path.go
// if the filesystem is OsFs use Lstat, else use fs.Stat
func lstatIfOs(fs afero.Fs, path string) (info os.FileInfo, err error) {
func LstatIfOs(fs afero.Fs, path string) (info os.FileInfo, err error) {
_, ok := fs.(*afero.OsFs)
if ok {
info, err = os.Lstat(path)

34
helpers/path_test.go
View File

@@ -638,40 +638,6 @@ func TestFileAndExt(t *testing.T) {
}
func TestGuessSection(t *testing.T) {
type test struct {
input, expected string
}
data := []test{
{"/", ""},
{"", ""},
{"/content", ""},
{"content/", ""},
{"/content/", ""}, // /content/ is a special case. It will never be the section
{"/blog", ""},
{"/blog/", "blog"},
{"blog", ""},
{"content/blog", ""},
{"/content/blog/", "blog"},
{"/content/blog", ""}, // Lack of trailing slash indicates 'blog' is not a directory.
{"content/blog/", "blog"},
{"/contents/myblog/", "contents"},
{"/contents/yourblog", "contents"},
{"/contents/ourblog/", "contents"},
{"/content/myblog/", "myblog"},
{"/content/yourblog", ""},
{"/content/ourblog/", "ourblog"},
}
for i, d := range data {
expected := GuessSection(filepath.FromSlash(d.input))
if d.expected != expected {
t.Errorf("Test %d failed. Expected %q got %q", i, d.expected, expected)
}
}
}
func TestPathPrep(t *testing.T) {
}

36
helpers/pathspec.go
View File

@@ -15,6 +15,7 @@ package helpers
import (
"fmt"
"strings"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/hugofs"
@@ -31,6 +32,7 @@ type PathSpec struct {
canonifyURLs bool
language *Language
//StatsCounter *siteSta
// pagination path handling
paginatePath string
@@ -38,10 +40,12 @@ type PathSpec struct {
theme string
// Directories
contentDir string
themesDir string
layoutDir string
workingDir string
staticDirs []string
PublishDir string
// The PathSpec looks up its config settings in both the current language
// and then in the global Viper config.
@@ -52,6 +56,8 @@ type PathSpec struct {
defaultContentLanguage string
multilingual bool
ProcessingStats *ProcessingStats
// The file systems to use
Fs *hugofs.Fs
@@ -79,6 +85,11 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) {
staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...)
}
var lang string
if l, ok := cfg.(*Language); ok {
lang = l.Lang
}
ps := &PathSpec{
Fs: fs,
Cfg: cfg,
@@ -91,13 +102,23 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) {
defaultContentLanguage: cfg.GetString("defaultContentLanguage"),
paginatePath: cfg.GetString("paginatePath"),
BaseURL: baseURL,
contentDir: cfg.GetString("contentDir"),
themesDir: cfg.GetString("themesDir"),
layoutDir: cfg.GetString("layoutDir"),
workingDir: cfg.GetString("workingDir"),
staticDirs: staticDirs,
theme: cfg.GetString("theme"),
ProcessingStats: NewProcessingStats(lang),
}
publishDir := ps.AbsPathify(cfg.GetString("publishDir")) + FilePathSeparator
// If root, remove the second '/'
if publishDir == "//" {
publishDir = FilePathSeparator
}
ps.PublishDir = publishDir
if language, ok := cfg.(*Language); ok {
ps.language = language
}
@@ -129,6 +150,11 @@ func (p *PathSpec) PaginatePath() string {
return p.paginatePath
}
// ContentDir returns the configured workingDir.
func (p *PathSpec) ContentDir() string {
return p.contentDir
}
// WorkingDir returns the configured workingDir.
func (p *PathSpec) WorkingDir() string {
return p.workingDir
@@ -153,3 +179,13 @@ func (p *PathSpec) Theme() string {
func (p *PathSpec) ThemesDir() string {
return p.themesDir
}
// PermalinkForBaseURL creates a permalink from the given link and baseURL.
func (p *PathSpec) PermalinkForBaseURL(link, baseURL string) string {
link = strings.TrimPrefix(link, "/")
if !strings.HasSuffix(baseURL, "/") {
baseURL += "/"
}
return baseURL + link
}

116
helpers/processing_stats.go Normal file
View File

@@ -0,0 +1,116 @@
// Copyright 2017 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package helpers
import (
"io"
"strconv"
"sync/atomic"
"github.com/olekukonko/tablewriter"
)
type ProcessingStats struct {
Name string
Pages uint64
PaginatorPages uint64
Static uint64
ProcessedImages uint64
Files uint64
Aliases uint64
Sitemaps uint64
Cleaned uint64
}
type processingStatsTitleVal struct {
name string
val uint64
}
func (s *ProcessingStats) toVals() []processingStatsTitleVal {
return []processingStatsTitleVal{
processingStatsTitleVal{"Pages", s.Pages},
processingStatsTitleVal{"Paginator pages", s.PaginatorPages},
processingStatsTitleVal{"Non-page files", s.Files},
processingStatsTitleVal{"Static files", s.Static},
processingStatsTitleVal{"Processed images", s.ProcessedImages},
processingStatsTitleVal{"Aliases", s.Aliases},
processingStatsTitleVal{"Sitemaps", s.Sitemaps},
processingStatsTitleVal{"Cleaned", s.Cleaned},
}
}
func NewProcessingStats(name string) *ProcessingStats {
return &ProcessingStats{Name: name}
}
func (s *ProcessingStats) Incr(counter *uint64) {
atomic.AddUint64(counter, 1)
}
func (s *ProcessingStats) Add(counter *uint64, amount int) {
atomic.AddUint64(counter, uint64(amount))
}
func (s *ProcessingStats) Table(w io.Writer) {
titleVals := s.toVals()
data := make([][]string, len(titleVals))
for i, tv := range titleVals {
data[i] = []string{tv.name, strconv.Itoa(int(tv.val))}
}
table := tablewriter.NewWriter(w)
table.AppendBulk(data)
table.SetHeader([]string{"", s.Name})
table.SetBorder(false)
table.Render()
}
func ProcessingStatsTable(w io.Writer, stats ...*ProcessingStats) {
names := make([]string, len(stats)+1)
var data [][]string
for i := 0; i < len(stats); i++ {
stat := stats[i]
names[i+1] = stat.Name
titleVals := stat.toVals()
if i == 0 {
data = make([][]string, len(titleVals))
}
for j, tv := range titleVals {
if i == 0 {
data[j] = []string{tv.name, strconv.Itoa(int(tv.val))}
} else {
data[j] = append(data[j], strconv.Itoa(int(tv.val)))
}
}
}
table := tablewriter.NewWriter(w)
table.AppendBulk(data)
table.SetHeader(names)
table.SetBorder(false)
table.Render()
}