Add a consolidated file cache

This commits reworks how file caching is performed in Hugo. Now there is only one way, and it can be configured.

This is the default configuration:

```toml
[caches]
[caches.getjson]
dir = ":cacheDir"
maxAge = -1
[caches.getcsv]
dir = ":cacheDir"
maxAge = -1
[caches.images]
dir = ":resourceDir/_gen"
maxAge = -1
[caches.assets]
dir = ":resourceDir/_gen"
maxAge = -1
```

You can override any of these cache setting in your own `config.toml`.

The placeholders explained:

`:cacheDir`: This is the value of the `cacheDir` config option if set (can also be set via OS env variable `HUGO_CACHEDIR`). It will fall back to `/opt/build/cache/hugo_cache/` on Netlify, or a `hugo_cache` directory below the OS temp dir for the others.
`:resourceDir`: This is the value of the `resourceDir` config option.

`maxAge` is the time in seconds before a cache entry will be evicted, -1 means forever and 0 effectively turns that particular cache off.

This means that if you run your builds on Netlify, all caches configured with `:cacheDir` will be saved and restored on the next build. For other CI vendors, please read their documentation. For an CircleCI example, see 6c3960a8f4/.circleci/config.yml

Fixes #5404
This commit is contained in:
Bjørn Erik Pedersen
2018-11-08 10:24:13 +01:00
parent 7d78a2afd3
commit f7aeaa6129
26 changed files with 1192 additions and 543 deletions

View File

@@ -24,6 +24,8 @@ import (
"strings"
"unicode"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/common/hugio"
_errors "github.com/pkg/errors"
"github.com/spf13/afero"
@@ -575,6 +577,59 @@ func OpenFileForWriting(fs afero.Fs, filename string) (afero.File, error) {
return f, err
}
// GetCacheDir returns a cache dir from the given filesystem and config.
// The dir will be created if it does not exist.
func GetCacheDir(fs afero.Fs, cfg config.Provider) (string, error) {
cacheDir := getCacheDir(cfg)
if cacheDir != "" {
exists, err := DirExists(cacheDir, fs)
if err != nil {
return "", err
}
if !exists {
err := fs.MkdirAll(cacheDir, 0777) // Before umask
if err != nil {
return "", _errors.Wrap(err, "failed to create cache dir")
}
}
return cacheDir, nil
}
// Fall back to a cache in /tmp.
return GetTempDir("hugo_cache", fs), nil
}
func getCacheDir(cfg config.Provider) string {
// Always use the cacheDir config if set.
cacheDir := cfg.GetString("cacheDir")
if len(cacheDir) > 1 {
return addTrailingFileSeparator(cacheDir)
}
// Both of these are fairly distinctive OS env keys used by Netlify.
if os.Getenv("DEPLOY_PRIME_URL") != "" && os.Getenv("PULL_REQUEST") != "" {
// Netlify's cache behaviour is not documented, the currently best example
// is this project:
// https://github.com/philhawksworth/content-shards/blob/master/gulpfile.js
return "/opt/build/cache/hugo_cache/"
}
// This will fall back to an hugo_cache folder in the tmp dir, which should work fine for most CI
// providers. See this for a working CircleCI setup:
// https://github.com/bep/hugo-sass-test/blob/6c3960a8f4b90e8938228688bc49bdcdd6b2d99e/.circleci/config.yml
// If not, they can set the HUGO_CACHEDIR environment variable or cacheDir config key.
return ""
}
func addTrailingFileSeparator(s string) string {
if !strings.HasSuffix(s, FilePathSeparator) {
s = s + FilePathSeparator
}
return s
}
// GetTempDir returns a temporary directory with the given sub path.
func GetTempDir(subPath string, fs afero.Fs) string {
return afero.GetTempDir(fs, subPath)