Cache processed images by their source path

Fixes #6269

resources/image.go

@@ -24,6 +24,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"path/filepath"
 	"strings"
 	"sync"
 
@@ -319,10 +320,13 @@ func (i *imageResource) getImageMetaCacheTargetPath() string {
 
 	cfg := i.getSpec().imaging.Cfg
 	df := i.getResourcePaths().relTargetDirFile
+	if fi := i.getFileInfo(); fi != nil {
+		df.dir = filepath.Dir(fi.Meta().Path())
+	}
 	p1, _ := helpers.FileAndExt(df.file)
 	h, _ := i.hash()
 	idStr := internal.HashString(h, i.size(), imageMetaVersionNumber, cfg)
-	return path.Join(df.dir, fmt.Sprintf("%s%s.json", p1, idStr))
+	return path.Join(df.dir, fmt.Sprintf("%s_%s.json", p1, idStr))
 }
 
 func (i *imageResource) relTargetPathFromConfig(conf images.ImageConfig) dirFile {

resources/image_cache.go

@@ -73,11 +73,18 @@ func (c *imageCache) getOrCreate(
 	parent *imageResource, conf images.ImageConfig,
 	createImage func() (*imageResource, image.Image, error)) (*resourceAdapter, error) {
 	relTarget := parent.relTargetPathFromConfig(conf)
-	key := parent.relTargetPathForRel(relTarget.path(), false, false, false)
+	memKey := parent.relTargetPathForRel(relTarget.path(), false, false, false)
+
+	// For the file cache we want to generate and store it once if possible.
+	fileKeyPath := relTarget
+	if fi := parent.root.getFileInfo(); fi != nil {
+		fileKeyPath.dir = filepath.ToSlash(filepath.Dir(fi.Meta().Path()))
+	}
+	fileKey := fileKeyPath.path()
 
 	// First check the in-memory store, then the disk.
 	c.mu.RLock()
-	cachedImage, found := c.store[key]
+	cachedImage, found := c.store[memKey]
 	c.mu.RUnlock()
 
 	if found {
@@ -133,7 +140,7 @@ func (c *imageCache) getOrCreate(
 	//  but the count of processed image variations for this site.
 	c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages)
 
-	_, err := c.fileCache.ReadOrCreate(key, read, create)
+	_, err := c.fileCache.ReadOrCreate(fileKey, read, create)
 	if err != nil {
 		return nil, err
 	}
@@ -142,13 +149,13 @@ func (c *imageCache) getOrCreate(
 	img.setSourceFs(c.fileCache.Fs)
 
 	c.mu.Lock()
-	if cachedImage, found = c.store[key]; found {
+	if cachedImage, found = c.store[memKey]; found {
 		c.mu.Unlock()
 		return cachedImage, nil
 	}
 
 	imgAdapter := newResourceAdapter(parent.getSpec(), true, img)
-	c.store[key] = imgAdapter
+	c.store[memKey] = imgAdapter
 	c.mu.Unlock()
 
 	return imgAdapter, nil

resources/image_test.go

@@ -18,6 +18,7 @@ import (
 	"math/big"
 	"math/rand"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -90,17 +91,16 @@ func TestImageTransformBasic(t *testing.T) {
 	resized0x, err := image.Resize("x200")
 	c.Assert(err, qt.IsNil)
 	assertWidthHeight(resized0x, 320, 200)
-	assertFileCache(c, fileCache, resized0x.RelPermalink(), 320, 200)
+	assertFileCache(c, fileCache, path.Base(resized0x.RelPermalink()), 320, 200)
 
 	resizedx0, err := image.Resize("200x")
 	c.Assert(err, qt.IsNil)
 	assertWidthHeight(resizedx0, 200, 125)
-	assertFileCache(c, fileCache, resizedx0.RelPermalink(), 200, 125)
+	assertFileCache(c, fileCache, path.Base(resizedx0.RelPermalink()), 200, 125)
 
 	resizedAndRotated, err := image.Resize("x200 r90")
 	c.Assert(err, qt.IsNil)
 	assertWidthHeight(resizedAndRotated, 125, 200)
-	assertFileCache(c, fileCache, resizedAndRotated.RelPermalink(), 125, 200)
 
 	assertWidthHeight(resized, 300, 200)
 	c.Assert(resized.RelPermalink(), qt.Equals, "/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_300x200_resize_q68_linear.jpg")
@@ -121,19 +121,16 @@ func TestImageTransformBasic(t *testing.T) {
 	c.Assert(err, qt.IsNil)
 	c.Assert(filled.RelPermalink(), qt.Equals, "/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_bottomleft.jpg")
 	assertWidthHeight(filled, 200, 100)
-	assertFileCache(c, fileCache, filled.RelPermalink(), 200, 100)
 
 	smart, err := image.Fill("200x100 smart")
 	c.Assert(err, qt.IsNil)
 	c.Assert(smart.RelPermalink(), qt.Equals, fmt.Sprintf("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_smart%d.jpg", 1))
 	assertWidthHeight(smart, 200, 100)
-	assertFileCache(c, fileCache, smart.RelPermalink(), 200, 100)
 
 	// Check cache
 	filledAgain, err := image.Fill("200x100 bottomLeft")
 	c.Assert(err, qt.IsNil)
 	c.Assert(filled, eq, filledAgain)
-	assertFileCache(c, fileCache, filledAgain.RelPermalink(), 200, 100)
 }
 
 // https://github.com/gohugoio/hugo/issues/4261
@@ -294,7 +291,6 @@ func TestImageResizeInSubPath(t *testing.T) {
 	c := qt.New(t)
 
 	image := fetchImage(c, "sub/gohugoio2.png")
-	fileCache := image.(specProvider).getSpec().FileCaches.ImageCache().Fs
 
 	c.Assert(image.MediaType(), eq, media.PNGType)
 	c.Assert(image.RelPermalink(), qt.Equals, "/a/sub/gohugoio2.png")
@@ -306,7 +302,6 @@ func TestImageResizeInSubPath(t *testing.T) {
 	c.Assert(resized.RelPermalink(), qt.Equals, "/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png")
 	c.Assert(resized.Width(), qt.Equals, 101)
 
-	assertFileCache(c, fileCache, resized.RelPermalink(), 101, 101)
 	publishedImageFilename := filepath.Clean(resized.RelPermalink())
 
 	spec := image.(specProvider).getSpec()
@@ -321,7 +316,6 @@ func TestImageResizeInSubPath(t *testing.T) {
 	c.Assert(err, qt.IsNil)
 	c.Assert(resizedAgain.RelPermalink(), qt.Equals, "/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png")
 	c.Assert(resizedAgain.Width(), qt.Equals, 101)
-	assertFileCache(c, fileCache, resizedAgain.RelPermalink(), 101, 101)
 	assertImageFile(c, image.(specProvider).getSpec().BaseFs.PublishFs, publishedImageFilename, 101, 101)
 }
 
@@ -529,7 +523,7 @@ func TestImageOperationsGolden(t *testing.T) {
 		return
 	}
 
-	dir1 := filepath.Join(workDir, "resources/_gen/images/a")
+	dir1 := filepath.Join(workDir, "resources/_gen/images")
 	dir2 := filepath.FromSlash("testdata/golden")
 
 	// The two dirs above should now be the same.

resources/resource.go

@@ -22,6 +22,8 @@ import (
 	"path/filepath"
 	"sync"
 
+	"github.com/gohugoio/hugo/hugofs"
+
 	"github.com/gohugoio/hugo/media"
 	"github.com/gohugoio/hugo/source"
 
@@ -172,6 +174,7 @@ type fileInfo interface {
 	getSourceFilename() string
 	setSourceFilename(string)
 	setSourceFs(afero.Fs)
+	getFileInfo() hugofs.FileMetaInfo
 	hash() (string, error)
 	size() int
 }
@@ -537,7 +540,7 @@ type resourceFileInfo struct {
 	// the path to the file on the real filesystem.
 	sourceFilename string
 
-	fi os.FileInfo
+	fi hugofs.FileMetaInfo
 
 	// A hash of the source content. Is only calculated in caching situations.
 	h *resourceHash
@@ -555,6 +558,10 @@ func (fi *resourceFileInfo) ReadSeekCloser() (hugio.ReadSeekCloser, error) {
 	return f, nil
 }
 
+func (fi *resourceFileInfo) getFileInfo() hugofs.FileMetaInfo {
+	return fi.fi
+}
+
 func (fi *resourceFileInfo) getSourceFilename() string {
 	return fi.sourceFilename
 }

resources/resource_spec.go

@@ -22,6 +22,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/gohugoio/hugo/hugofs"
+
 	"github.com/gohugoio/hugo/helpers"
 
 	"github.com/gohugoio/hugo/cache/filecache"
@@ -194,8 +196,13 @@ func (r *Spec) newGenericResourceWithBase(
 		relTargetDirFile:   dirFile{dir: fpath, file: fname},
 	}
 
+	var fim hugofs.FileMetaInfo
+	if osFileInfo != nil {
+		fim = osFileInfo.(hugofs.FileMetaInfo)
+	}
+
 	gfi := &resourceFileInfo{
-		fi:                   osFileInfo,
+		fi:                   fim,
 		openReadSeekerCloser: openReadSeekerCloser,
 		sourceFs:             sourceFs,
 		sourceFilename:       sourceFilename,

resources/resource_transformers/htesting/testhelpers.go

@@ -39,7 +39,7 @@ func NewTestResourceSpec() (*resources.Spec, error) {
 
 	cfg.Set("imaging", imagingCfg)
 
-	fs := hugofs.NewMem(cfg)
+	fs := hugofs.NewFrom(hugofs.NewBaseFileDecorator(afero.NewMemMapFs()), cfg)
 
 	s, err := helpers.NewPathSpec(fs, cfg, nil)
 	if err != nil {

resources/testhelpers_test.go

@@ -66,6 +66,8 @@ func newTestResourceSpec(desc specDescriptor) *Spec {
 		afs = afero.NewMemMapFs()
 	}
 
+	afs = hugofs.NewBaseFileDecorator(afs)
+
 	c := desc.c
 
 	cfg := createTestCfg()
@@ -118,7 +120,7 @@ func newTestResourceOsFs(c *qt.C) (*Spec, string) {
 
 	cfg.Set("workingDir", workDir)
 
-	fs := hugofs.NewFrom(hugofs.Os, cfg)
+	fs := hugofs.NewFrom(hugofs.NewBaseFileDecorator(hugofs.Os), cfg)
 	fs.Destination = &afero.MemMapFs{}
 
 	s, err := helpers.NewPathSpec(fs, cfg, nil)