Add a remote retry for resources.Get

cache/filecache/filecache.go

@@ -48,6 +48,9 @@ type Cache struct {
 	// 0 is effectively turning this cache off.
 	maxAge time.Duration
 
+	// Number of retries on create error.
+	retries int
+
 	// When set, we just remove this entire root directory on expiration.
 	pruneAllRootDir string
 
@@ -84,11 +87,12 @@ type ItemInfo struct {
 }
 
 // NewCache creates a new file cache with the given filesystem and max age.
-func NewCache(fs afero.Fs, maxAge time.Duration, pruneAllRootDir string) *Cache {
+func NewCache(fs afero.Fs, maxAge time.Duration, retries int, pruneAllRootDir string) *Cache {
 	return &Cache{
 		Fs:              fs,
 		nlocker:         &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})},
 		maxAge:          maxAge,
+		retries:         retries,
 		pruneAllRootDir: pruneAllRootDir,
 	}
 }
@@ -175,7 +179,19 @@ func (c *Cache) GetOrCreate(id string, create func() (io.ReadCloser, error)) (It
 		return info, r, nil
 	}
 
-	r, err := create()
+	var (
+		r   io.ReadCloser
+		err error
+	)
+
+	for i := -1; i < c.retries; i++ {
+		r, err = create()
+		if err == nil || c.retries == 0 {
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+
 	if err != nil {
 		return info, nil, err
 	}
@@ -206,7 +222,19 @@ func (c *Cache) GetOrCreateBytes(id string, create func() ([]byte, error)) (Item
 		return info, b, err
 	}
 
-	b, err := create()
+	var (
+		b   []byte
+		err error
+	)
+
+	for i := -1; i < c.retries; i++ {
+		b, err = create()
+		if err == nil || c.retries == 0 {
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+
 	if err != nil {
 		return info, nil, err
 	}
@@ -360,7 +388,7 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) {
 			pruneAllRootDir = "pkg"
 		}
 
-		m[k] = NewCache(bfs, v.MaxAge, pruneAllRootDir)
+		m[k] = NewCache(bfs, v.MaxAge, v.retries, pruneAllRootDir)
 	}
 
 	return m, nil