erofs: allow multiple layer content cache directories

The layer content cache was a single directory, so every source of pre-converted blobs had to be merged into one tree. A shared read-only fleet cache, a host-local cache, and a per-image warm cache could not coexist. Replace layer_content_cache with layer_content_caches, a list. Each directory is checked in order and the first hit is staged into the snapshot; a layer found in none of them falls back to the normal download-and-convert path. Cache directories are no longer required to exist at startup. A missing one is indistinguishable from an empty one at lookup time (both are simply a miss), and it may well be mounted after the daemon starts, so the only check left is that each path is absolute -- a relative one would be symlinked into the snapshot dir and dangle. Signed-off-by: Maksym Pavlenko <pavlenko.maksym@gmail.com>
2026-08-08 17:11:15 +00:00 · 2026-07-30 15:38:50 -07:00
parent 1a34bf320a
commit 7df6bb0a67
4 changed files with 156 additions and 89 deletions
--- a/cmd/ctr/commands/images/build_erofs_cache.go
+++ b/cmd/ctr/commands/images/build_erofs_cache.go
@@ -44,7 +44,7 @@ var buildErofsCacheCommand = &cli.Command{
 	Description: `Convert each layer of an already-pulled image into a directory of
 diffID-keyed erofs blobs (<cache_dir>/<algorithm>/<xx>/<hex>.erofs, where <xx> is
 the first two characters of <hex>) for the erofs
-snapshotter's layer_content_cache. Layers are read from the content store; no
+snapshotter's layer_content_caches. Layers are read from the content store; no
 converted image is produced. The directory can then be synced to the read-only
 location the fleet mounts. Requires mkfs.erofs.

--- a/plugins/snapshots/erofs/erofs.go
+++ b/plugins/snapshots/erofs/erofs.go
@@ -52,12 +52,13 @@ type SnapshotterConfig struct {
 	remapIDs    bool
 	// dmverityMode controls dm-verity behavior: "auto" (use if .dmverity exists), "on" (require .dmverity), "off" (disable)
 	dmverityMode string
-	// layerContentCache is a directory of pre-converted, diffID-keyed erofs
-	// layer blobs. When set and an unpacked layer's blob is present, the
-	// snapshotter stages the blob (as a symlink) into the active snapshot,
-	// skipping the download and tar->erofs conversion. Only parentless Prepares
-	// can be served. Empty disables the feature.
-	layerContentCache string
+	// layerContentCaches lists directories of pre-converted, diffID-keyed erofs
+	// layer blobs. Each is checked one by one; the first hit is staged into the
+	// snapshot (symlinked) instead of downloading and converting the layer. A
+	// directory that doesn't exist is treated as a cache miss. Layers missing
+	// from all of them are converted normally. Only parentless Prepares can be
+	// served.
+	layerContentCaches []string
 }

 // Opt is an option to configure the erofs snapshotter
@@ -105,13 +106,13 @@ func WithRemapIDs() Opt {
 	}
 }

-// WithLayerContentCache configures a read-only directory of pre-converted,
+// WithLayerContentCaches configures read-only directories of pre-converted,
 // diffID-keyed erofs layer blobs that the snapshotter sources layers from on
-// pull instead of downloading and converting them. See the layerContentCache
+// pull instead of downloading and converting them. See the layerContentCaches
 // field for details.
-func WithLayerContentCache(path string) Opt {
+func WithLayerContentCaches(paths ...string) Opt {
 	return func(config *SnapshotterConfig) {
-		config.layerContentCache = path
+		config.layerContentCaches = paths
 	}
 }

@@ -122,16 +123,16 @@ type MetaStore interface {
 }

 type snapshotter struct {
-	root              string
-	ms                MetaStore
-	ovlOptions        []string
-	enableFsverity    bool
-	setImmutable      bool
-	defaultWritable   int64
-	blockMode         bool
-	remapIDs          bool
-	dmverityMode      string
-	layerContentCache string
+	root               string
+	ms                 MetaStore
+	ovlOptions         []string
+	enableFsverity     bool
+	setImmutable       bool
+	defaultWritable    int64
+	blockMode          bool
+	remapIDs           bool
+	dmverityMode       string
+	layerContentCaches []string
 }

 // NewSnapshotter returns a Snapshotter which uses EROFS+OverlayFS. The layers
@@ -177,13 +178,25 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) {
 	// so fsverity and IMMUTABLE_FL can't be applied without mutating that blob out
 	// from under other snapshots. Reject them explicitly instead of silently
 	// skipping; dm-verity is the cache's integrity mechanism.
-	if config.layerContentCache != "" {
+	if len(config.layerContentCaches) > 0 {
 		if config.enableFsverity {
-			return nil, fmt.Errorf("enable_fsverity is incompatible with layer_content_cache; use dm-verity for cache integrity")
+			return nil, fmt.Errorf("enable_fsverity is incompatible with layer_content_caches; use dm-verity for cache integrity")
 		}
 		if config.setImmutable {
-			return nil, fmt.Errorf("set_immutable is incompatible with layer_content_cache")
+			return nil, fmt.Errorf("set_immutable is incompatible with layer_content_caches")
 		}
+
+		// A cache dir is symlinked into snapshots, so a relative one would resolve
+		// against the snapshot dir and dangle. The check is only lexical: dirs are
+		// not required to exist, as a missing one just yields a cache miss and may
+		// well be provisioned after startup.
+		for _, dir := range config.layerContentCaches {
+			if !filepath.IsAbs(dir) {
+				return nil, fmt.Errorf("layer_content_caches %q must be an absolute path", dir)
+			}
+		}
+
+		log.L.WithField("dirs", config.layerContentCaches).Info("erofs layer content cache enabled")
 	}

 	// Check fsverity support if enabled
@@ -202,24 +215,6 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) {
 		return nil, fmt.Errorf("setting IMMUTABLE_FL is only supported on Linux")
 	}

-	// Resolve the cache dir to an absolute path so materialized layer blobs are
-	// absolute symlinks, independent of the process working directory, and verify
-	// it exists and is a directory so misconfiguration fails fast at startup.
-	if config.layerContentCache != "" {
-		abs, err := filepath.Abs(config.layerContentCache)
-		if err != nil {
-			return nil, fmt.Errorf("failed to resolve layer_content_cache path %q: %w", config.layerContentCache, err)
-		}
-		fi, err := os.Stat(abs)
-		if err != nil {
-			return nil, fmt.Errorf("failed to access layer_content_cache %q: %w", abs, err)
-		}
-		if !fi.IsDir() {
-			return nil, fmt.Errorf("layer_content_cache %q is not a directory", abs)
-		}
-		config.layerContentCache = abs
-	}
-
 	ms, err := storage.NewMetaStore(filepath.Join(root, "metadata.db"))
 	if err != nil {
 		return nil, err
@@ -230,16 +225,16 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) {
 	}

 	return &snapshotter{
-		root:              root,
-		ms:                ms,
-		ovlOptions:        config.ovlOptions,
-		enableFsverity:    config.enableFsverity,
-		setImmutable:      config.setImmutable,
-		defaultWritable:   config.defaultSize,
-		blockMode:         config.defaultSize > 0,
-		remapIDs:          config.remapIDs,
-		dmverityMode:      config.dmverityMode,
-		layerContentCache: config.layerContentCache,
+		root:               root,
+		ms:                 ms,
+		ovlOptions:         config.ovlOptions,
+		enableFsverity:     config.enableFsverity,
+		setImmutable:       config.setImmutable,
+		defaultWritable:    config.defaultSize,
+		blockMode:          config.defaultSize > 0,
+		remapIDs:           config.remapIDs,
+		dmverityMode:       config.dmverityMode,
+		layerContentCaches: config.layerContentCaches,
 	}, nil
 }

@@ -700,21 +695,17 @@ func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...s
 	return s.createSnapshot(ctx, snapshots.KindActive, key, parent, opts)
 }

-// cacheBlobPath returns the expected path of the cached erofs blob for a diffID.
-func (s *snapshotter) cacheBlobPath(diffID digest.Digest) string {
-	return erofsutils.CacheBlobPath(s.layerContentCache, diffID)
-}
-
 // lookupCache returns the absolute path of the cached erofs blob that can serve
-// the layer being prepared, or "" on a miss. It gates on: the cache being
-// configured, the Prepare being an image-layer extraction (carries the
-// snapshot.ref and diff-id labels), and the diffID blob being present. Misses
-// (cache disabled, non-extraction Prepare, missing entries, unreadable cache dirs
-// such as a FUSE mount being down, malformed labels) all return "" so pulls keep
-// working. Any dm-verity sidecar is derived from the blob path (via
-// dmverity.MetadataPath) when the blob is materialized (prepareDirectory).
+// the layer being prepared, or "" on a miss. It gates on: at least one cache
+// being configured, the Prepare being an image-layer extraction (carries the
+// snapshot.ref and diff-id labels), and the diffID blob being present. Caches
+// are checked one by one and the first hit wins. Every miss (cache disabled,
+// non-extraction Prepare, absent or unreadable cache dir, missing entry,
+// malformed labels) returns "" so pulls keep working. Any dm-verity sidecar is
+// derived from the blob path (via dmverity.MetadataPath) when the blob is
+// staged (prepareDirectory).
 func (s *snapshotter) lookupCache(ctx context.Context, opts ...snapshots.Opt) string {
-	if s.layerContentCache == "" {
+	if len(s.layerContentCaches) == 0 {
 		return ""
 	}

@@ -737,18 +728,24 @@ func (s *snapshotter) lookupCache(ctx context.Context, opts ...snapshots.Opt) st
 		return ""
 	}

-	blob := s.cacheBlobPath(diffID)
-	if _, err := os.Stat(blob); err != nil {
-		if os.IsNotExist(err) {
-			log.G(ctx).WithField("blob", blob).Trace("erofs layer cache miss")
-		} else {
-			log.G(ctx).WithError(err).WithField("blob", blob).
-				Warn("erofs layer cache: failed to stat cache blob, treating as cache miss")
+	for _, dir := range s.layerContentCaches {
+		blob := erofsutils.CacheBlobPath(dir, diffID)
+		if _, err := os.Stat(blob); err != nil {
+			if !os.IsNotExist(err) {
+				// An unreadable cache (a down FUSE mount, a permission change since
+				// startup) shouldn't fail the pull or mask a hit in a later cache.
+				log.G(ctx).WithError(err).WithField("blob", blob).
+					Warn("erofs layer cache: failed to stat cache blob, skipping this cache")
+			}
+			continue
 		}
-		return ""
+		// Absolute, since the configured dirs are validated as such: the hit is
+		// symlinked into the snapshot dir, where a relative target would dangle.
+		return blob
 	}

-	return blob
+	log.G(ctx).WithField("diffID", diffID.String()).Trace("erofs layer cache miss")
+	return ""
 }

 func (s *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) {
--- a/plugins/snapshots/erofs/erofs_linux_test.go
+++ b/plugins/snapshots/erofs/erofs_linux_test.go
@@ -990,7 +990,7 @@ func TestCacheHit(t *testing.T) {
 	diffID := digest.Digest(cacheTestDiffID)
 	blob := writeCacheBlob(t, cacheDir, diffID, []byte("fake erofs blob"))

-	s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir))
+	s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir))

 	target := cacheTestChainID
 	key := stageCacheHit(t, ctx, s, target, diffID)
@@ -1031,7 +1031,7 @@ func TestCacheSidecar(t *testing.T) {
 	require.NoError(t, os.WriteFile(dmverity.MetadataPath(blob), []byte(testDmverityMetadata), 0644))

 	// dmverity_mode defaults to "auto": use the sidecar if present.
-	s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir))
+	s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir))

 	target := cacheTestChainID
 	key := stageCacheHit(t, ctx, s, target, diffID)
@@ -1078,7 +1078,7 @@ func TestCacheMiss(t *testing.T) {
 	})

 	t.Run("blob absent", func(t *testing.T) {
-		s := newCacheSnapshotter(t, WithLayerContentCache(t.TempDir()))
+		s := newCacheSnapshotter(t, WithLayerContentCaches(t.TempDir()))
 		mounts, err := s.Prepare(ctx, "extract-1 "+target, "", extractionOpt(target, diffID))
 		assertFellThrough(t, s, mounts, err, false)
 	})
@@ -1086,7 +1086,7 @@ func TestCacheMiss(t *testing.T) {
 	t.Run("no extraction labels", func(t *testing.T) {
 		cacheDir := t.TempDir()
 		writeCacheBlob(t, cacheDir, diffID, []byte("blob"))
-		s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir))
+		s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir))
 		// A container-rootfs Prepare carries no snapshot.ref/diff-id labels.
 		mounts, err := s.Prepare(ctx, "container-rootfs", "")
 		require.NoError(t, err)
@@ -1096,7 +1096,7 @@ func TestCacheMiss(t *testing.T) {
 	t.Run("view is never short-circuited", func(t *testing.T) {
 		cacheDir := t.TempDir()
 		writeCacheBlob(t, cacheDir, diffID, []byte("blob"))
-		s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir))
+		s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir))
 		// Even with matching labels and a cached blob, a View must not commit.
 		// It's read-only, but via the KindView roFlag in mounts(), not the cache.
 		mounts, err := s.View(ctx, "view-1", "", extractionOpt(target, diffID))
@@ -1122,7 +1122,7 @@ func TestCacheParentedPrepare(t *testing.T) {
 	writeCacheBlob(t, cacheDir, parentDiffID, []byte("fake parent blob"))
 	writeCacheBlob(t, cacheDir, childDiffID, []byte("fake child blob"))

-	s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir))
+	s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir))

 	// The first layer has no parent, so it is served from the cache as usual.
 	require.NoError(t, s.Commit(ctx, parentChain, stageCacheHit(t, ctx, s, parentChain, parentDiffID)))
@@ -1152,7 +1152,7 @@ func TestCacheRemove(t *testing.T) {
 	sidecar := dmverity.MetadataPath(blob)
 	require.NoError(t, os.WriteFile(sidecar, []byte(testDmverityMetadata), 0644))

-	s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir))
+	s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir))

 	target := cacheTestChainID
 	key := stageCacheHit(t, ctx, s, target, diffID)
@@ -1189,7 +1189,7 @@ func TestCacheDmverity(t *testing.T) {
 		blob := writeCacheBlob(t, cacheDir, diffID, []byte("fake erofs blob"))
 		require.NoError(t, os.WriteFile(dmverity.MetadataPath(blob), []byte(testDmverityMetadata), 0644))

-		s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir), WithDmverityMode("on"))
+		s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir), WithDmverityMode("on"))
 		key := stageCacheHit(t, ctx, s, target, diffID)

 		_, err := os.Stat(dmverity.MetadataPath(s.layerBlobPath(snapshotID(t, ctx, s, key))))
@@ -1200,7 +1200,7 @@ func TestCacheDmverity(t *testing.T) {
 		cacheDir := t.TempDir()
 		writeCacheBlob(t, cacheDir, diffID, []byte("fake erofs blob")) // no sidecar

-		s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir), WithDmverityMode("on"))
+		s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir), WithDmverityMode("on"))

 		// dmverity_mode=on requires a sidecar; a cache entry without one is a hard
 		// error rather than a silent fallback.
@@ -1211,3 +1211,71 @@ func TestCacheDmverity(t *testing.T) {
 		assert.Error(t, err, "no snapshot should be committed on failure")
 	})
 }
+
+// TestCacheMultipleDirs covers the layer_content_caches search path: directories
+// are searched in configured order, the first hit wins, later caches are reached
+// when earlier ones lack the blob, and a miss in every cache falls through to a
+// normal writable snapshot.
+func TestCacheMultipleDirs(t *testing.T) {
+	ctx := namespaces.WithNamespace(context.Background(), "test")
+	diffID := digest.Digest(cacheTestDiffID)
+	target := cacheTestChainID
+
+	// stagedBlob returns the cache blob the snapshot's layer.erofs symlink points
+	// at, i.e. which of the configured caches actually served the layer.
+	stagedBlob := func(t *testing.T, s *snapshotter, key string) string {
+		t.Helper()
+		dst, err := os.Readlink(s.layerBlobPath(snapshotID(t, ctx, s, key)))
+		require.NoError(t, err)
+		return dst
+	}
+
+	t.Run("first cache with the blob wins", func(t *testing.T) {
+		first, second := t.TempDir(), t.TempDir()
+		// Both caches hold the diffID; the earlier one must be the one used.
+		firstBlob := writeCacheBlob(t, first, diffID, []byte("from first"))
+		writeCacheBlob(t, second, diffID, []byte("from second"))
+
+		s := newCacheSnapshotter(t, WithLayerContentCaches(first, second))
+		key := stageCacheHit(t, ctx, s, target, diffID)
+		assert.Equal(t, firstBlob, stagedBlob(t, s, key))
+	})
+
+	t.Run("falls through to a later cache", func(t *testing.T) {
+		empty, populated := t.TempDir(), t.TempDir()
+		blob := writeCacheBlob(t, populated, diffID, []byte("from second"))
+
+		s := newCacheSnapshotter(t, WithLayerContentCaches(empty, populated))
+		key := stageCacheHit(t, ctx, s, target, diffID)
+		assert.Equal(t, blob, stagedBlob(t, s, key))
+	})
+
+	t.Run("miss in every cache falls back to a writable snapshot", func(t *testing.T) {
+		s := newCacheSnapshotter(t, WithLayerContentCaches(
+			filepath.Join(t.TempDir(), "missing"), t.TempDir(), t.TempDir()))
+
+		mounts, err := s.Prepare(ctx, "extract-1 "+target, "", extractionOpt(target, diffID))
+		require.NoError(t, err)
+		require.NotEmpty(t, mounts)
+		for _, m := range mounts {
+			assert.False(t, m.ReadOnly(), "a miss in every cache must return writable mounts")
+		}
+		_, err = s.Stat(ctx, target)
+		assert.Error(t, err, "target chainID must not be committed on a miss")
+	})
+}
+
+// TestCacheDirMustBeAbsolute covers the one thing NewSnapshotter checks about a
+// configured cache dir: it must be absolute, since a relative one would be
+// symlinked into the snapshot dir and dangle. The empty string is covered by the
+// same check, which otherwise resolves to the daemon's working directory.
+func TestCacheDirMustBeAbsolute(t *testing.T) {
+	requireErofs(t)
+
+	for _, dir := range []string{"relative-cache", "./cache", "", "a/b"} {
+		t.Run(fmt.Sprintf("%q", dir), func(t *testing.T) {
+			_, err := NewSnapshotter(t.TempDir(), WithLayerContentCaches(dir))
+			assert.ErrorContains(t, err, "must be an absolute path")
+		})
+	}
+}
--- a/plugins/snapshots/erofs/plugin/plugin.go
+++ b/plugins/snapshots/erofs/plugin/plugin.go
@@ -57,14 +57,16 @@ type Config struct {
 	// Linux only
 	DmverityMode string `toml:"dmverity_mode"`

-	// LayerContentCache is a directory of pre-converted, diffID-keyed erofs
-	// layer blobs. When set, layers already present in the cache are committed
-	// without being downloaded or converted. Empty disables the feature.
+	// LayerContentCaches lists directories of pre-converted, diffID-keyed erofs
+	// layer blobs. Each is checked one by one and the first hit is used instead
+	// of downloading and converting the layer; a directory that doesn't exist is
+	// treated as a cache miss. Layers missing from all of them are converted
+	// normally.
 	//
 	// Only layers prepared without a parent can be served from the cache. With
 	// sequential unpacking that is the first layer alone, so getting hits for a
 	// whole image needs max_concurrent_unpacks > 1, which is not the default.
-	LayerContentCache string `toml:"layer_content_cache"`
+	LayerContentCaches []string `toml:"layer_content_caches"`
 }

 func init() {
@@ -110,8 +112,8 @@ func init() {
 				opts = append(opts, erofs.WithDmverityMode(config.DmverityMode))
 			}

-			if config.LayerContentCache != "" {
-				opts = append(opts, erofs.WithLayerContentCache(config.LayerContentCache))
+			if len(config.LayerContentCaches) > 0 {
+				opts = append(opts, erofs.WithLayerContentCaches(config.LayerContentCaches...))
 			}

 			// Don't bother supporting overlay's slow_chown, only RemapIDs