dilim/containerd
1
0
Fork 0
mirror of https://github.com/containerd/containerd.git synced 2026-08-09 09:33:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13900 from mxpv/multi-cache

Browse Source 
erofs: allow multiple cache directories
This commit is contained in:
Maksym Pavlenko
2026-08-05 16:37:26 +00:00
committed by GitHub
parent d6bdba3d1b 7df6bb0a67
commit c9caf7e8cf
4 changed files with 156 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/ctr/commands/images/build_erofs_cache.go
View File

@@ -44,7 +44,7 @@ var buildErofsCacheCommand = &cli.Command{
Description: `Convert each layer of an already-pulled image into a directory of
diffID-keyed erofs blobs (<cache_dir>/<algorithm>/<xx>/<hex>.erofs, where <xx> is
the first two characters of <hex>) for the erofs
snapshotter's layer_content_cache. Layers are read from the content store; no
snapshotter's layer_content_caches. Layers are read from the content store; no
converted image is produced. The directory can then be synced to the read-only
location the fleet mounts. Requires mkfs.erofs.

143
plugins/snapshots/erofs/erofs.go
View File

@@ -52,12 +52,13 @@ type SnapshotterConfig struct {
remapIDs bool
// dmverityMode controls dm-verity behavior: "auto" (use if .dmverity exists), "on" (require .dmverity), "off" (disable)
dmverityMode string
// layerContentCache is a directory of pre-converted, diffID-keyed erofs
// layer blobs. When set and an unpacked layer's blob is present, the
// snapshotter stages the blob (as a symlink) into the active snapshot,
// skipping the download and tar->erofs conversion. Only parentless Prepares
// can be served. Empty disables the feature.
layerContentCache string
// layerContentCaches lists directories of pre-converted, diffID-keyed erofs
// layer blobs. Each is checked one by one; the first hit is staged into the
// snapshot (symlinked) instead of downloading and converting the layer. A
// directory that doesn't exist is treated as a cache miss. Layers missing
// from all of them are converted normally. Only parentless Prepares can be
// served.
layerContentCaches []string
}
// Opt is an option to configure the erofs snapshotter
@@ -105,13 +106,13 @@ func WithRemapIDs() Opt {
}
}
// WithLayerContentCache configures a read-only directory of pre-converted,
// WithLayerContentCaches configures read-only directories of pre-converted,
// diffID-keyed erofs layer blobs that the snapshotter sources layers from on
// pull instead of downloading and converting them. See the layerContentCache
// pull instead of downloading and converting them. See the layerContentCaches
// field for details.
func WithLayerContentCache(path string) Opt {
func WithLayerContentCaches(paths ...string) Opt {
return func(config *SnapshotterConfig) {
config.layerContentCache = path
config.layerContentCaches = paths
}
}
@@ -122,16 +123,16 @@ type MetaStore interface {
}
type snapshotter struct {
root string
ms MetaStore
ovlOptions []string
enableFsverity bool
setImmutable bool
defaultWritable int64
blockMode bool
remapIDs bool
dmverityMode string
layerContentCache string
root string
ms MetaStore
ovlOptions []string
enableFsverity bool
setImmutable bool
defaultWritable int64
blockMode bool
remapIDs bool
dmverityMode string
layerContentCaches []string
}
// NewSnapshotter returns a Snapshotter which uses EROFS+OverlayFS. The layers
@@ -177,13 +178,25 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) {
// so fsverity and IMMUTABLE_FL can't be applied without mutating that blob out
// from under other snapshots. Reject them explicitly instead of silently
// skipping; dm-verity is the cache's integrity mechanism.
if config.layerContentCache != "" {
if len(config.layerContentCaches) > 0 {
if config.enableFsverity {
return nil, fmt.Errorf("enable_fsverity is incompatible with layer_content_cache; use dm-verity for cache integrity")
return nil, fmt.Errorf("enable_fsverity is incompatible with layer_content_caches; use dm-verity for cache integrity")
}
if config.setImmutable {
return nil, fmt.Errorf("set_immutable is incompatible with layer_content_cache")
return nil, fmt.Errorf("set_immutable is incompatible with layer_content_caches")
}
// A cache dir is symlinked into snapshots, so a relative one would resolve
// against the snapshot dir and dangle. The check is only lexical: dirs are
// not required to exist, as a missing one just yields a cache miss and may
// well be provisioned after startup.
for _, dir := range config.layerContentCaches {
if !filepath.IsAbs(dir) {
return nil, fmt.Errorf("layer_content_caches %q must be an absolute path", dir)
}
}
log.L.WithField("dirs", config.layerContentCaches).Info("erofs layer content cache enabled")
}
// Check fsverity support if enabled
@@ -202,24 +215,6 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) {
return nil, fmt.Errorf("setting IMMUTABLE_FL is only supported on Linux")
}
// Resolve the cache dir to an absolute path so materialized layer blobs are
// absolute symlinks, independent of the process working directory, and verify
// it exists and is a directory so misconfiguration fails fast at startup.
if config.layerContentCache != "" {
abs, err := filepath.Abs(config.layerContentCache)
if err != nil {
return nil, fmt.Errorf("failed to resolve layer_content_cache path %q: %w", config.layerContentCache, err)
}
fi, err := os.Stat(abs)
if err != nil {
return nil, fmt.Errorf("failed to access layer_content_cache %q: %w", abs, err)
}
if !fi.IsDir() {
return nil, fmt.Errorf("layer_content_cache %q is not a directory", abs)
}
config.layerContentCache = abs
}
ms, err := storage.NewMetaStore(filepath.Join(root, "metadata.db"))
if err != nil {
return nil, err
@@ -230,16 +225,16 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) {
}
return &snapshotter{
root: root,
ms: ms,
ovlOptions: config.ovlOptions,
enableFsverity: config.enableFsverity,
setImmutable: config.setImmutable,
defaultWritable: config.defaultSize,
blockMode: config.defaultSize > 0,
remapIDs: config.remapIDs,
dmverityMode: config.dmverityMode,
layerContentCache: config.layerContentCache,
root: root,
ms: ms,
ovlOptions: config.ovlOptions,
enableFsverity: config.enableFsverity,
setImmutable: config.setImmutable,
defaultWritable: config.defaultSize,
blockMode: config.defaultSize > 0,
remapIDs: config.remapIDs,
dmverityMode: config.dmverityMode,
layerContentCaches: config.layerContentCaches,
}, nil
}
@@ -700,21 +695,17 @@ func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...s
return s.createSnapshot(ctx, snapshots.KindActive, key, parent, opts)
}
// cacheBlobPath returns the expected path of the cached erofs blob for a diffID.
func (s *snapshotter) cacheBlobPath(diffID digest.Digest) string {
return erofsutils.CacheBlobPath(s.layerContentCache, diffID)
}
// lookupCache returns the absolute path of the cached erofs blob that can serve
// the layer being prepared, or "" on a miss. It gates on: the cache being
// configured, the Prepare being an image-layer extraction (carries the
// snapshot.ref and diff-id labels), and the diffID blob being present. Misses
// (cache disabled, non-extraction Prepare, missing entries, unreadable cache dirs
// such as a FUSE mount being down, malformed labels) all return "" so pulls keep
// working. Any dm-verity sidecar is derived from the blob path (via
// dmverity.MetadataPath) when the blob is materialized (prepareDirectory).
// the layer being prepared, or "" on a miss. It gates on: at least one cache
// being configured, the Prepare being an image-layer extraction (carries the
// snapshot.ref and diff-id labels), and the diffID blob being present. Caches
// are checked one by one and the first hit wins. Every miss (cache disabled,
// non-extraction Prepare, absent or unreadable cache dir, missing entry,
// malformed labels) returns "" so pulls keep working. Any dm-verity sidecar is
// derived from the blob path (via dmverity.MetadataPath) when the blob is
// staged (prepareDirectory).
func (s *snapshotter) lookupCache(ctx context.Context, opts ...snapshots.Opt) string {
if s.layerContentCache == "" {
if len(s.layerContentCaches) == 0 {
return ""
}
@@ -737,18 +728,24 @@ func (s *snapshotter) lookupCache(ctx context.Context, opts ...snapshots.Opt) st
return ""
}
blob := s.cacheBlobPath(diffID)
if _, err := os.Stat(blob); err != nil {
if os.IsNotExist(err) {
log.G(ctx).WithField("blob", blob).Trace("erofs layer cache miss")
} else {
log.G(ctx).WithError(err).WithField("blob", blob).
Warn("erofs layer cache: failed to stat cache blob, treating as cache miss")
for _, dir := range s.layerContentCaches {
blob := erofsutils.CacheBlobPath(dir, diffID)
if _, err := os.Stat(blob); err != nil {
if !os.IsNotExist(err) {
// An unreadable cache (a down FUSE mount, a permission change since
// startup) shouldn't fail the pull or mask a hit in a later cache.
log.G(ctx).WithError(err).WithField("blob", blob).
Warn("erofs layer cache: failed to stat cache blob, skipping this cache")
}
continue
}
return ""
// Absolute, since the configured dirs are validated as such: the hit is
// symlinked into the snapshot dir, where a relative target would dangle.
return blob
}
return blob
log.G(ctx).WithField("diffID", diffID.String()).Trace("erofs layer cache miss")
return ""
}
func (s *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) {

86
plugins/snapshots/erofs/erofs_linux_test.go
View File

@@ -990,7 +990,7 @@ func TestCacheHit(t *testing.T) {
diffID := digest.Digest(cacheTestDiffID)
blob := writeCacheBlob(t, cacheDir, diffID, []byte("fake erofs blob"))
s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir))
s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir))
target := cacheTestChainID
key := stageCacheHit(t, ctx, s, target, diffID)
@@ -1031,7 +1031,7 @@ func TestCacheSidecar(t *testing.T) {
require.NoError(t, os.WriteFile(dmverity.MetadataPath(blob), []byte(testDmverityMetadata), 0644))
// dmverity_mode defaults to "auto": use the sidecar if present.
s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir))
s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir))
target := cacheTestChainID
key := stageCacheHit(t, ctx, s, target, diffID)
@@ -1078,7 +1078,7 @@ func TestCacheMiss(t *testing.T) {
})
t.Run("blob absent", func(t *testing.T) {
s := newCacheSnapshotter(t, WithLayerContentCache(t.TempDir()))
s := newCacheSnapshotter(t, WithLayerContentCaches(t.TempDir()))
mounts, err := s.Prepare(ctx, "extract-1 "+target, "", extractionOpt(target, diffID))
assertFellThrough(t, s, mounts, err, false)
})
@@ -1086,7 +1086,7 @@ func TestCacheMiss(t *testing.T) {
t.Run("no extraction labels", func(t *testing.T) {
cacheDir := t.TempDir()
writeCacheBlob(t, cacheDir, diffID, []byte("blob"))
s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir))
s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir))
// A container-rootfs Prepare carries no snapshot.ref/diff-id labels.
mounts, err := s.Prepare(ctx, "container-rootfs", "")
require.NoError(t, err)
@@ -1096,7 +1096,7 @@ func TestCacheMiss(t *testing.T) {
t.Run("view is never short-circuited", func(t *testing.T) {
cacheDir := t.TempDir()
writeCacheBlob(t, cacheDir, diffID, []byte("blob"))
s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir))
s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir))
// Even with matching labels and a cached blob, a View must not commit.
// It's read-only, but via the KindView roFlag in mounts(), not the cache.
mounts, err := s.View(ctx, "view-1", "", extractionOpt(target, diffID))
@@ -1122,7 +1122,7 @@ func TestCacheParentedPrepare(t *testing.T) {
writeCacheBlob(t, cacheDir, parentDiffID, []byte("fake parent blob"))
writeCacheBlob(t, cacheDir, childDiffID, []byte("fake child blob"))
s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir))
s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir))
// The first layer has no parent, so it is served from the cache as usual.
require.NoError(t, s.Commit(ctx, parentChain, stageCacheHit(t, ctx, s, parentChain, parentDiffID)))
@@ -1152,7 +1152,7 @@ func TestCacheRemove(t *testing.T) {
sidecar := dmverity.MetadataPath(blob)
require.NoError(t, os.WriteFile(sidecar, []byte(testDmverityMetadata), 0644))
s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir))
s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir))
target := cacheTestChainID
key := stageCacheHit(t, ctx, s, target, diffID)
@@ -1189,7 +1189,7 @@ func TestCacheDmverity(t *testing.T) {
blob := writeCacheBlob(t, cacheDir, diffID, []byte("fake erofs blob"))
require.NoError(t, os.WriteFile(dmverity.MetadataPath(blob), []byte(testDmverityMetadata), 0644))
s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir), WithDmverityMode("on"))
s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir), WithDmverityMode("on"))
key := stageCacheHit(t, ctx, s, target, diffID)
_, err := os.Stat(dmverity.MetadataPath(s.layerBlobPath(snapshotID(t, ctx, s, key))))
@@ -1200,7 +1200,7 @@ func TestCacheDmverity(t *testing.T) {
cacheDir := t.TempDir()
writeCacheBlob(t, cacheDir, diffID, []byte("fake erofs blob")) // no sidecar
s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir), WithDmverityMode("on"))
s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir), WithDmverityMode("on"))
// dmverity_mode=on requires a sidecar; a cache entry without one is a hard
// error rather than a silent fallback.
@@ -1211,3 +1211,71 @@ func TestCacheDmverity(t *testing.T) {
assert.Error(t, err, "no snapshot should be committed on failure")
})
}
// TestCacheMultipleDirs covers the layer_content_caches search path: directories
// are searched in configured order, the first hit wins, later caches are reached
// when earlier ones lack the blob, and a miss in every cache falls through to a
// normal writable snapshot.
func TestCacheMultipleDirs(t *testing.T) {
ctx := namespaces.WithNamespace(context.Background(), "test")
diffID := digest.Digest(cacheTestDiffID)
target := cacheTestChainID
// stagedBlob returns the cache blob the snapshot's layer.erofs symlink points
// at, i.e. which of the configured caches actually served the layer.
stagedBlob := func(t *testing.T, s *snapshotter, key string) string {
t.Helper()
dst, err := os.Readlink(s.layerBlobPath(snapshotID(t, ctx, s, key)))
require.NoError(t, err)
return dst
}
t.Run("first cache with the blob wins", func(t *testing.T) {
first, second := t.TempDir(), t.TempDir()
// Both caches hold the diffID; the earlier one must be the one used.
firstBlob := writeCacheBlob(t, first, diffID, []byte("from first"))
writeCacheBlob(t, second, diffID, []byte("from second"))
s := newCacheSnapshotter(t, WithLayerContentCaches(first, second))
key := stageCacheHit(t, ctx, s, target, diffID)
assert.Equal(t, firstBlob, stagedBlob(t, s, key))
})
t.Run("falls through to a later cache", func(t *testing.T) {
empty, populated := t.TempDir(), t.TempDir()
blob := writeCacheBlob(t, populated, diffID, []byte("from second"))
s := newCacheSnapshotter(t, WithLayerContentCaches(empty, populated))
key := stageCacheHit(t, ctx, s, target, diffID)
assert.Equal(t, blob, stagedBlob(t, s, key))
})
t.Run("miss in every cache falls back to a writable snapshot", func(t *testing.T) {
s := newCacheSnapshotter(t, WithLayerContentCaches(
filepath.Join(t.TempDir(), "missing"), t.TempDir(), t.TempDir()))
mounts, err := s.Prepare(ctx, "extract-1 "+target, "", extractionOpt(target, diffID))
require.NoError(t, err)
require.NotEmpty(t, mounts)
for _, m := range mounts {
assert.False(t, m.ReadOnly(), "a miss in every cache must return writable mounts")
}
_, err = s.Stat(ctx, target)
assert.Error(t, err, "target chainID must not be committed on a miss")
})
}
// TestCacheDirMustBeAbsolute covers the one thing NewSnapshotter checks about a
// configured cache dir: it must be absolute, since a relative one would be
// symlinked into the snapshot dir and dangle. The empty string is covered by the
// same check, which otherwise resolves to the daemon's working directory.
func TestCacheDirMustBeAbsolute(t *testing.T) {
requireErofs(t)
for _, dir := range []string{"relative-cache", "./cache", "", "a/b"} {
t.Run(fmt.Sprintf("%q", dir), func(t *testing.T) {
_, err := NewSnapshotter(t.TempDir(), WithLayerContentCaches(dir))
assert.ErrorContains(t, err, "must be an absolute path")
})
}
}

14
plugins/snapshots/erofs/plugin/plugin.go
View File

@@ -57,14 +57,16 @@ type Config struct {
// Linux only
DmverityMode string `toml:"dmverity_mode"`
// LayerContentCache is a directory of pre-converted, diffID-keyed erofs
// layer blobs. When set, layers already present in the cache are committed
// without being downloaded or converted. Empty disables the feature.
// LayerContentCaches lists directories of pre-converted, diffID-keyed erofs
// layer blobs. Each is checked one by one and the first hit is used instead
// of downloading and converting the layer; a directory that doesn't exist is
// treated as a cache miss. Layers missing from all of them are converted
// normally.
//
// Only layers prepared without a parent can be served from the cache. With
// sequential unpacking that is the first layer alone, so getting hits for a
// whole image needs max_concurrent_unpacks > 1, which is not the default.
LayerContentCache string `toml:"layer_content_cache"`
LayerContentCaches []string `toml:"layer_content_caches"`
}
func init() {
@@ -110,8 +112,8 @@ func init() {
opts = append(opts, erofs.WithDmverityMode(config.DmverityMode))
}
if config.LayerContentCache != "" {
opts = append(opts, erofs.WithLayerContentCache(config.LayerContentCache))
if len(config.LayerContentCaches) > 0 {
opts = append(opts, erofs.WithLayerContentCaches(config.LayerContentCaches...))
}
// Don't bother supporting overlay's slow_chown, only RemapIDs