diff --git a/cmd/ctr/commands/images/build_erofs_cache.go b/cmd/ctr/commands/images/build_erofs_cache.go index 95220f8b88..1c1c7dff9d 100644 --- a/cmd/ctr/commands/images/build_erofs_cache.go +++ b/cmd/ctr/commands/images/build_erofs_cache.go @@ -44,7 +44,7 @@ var buildErofsCacheCommand = &cli.Command{ Description: `Convert each layer of an already-pulled image into a directory of diffID-keyed erofs blobs (///.erofs, where is the first two characters of ) for the erofs -snapshotter's layer_content_cache. Layers are read from the content store; no +snapshotter's layer_content_caches. Layers are read from the content store; no converted image is produced. The directory can then be synced to the read-only location the fleet mounts. Requires mkfs.erofs. diff --git a/plugins/snapshots/erofs/erofs.go b/plugins/snapshots/erofs/erofs.go index 4892bb420f..46352cd4c0 100644 --- a/plugins/snapshots/erofs/erofs.go +++ b/plugins/snapshots/erofs/erofs.go @@ -52,12 +52,13 @@ type SnapshotterConfig struct { remapIDs bool // dmverityMode controls dm-verity behavior: "auto" (use if .dmverity exists), "on" (require .dmverity), "off" (disable) dmverityMode string - // layerContentCache is a directory of pre-converted, diffID-keyed erofs - // layer blobs. When set and an unpacked layer's blob is present, the - // snapshotter stages the blob (as a symlink) into the active snapshot, - // skipping the download and tar->erofs conversion. Only parentless Prepares - // can be served. Empty disables the feature. - layerContentCache string + // layerContentCaches lists directories of pre-converted, diffID-keyed erofs + // layer blobs. Each is checked one by one; the first hit is staged into the + // snapshot (symlinked) instead of downloading and converting the layer. A + // directory that doesn't exist is treated as a cache miss. Layers missing + // from all of them are converted normally. Only parentless Prepares can be + // served. + layerContentCaches []string } // Opt is an option to configure the erofs snapshotter @@ -105,13 +106,13 @@ func WithRemapIDs() Opt { } } -// WithLayerContentCache configures a read-only directory of pre-converted, +// WithLayerContentCaches configures read-only directories of pre-converted, // diffID-keyed erofs layer blobs that the snapshotter sources layers from on -// pull instead of downloading and converting them. See the layerContentCache +// pull instead of downloading and converting them. See the layerContentCaches // field for details. -func WithLayerContentCache(path string) Opt { +func WithLayerContentCaches(paths ...string) Opt { return func(config *SnapshotterConfig) { - config.layerContentCache = path + config.layerContentCaches = paths } } @@ -122,16 +123,16 @@ type MetaStore interface { } type snapshotter struct { - root string - ms MetaStore - ovlOptions []string - enableFsverity bool - setImmutable bool - defaultWritable int64 - blockMode bool - remapIDs bool - dmverityMode string - layerContentCache string + root string + ms MetaStore + ovlOptions []string + enableFsverity bool + setImmutable bool + defaultWritable int64 + blockMode bool + remapIDs bool + dmverityMode string + layerContentCaches []string } // NewSnapshotter returns a Snapshotter which uses EROFS+OverlayFS. The layers @@ -177,13 +178,25 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) { // so fsverity and IMMUTABLE_FL can't be applied without mutating that blob out // from under other snapshots. Reject them explicitly instead of silently // skipping; dm-verity is the cache's integrity mechanism. - if config.layerContentCache != "" { + if len(config.layerContentCaches) > 0 { if config.enableFsverity { - return nil, fmt.Errorf("enable_fsverity is incompatible with layer_content_cache; use dm-verity for cache integrity") + return nil, fmt.Errorf("enable_fsverity is incompatible with layer_content_caches; use dm-verity for cache integrity") } if config.setImmutable { - return nil, fmt.Errorf("set_immutable is incompatible with layer_content_cache") + return nil, fmt.Errorf("set_immutable is incompatible with layer_content_caches") } + + // A cache dir is symlinked into snapshots, so a relative one would resolve + // against the snapshot dir and dangle. The check is only lexical: dirs are + // not required to exist, as a missing one just yields a cache miss and may + // well be provisioned after startup. + for _, dir := range config.layerContentCaches { + if !filepath.IsAbs(dir) { + return nil, fmt.Errorf("layer_content_caches %q must be an absolute path", dir) + } + } + + log.L.WithField("dirs", config.layerContentCaches).Info("erofs layer content cache enabled") } // Check fsverity support if enabled @@ -202,24 +215,6 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) { return nil, fmt.Errorf("setting IMMUTABLE_FL is only supported on Linux") } - // Resolve the cache dir to an absolute path so materialized layer blobs are - // absolute symlinks, independent of the process working directory, and verify - // it exists and is a directory so misconfiguration fails fast at startup. - if config.layerContentCache != "" { - abs, err := filepath.Abs(config.layerContentCache) - if err != nil { - return nil, fmt.Errorf("failed to resolve layer_content_cache path %q: %w", config.layerContentCache, err) - } - fi, err := os.Stat(abs) - if err != nil { - return nil, fmt.Errorf("failed to access layer_content_cache %q: %w", abs, err) - } - if !fi.IsDir() { - return nil, fmt.Errorf("layer_content_cache %q is not a directory", abs) - } - config.layerContentCache = abs - } - ms, err := storage.NewMetaStore(filepath.Join(root, "metadata.db")) if err != nil { return nil, err @@ -230,16 +225,16 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) { } return &snapshotter{ - root: root, - ms: ms, - ovlOptions: config.ovlOptions, - enableFsverity: config.enableFsverity, - setImmutable: config.setImmutable, - defaultWritable: config.defaultSize, - blockMode: config.defaultSize > 0, - remapIDs: config.remapIDs, - dmverityMode: config.dmverityMode, - layerContentCache: config.layerContentCache, + root: root, + ms: ms, + ovlOptions: config.ovlOptions, + enableFsverity: config.enableFsverity, + setImmutable: config.setImmutable, + defaultWritable: config.defaultSize, + blockMode: config.defaultSize > 0, + remapIDs: config.remapIDs, + dmverityMode: config.dmverityMode, + layerContentCaches: config.layerContentCaches, }, nil } @@ -700,21 +695,17 @@ func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...s return s.createSnapshot(ctx, snapshots.KindActive, key, parent, opts) } -// cacheBlobPath returns the expected path of the cached erofs blob for a diffID. -func (s *snapshotter) cacheBlobPath(diffID digest.Digest) string { - return erofsutils.CacheBlobPath(s.layerContentCache, diffID) -} - // lookupCache returns the absolute path of the cached erofs blob that can serve -// the layer being prepared, or "" on a miss. It gates on: the cache being -// configured, the Prepare being an image-layer extraction (carries the -// snapshot.ref and diff-id labels), and the diffID blob being present. Misses -// (cache disabled, non-extraction Prepare, missing entries, unreadable cache dirs -// such as a FUSE mount being down, malformed labels) all return "" so pulls keep -// working. Any dm-verity sidecar is derived from the blob path (via -// dmverity.MetadataPath) when the blob is materialized (prepareDirectory). +// the layer being prepared, or "" on a miss. It gates on: at least one cache +// being configured, the Prepare being an image-layer extraction (carries the +// snapshot.ref and diff-id labels), and the diffID blob being present. Caches +// are checked one by one and the first hit wins. Every miss (cache disabled, +// non-extraction Prepare, absent or unreadable cache dir, missing entry, +// malformed labels) returns "" so pulls keep working. Any dm-verity sidecar is +// derived from the blob path (via dmverity.MetadataPath) when the blob is +// staged (prepareDirectory). func (s *snapshotter) lookupCache(ctx context.Context, opts ...snapshots.Opt) string { - if s.layerContentCache == "" { + if len(s.layerContentCaches) == 0 { return "" } @@ -737,18 +728,24 @@ func (s *snapshotter) lookupCache(ctx context.Context, opts ...snapshots.Opt) st return "" } - blob := s.cacheBlobPath(diffID) - if _, err := os.Stat(blob); err != nil { - if os.IsNotExist(err) { - log.G(ctx).WithField("blob", blob).Trace("erofs layer cache miss") - } else { - log.G(ctx).WithError(err).WithField("blob", blob). - Warn("erofs layer cache: failed to stat cache blob, treating as cache miss") + for _, dir := range s.layerContentCaches { + blob := erofsutils.CacheBlobPath(dir, diffID) + if _, err := os.Stat(blob); err != nil { + if !os.IsNotExist(err) { + // An unreadable cache (a down FUSE mount, a permission change since + // startup) shouldn't fail the pull or mask a hit in a later cache. + log.G(ctx).WithError(err).WithField("blob", blob). + Warn("erofs layer cache: failed to stat cache blob, skipping this cache") + } + continue } - return "" + // Absolute, since the configured dirs are validated as such: the hit is + // symlinked into the snapshot dir, where a relative target would dangle. + return blob } - return blob + log.G(ctx).WithField("diffID", diffID.String()).Trace("erofs layer cache miss") + return "" } func (s *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { diff --git a/plugins/snapshots/erofs/erofs_linux_test.go b/plugins/snapshots/erofs/erofs_linux_test.go index fa08c2a128..ec9c15c6b7 100644 --- a/plugins/snapshots/erofs/erofs_linux_test.go +++ b/plugins/snapshots/erofs/erofs_linux_test.go @@ -990,7 +990,7 @@ func TestCacheHit(t *testing.T) { diffID := digest.Digest(cacheTestDiffID) blob := writeCacheBlob(t, cacheDir, diffID, []byte("fake erofs blob")) - s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir)) + s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir)) target := cacheTestChainID key := stageCacheHit(t, ctx, s, target, diffID) @@ -1031,7 +1031,7 @@ func TestCacheSidecar(t *testing.T) { require.NoError(t, os.WriteFile(dmverity.MetadataPath(blob), []byte(testDmverityMetadata), 0644)) // dmverity_mode defaults to "auto": use the sidecar if present. - s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir)) + s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir)) target := cacheTestChainID key := stageCacheHit(t, ctx, s, target, diffID) @@ -1078,7 +1078,7 @@ func TestCacheMiss(t *testing.T) { }) t.Run("blob absent", func(t *testing.T) { - s := newCacheSnapshotter(t, WithLayerContentCache(t.TempDir())) + s := newCacheSnapshotter(t, WithLayerContentCaches(t.TempDir())) mounts, err := s.Prepare(ctx, "extract-1 "+target, "", extractionOpt(target, diffID)) assertFellThrough(t, s, mounts, err, false) }) @@ -1086,7 +1086,7 @@ func TestCacheMiss(t *testing.T) { t.Run("no extraction labels", func(t *testing.T) { cacheDir := t.TempDir() writeCacheBlob(t, cacheDir, diffID, []byte("blob")) - s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir)) + s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir)) // A container-rootfs Prepare carries no snapshot.ref/diff-id labels. mounts, err := s.Prepare(ctx, "container-rootfs", "") require.NoError(t, err) @@ -1096,7 +1096,7 @@ func TestCacheMiss(t *testing.T) { t.Run("view is never short-circuited", func(t *testing.T) { cacheDir := t.TempDir() writeCacheBlob(t, cacheDir, diffID, []byte("blob")) - s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir)) + s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir)) // Even with matching labels and a cached blob, a View must not commit. // It's read-only, but via the KindView roFlag in mounts(), not the cache. mounts, err := s.View(ctx, "view-1", "", extractionOpt(target, diffID)) @@ -1122,7 +1122,7 @@ func TestCacheParentedPrepare(t *testing.T) { writeCacheBlob(t, cacheDir, parentDiffID, []byte("fake parent blob")) writeCacheBlob(t, cacheDir, childDiffID, []byte("fake child blob")) - s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir)) + s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir)) // The first layer has no parent, so it is served from the cache as usual. require.NoError(t, s.Commit(ctx, parentChain, stageCacheHit(t, ctx, s, parentChain, parentDiffID))) @@ -1152,7 +1152,7 @@ func TestCacheRemove(t *testing.T) { sidecar := dmverity.MetadataPath(blob) require.NoError(t, os.WriteFile(sidecar, []byte(testDmverityMetadata), 0644)) - s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir)) + s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir)) target := cacheTestChainID key := stageCacheHit(t, ctx, s, target, diffID) @@ -1189,7 +1189,7 @@ func TestCacheDmverity(t *testing.T) { blob := writeCacheBlob(t, cacheDir, diffID, []byte("fake erofs blob")) require.NoError(t, os.WriteFile(dmverity.MetadataPath(blob), []byte(testDmverityMetadata), 0644)) - s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir), WithDmverityMode("on")) + s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir), WithDmverityMode("on")) key := stageCacheHit(t, ctx, s, target, diffID) _, err := os.Stat(dmverity.MetadataPath(s.layerBlobPath(snapshotID(t, ctx, s, key)))) @@ -1200,7 +1200,7 @@ func TestCacheDmverity(t *testing.T) { cacheDir := t.TempDir() writeCacheBlob(t, cacheDir, diffID, []byte("fake erofs blob")) // no sidecar - s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir), WithDmverityMode("on")) + s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir), WithDmverityMode("on")) // dmverity_mode=on requires a sidecar; a cache entry without one is a hard // error rather than a silent fallback. @@ -1211,3 +1211,71 @@ func TestCacheDmverity(t *testing.T) { assert.Error(t, err, "no snapshot should be committed on failure") }) } + +// TestCacheMultipleDirs covers the layer_content_caches search path: directories +// are searched in configured order, the first hit wins, later caches are reached +// when earlier ones lack the blob, and a miss in every cache falls through to a +// normal writable snapshot. +func TestCacheMultipleDirs(t *testing.T) { + ctx := namespaces.WithNamespace(context.Background(), "test") + diffID := digest.Digest(cacheTestDiffID) + target := cacheTestChainID + + // stagedBlob returns the cache blob the snapshot's layer.erofs symlink points + // at, i.e. which of the configured caches actually served the layer. + stagedBlob := func(t *testing.T, s *snapshotter, key string) string { + t.Helper() + dst, err := os.Readlink(s.layerBlobPath(snapshotID(t, ctx, s, key))) + require.NoError(t, err) + return dst + } + + t.Run("first cache with the blob wins", func(t *testing.T) { + first, second := t.TempDir(), t.TempDir() + // Both caches hold the diffID; the earlier one must be the one used. + firstBlob := writeCacheBlob(t, first, diffID, []byte("from first")) + writeCacheBlob(t, second, diffID, []byte("from second")) + + s := newCacheSnapshotter(t, WithLayerContentCaches(first, second)) + key := stageCacheHit(t, ctx, s, target, diffID) + assert.Equal(t, firstBlob, stagedBlob(t, s, key)) + }) + + t.Run("falls through to a later cache", func(t *testing.T) { + empty, populated := t.TempDir(), t.TempDir() + blob := writeCacheBlob(t, populated, diffID, []byte("from second")) + + s := newCacheSnapshotter(t, WithLayerContentCaches(empty, populated)) + key := stageCacheHit(t, ctx, s, target, diffID) + assert.Equal(t, blob, stagedBlob(t, s, key)) + }) + + t.Run("miss in every cache falls back to a writable snapshot", func(t *testing.T) { + s := newCacheSnapshotter(t, WithLayerContentCaches( + filepath.Join(t.TempDir(), "missing"), t.TempDir(), t.TempDir())) + + mounts, err := s.Prepare(ctx, "extract-1 "+target, "", extractionOpt(target, diffID)) + require.NoError(t, err) + require.NotEmpty(t, mounts) + for _, m := range mounts { + assert.False(t, m.ReadOnly(), "a miss in every cache must return writable mounts") + } + _, err = s.Stat(ctx, target) + assert.Error(t, err, "target chainID must not be committed on a miss") + }) +} + +// TestCacheDirMustBeAbsolute covers the one thing NewSnapshotter checks about a +// configured cache dir: it must be absolute, since a relative one would be +// symlinked into the snapshot dir and dangle. The empty string is covered by the +// same check, which otherwise resolves to the daemon's working directory. +func TestCacheDirMustBeAbsolute(t *testing.T) { + requireErofs(t) + + for _, dir := range []string{"relative-cache", "./cache", "", "a/b"} { + t.Run(fmt.Sprintf("%q", dir), func(t *testing.T) { + _, err := NewSnapshotter(t.TempDir(), WithLayerContentCaches(dir)) + assert.ErrorContains(t, err, "must be an absolute path") + }) + } +} diff --git a/plugins/snapshots/erofs/plugin/plugin.go b/plugins/snapshots/erofs/plugin/plugin.go index 0315e45564..008ddbd472 100644 --- a/plugins/snapshots/erofs/plugin/plugin.go +++ b/plugins/snapshots/erofs/plugin/plugin.go @@ -57,14 +57,16 @@ type Config struct { // Linux only DmverityMode string `toml:"dmverity_mode"` - // LayerContentCache is a directory of pre-converted, diffID-keyed erofs - // layer blobs. When set, layers already present in the cache are committed - // without being downloaded or converted. Empty disables the feature. + // LayerContentCaches lists directories of pre-converted, diffID-keyed erofs + // layer blobs. Each is checked one by one and the first hit is used instead + // of downloading and converting the layer; a directory that doesn't exist is + // treated as a cache miss. Layers missing from all of them are converted + // normally. // // Only layers prepared without a parent can be served from the cache. With // sequential unpacking that is the first layer alone, so getting hits for a // whole image needs max_concurrent_unpacks > 1, which is not the default. - LayerContentCache string `toml:"layer_content_cache"` + LayerContentCaches []string `toml:"layer_content_caches"` } func init() { @@ -110,8 +112,8 @@ func init() { opts = append(opts, erofs.WithDmverityMode(config.DmverityMode)) } - if config.LayerContentCache != "" { - opts = append(opts, erofs.WithLayerContentCache(config.LayerContentCache)) + if len(config.LayerContentCaches) > 0 { + opts = append(opts, erofs.WithLayerContentCaches(config.LayerContentCaches...)) } // Don't bother supporting overlay's slow_chown, only RemapIDs