diff --git a/core/metadata/snapshot.go b/core/metadata/snapshot.go index 5880df57ec..132de60ec7 100644 --- a/core/metadata/snapshot.go +++ b/core/metadata/snapshot.go @@ -39,7 +39,6 @@ import ( const ( inheritedLabelsPrefix = "containerd.io/snapshot/" - labelSnapshotRef = "containerd.io/snapshot.ref" ) type snapshotter struct { @@ -316,7 +315,7 @@ func (s *snapshotter) createSnapshot(ctx context.Context, key, parent string, re } var ( - target = base.Labels[labelSnapshotRef] + target = base.Labels[snapshots.LabelSnapshotRef] bparent string bkey string bopts = []snapshots.Opt{ @@ -388,10 +387,10 @@ func (s *snapshotter) createSnapshot(ctx context.Context, key, parent string, re if errdefs.IsAlreadyExists(err) { if target != "" { var tinfo *snapshots.Info - filter := fmt.Sprintf(`labels."containerd.io/snapshot.ref"==%s,parent==%q`, target, bparent) + filter := fmt.Sprintf(`labels.%q==%s,parent==%q`, snapshots.LabelSnapshotRef, target, bparent) if err := s.Snapshotter.Walk(ctx, func(ctx context.Context, i snapshots.Info) error { if tinfo == nil && i.Kind == snapshots.KindCommitted { - if i.Labels["containerd.io/snapshot.ref"] != target { + if i.Labels[snapshots.LabelSnapshotRef] != target { // Walk did not respect filter return nil } diff --git a/core/metadata/snapshot_test.go b/core/metadata/snapshot_test.go index e005cb898d..ae818aebed 100644 --- a/core/metadata/snapshot_test.go +++ b/core/metadata/snapshot_test.go @@ -70,7 +70,7 @@ func TestSnapshotterWithRef(t *testing.T) { key1 := "test1" test1opt := snapshots.WithLabels( map[string]string{ - labelSnapshotRef: key1, + snapshots.LabelSnapshotRef: key1, }, ) @@ -113,7 +113,7 @@ func TestSnapshotterWithRef(t *testing.T) { key2 := "test2" test2opt := snapshots.WithLabels( map[string]string{ - labelSnapshotRef: key2, + snapshots.LabelSnapshotRef: key2, }, ) @@ -340,7 +340,7 @@ func (s *tmpSnapshotter) create(ctx context.Context, key, parent string, kind sn base.Name = key base.Kind = kind - target := base.Labels[labelSnapshotRef] + target := base.Labels[snapshots.LabelSnapshotRef] if target != "" { for _, name := range s.targets[target] { if s.snapshots[name].Parent == parent { @@ -399,7 +399,7 @@ func (s *tmpSnapshotter) Commit(ctx context.Context, name, key string, opts ...s s.snapshots[name] = base delete(s.snapshots, key) - if target := base.Labels[labelSnapshotRef]; target != "" { + if target := base.Labels[snapshots.LabelSnapshotRef]; target != "" { s.targets[target] = append(s.targets[target], name) } diff --git a/core/snapshots/snapshotter.go b/core/snapshots/snapshotter.go index cb94e12110..9140a0644e 100644 --- a/core/snapshots/snapshotter.go +++ b/core/snapshots/snapshotter.go @@ -33,7 +33,17 @@ const ( // UnpackKeyFormat is the format for the snapshotter keys used for extraction UnpackKeyFormat = UnpackKeyPrefix + "-%s %s" inheritedLabelsPrefix = "containerd.io/snapshot/" - labelSnapshotRef = "containerd.io/snapshot.ref" + + // LabelSnapshotRef is set by the unpacker on the extraction Prepare to + // the target chainID. A snapshotter that already has the layer commits a + // snapshot named after this value and returns ErrAlreadyExists, which + // makes the unpacker skip fetching and applying the layer (the remote + // snapshot protocol). It is inherited by FilterInheritedLabels. + LabelSnapshotRef = "containerd.io/snapshot.ref" + + // LabelSnapshotDiffID is set by the unpacker on the extraction Prepare to + // the uncompressed digest (diffID) of the layer being unpacked. + LabelSnapshotDiffID = "containerd.io/snapshot/diff-id" // LabelSnapshotUIDMapping is the label used for UID mappings LabelSnapshotUIDMapping = "containerd.io/snapshot/uidmapping" @@ -397,7 +407,7 @@ func FilterInheritedLabels(labels map[string]string) map[string]string { filtered := make(map[string]string) for k, v := range labels { - if k == labelSnapshotRef || strings.HasPrefix(k, inheritedLabelsPrefix) { + if k == LabelSnapshotRef || strings.HasPrefix(k, inheritedLabelsPrefix) { filtered[k] = v } } diff --git a/core/unpack/unpacker.go b/core/unpack/unpacker.go index 67f2dcb7c4..ec55e65317 100644 --- a/core/unpack/unpacker.go +++ b/core/unpack/unpacker.go @@ -49,9 +49,7 @@ import ( ) const ( - labelSnapshotRef = "containerd.io/snapshot.ref" labelSnapshotParent = "containerd.io/snapshot/parent-chain-id" - labelSnapshotDiffID = "containerd.io/snapshot/diff-id" unpackSpanPrefix = "pkg.unpack.unpacker" ) @@ -397,8 +395,8 @@ func (u *Unpacker) unpack( if snapshotLabels == nil { snapshotLabels = make(map[string]string) } - snapshotLabels[labelSnapshotRef] = chainID - snapshotLabels[labelSnapshotDiffID] = diffIDs[i].String() + snapshotLabels[snapshots.LabelSnapshotRef] = chainID + snapshotLabels[snapshots.LabelSnapshotDiffID] = diffIDs[i].String() if i > 0 { snapshotLabels[labelSnapshotParent] = chainIDs[i-1].String() } diff --git a/internal/erofsutils/mount.go b/internal/erofsutils/mount.go index cc9a4a25f5..6c3ef9f648 100644 --- a/internal/erofsutils/mount.go +++ b/internal/erofsutils/mount.go @@ -29,10 +29,22 @@ import ( "github.com/containerd/errdefs" "github.com/containerd/log" + "github.com/opencontainers/go-digest" "github.com/containerd/containerd/v2/core/mount" ) +// CacheBlobPath returns the layer content cache path for a diffID, following the +// ///.erofs layout, where is the first two characters of +// the encoded digest. The extra prefix directory shards blobs so no single +// directory grows unwieldy for a large cache. It is the single source of the +// layout, shared by the cache producer (ctr build-erofs-cache) and the erofs +// snapshotter that reads it, so the two never drift. +func CacheBlobPath(dir string, diffID digest.Digest) string { + enc := diffID.Encoded() + return filepath.Join(dir, diffID.Algorithm().String(), enc[:2], enc+".erofs") +} + // IsErofsMediaType returns true if the media type is an EROFS layer type. func IsErofsMediaType(mt string) bool { return strings.HasPrefix(mt, "application/vnd.erofs.layer") diff --git a/internal/erofsutils/mount_test.go b/internal/erofsutils/mount_test.go new file mode 100644 index 0000000000..4314fb9aca --- /dev/null +++ b/internal/erofsutils/mount_test.go @@ -0,0 +1,37 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package erofsutils + +import ( + "path/filepath" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" +) + +func TestCacheBlobPath(t *testing.T) { + diffID := digest.FromString("hello") + enc := diffID.Encoded() + + got := CacheBlobPath("/cache", diffID) + + // ///.erofs, where is the first two hex characters. + want := filepath.Join("/cache", "sha256", enc[:2], enc+".erofs") + assert.Equal(t, want, got) + assert.Equal(t, enc[:2], filepath.Base(filepath.Dir(got)), "blob must live under a 2-char prefix dir") +} diff --git a/plugins/snapshots/erofs/erofs.go b/plugins/snapshots/erofs/erofs.go index 2f038706f2..4af0f034e7 100644 --- a/plugins/snapshots/erofs/erofs.go +++ b/plugins/snapshots/erofs/erofs.go @@ -18,6 +18,7 @@ package erofs import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -27,11 +28,13 @@ import ( "github.com/containerd/continuity/fs" "github.com/containerd/errdefs" "github.com/containerd/log" + "github.com/opencontainers/go-digest" "github.com/containerd/containerd/v2/core/mount" "github.com/containerd/containerd/v2/core/snapshots" "github.com/containerd/containerd/v2/core/snapshots/storage" "github.com/containerd/containerd/v2/internal/dmverity" + "github.com/containerd/containerd/v2/internal/erofsutils" "github.com/containerd/containerd/v2/internal/fsverity" "github.com/containerd/containerd/v2/internal/userns" ) @@ -49,6 +52,12 @@ type SnapshotterConfig struct { remapIDs bool // dmverityMode controls dm-verity behavior: "auto" (use if .dmverity exists), "on" (require .dmverity), "off" (disable) dmverityMode string + // layerContentCache is a directory of pre-converted, diffID-keyed erofs + // layer blobs. When set and an unpacked layer's blob is present, the + // snapshotter commits the layer immediately (symlinking the blob) and + // returns ErrAlreadyExists, skipping the download and tar->erofs + // conversion. Empty disables the feature. + layerContentCache string } // Opt is an option to configure the erofs snapshotter @@ -96,6 +105,16 @@ func WithRemapIDs() Opt { } } +// WithLayerContentCache configures a read-only directory of pre-converted, +// diffID-keyed erofs layer blobs that the snapshotter sources layers from on +// pull instead of downloading and converting them. See the layerContentCache +// field for details. +func WithLayerContentCache(path string) Opt { + return func(config *SnapshotterConfig) { + config.layerContentCache = path + } +} + type MetaStore interface { TransactionContext(ctx context.Context, writable bool) (context.Context, storage.Transactor, error) WithTransaction(ctx context.Context, writable bool, fn storage.TransactionCallback) error @@ -103,15 +122,16 @@ type MetaStore interface { } type snapshotter struct { - root string - ms *storage.MetaStore - ovlOptions []string - enableFsverity bool - setImmutable bool - defaultWritable int64 - blockMode bool - remapIDs bool - dmverityMode string + root string + ms MetaStore + ovlOptions []string + enableFsverity bool + setImmutable bool + defaultWritable int64 + blockMode bool + remapIDs bool + dmverityMode string + layerContentCache string } // NewSnapshotter returns a Snapshotter which uses EROFS+OverlayFS. The layers @@ -153,6 +173,20 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) { } } + // Cache blobs may live on a read-only mount the snapshotter can't modify, so + // fsverity and IMMUTABLE_FL can't be applied to them. Be explicit about this to + // the user instead of ignoring them silently (they're bypassed because cache + // hits commit during Prepare and skip Commit); dm-verity is the cache's + // integrity mechanism. + if config.layerContentCache != "" { + if config.enableFsverity { + return nil, fmt.Errorf("enable_fsverity is incompatible with layer_content_cache; use dm-verity for cache integrity") + } + if config.setImmutable { + return nil, fmt.Errorf("set_immutable is incompatible with layer_content_cache") + } + } + // Check fsverity support if enabled if config.enableFsverity { // TODO: Call specific function here @@ -169,6 +203,24 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) { return nil, fmt.Errorf("setting IMMUTABLE_FL is only supported on Linux") } + // Resolve the cache dir to an absolute path so materialized layer blobs are + // absolute symlinks, independent of the process working directory, and verify + // it exists and is a directory so misconfiguration fails fast at startup. + if config.layerContentCache != "" { + abs, err := filepath.Abs(config.layerContentCache) + if err != nil { + return nil, fmt.Errorf("failed to resolve layer_content_cache path %q: %w", config.layerContentCache, err) + } + fi, err := os.Stat(abs) + if err != nil { + return nil, fmt.Errorf("failed to access layer_content_cache %q: %w", abs, err) + } + if !fi.IsDir() { + return nil, fmt.Errorf("layer_content_cache %q is not a directory", abs) + } + config.layerContentCache = abs + } + ms, err := storage.NewMetaStore(filepath.Join(root, "metadata.db")) if err != nil { return nil, err @@ -179,15 +231,16 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) { } return &snapshotter{ - root: root, - ms: ms, - ovlOptions: config.ovlOptions, - enableFsverity: config.enableFsverity, - setImmutable: config.setImmutable, - defaultWritable: config.defaultSize, - blockMode: config.defaultSize > 0, - remapIDs: config.remapIDs, - dmverityMode: config.dmverityMode, + root: root, + ms: ms, + ovlOptions: config.ovlOptions, + enableFsverity: config.enableFsverity, + setImmutable: config.setImmutable, + defaultWritable: config.defaultSize, + blockMode: config.defaultSize > 0, + remapIDs: config.remapIDs, + dmverityMode: config.dmverityMode, + layerContentCache: config.layerContentCache, }, nil } @@ -245,7 +298,7 @@ func (s *snapshotter) lowerPath(id string) (string, error) { return layerBlob, nil } -func (s *snapshotter) prepareDirectory(ctx context.Context, snapshotDir string, kind snapshots.Kind) (string, error) { +func (s *snapshotter) prepareDirectory(ctx context.Context, snapshotDir string, kind snapshots.Kind, entry *cacheEntry) (string, error) { td, err := os.MkdirTemp(snapshotDir, "new-") if err != nil { return "", fmt.Errorf("failed to create temp dir: %w", err) @@ -267,6 +320,26 @@ func (s *snapshotter) prepareDirectory(ctx context.Context, snapshotDir string, } } + // Layer content cache hit: stage the pre-converted blob as a symlink so the + // caller's rename publishes a ready committed layer. + if entry != nil { + layerBlob := filepath.Join(td, "layer.erofs") + if err := os.Symlink(entry.blob, layerBlob); err != nil { + return td, fmt.Errorf("failed to symlink cached layer blob: %w", err) + } + // Copy the dm-verity sidecar alongside the blob (unless dm-verity is off, + // when it's never consumed) so mount-time metadata resolution and the + // pinned root hash match locally-converted layers. A missing sidecar is + // fine except with dmverity_mode "on", which requires it. + if s.dmverityMode != "off" { + if err := fs.CopyFile(dmverity.MetadataPath(layerBlob), dmverity.MetadataPath(entry.blob)); err != nil { + if s.dmverityMode == "on" || !errors.Is(err, os.ErrNotExist) { + return td, fmt.Errorf("failed to copy dm-verity sidecar: %w", err) + } + } + } + } + return td, nil } @@ -503,6 +576,12 @@ func (s *snapshotter) mounts(snap storage.Snapshot, info snapshots.Info) ([]moun }), nil } +// createSnapshot creates an active (or view) snapshot and returns its mounts. +// On an image-layer extraction whose diffID blob is in the layer content cache, +// it instead stages the cached blob and commits the snapshot as the target +// chainID in the same transaction, then returns ErrAlreadyExists (the +// remote-snapshot signal that makes the unpacker skip the layer download and +// conversion). func (s *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts []snapshots.Opt) (_ []mount.Mount, err error) { var ( snap storage.Snapshot @@ -510,8 +589,21 @@ func (s *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k info snapshots.Info ) + // Only image-layer extractions (active snapshots) can be served from the + // layer content cache; View and container-rootfs Prepares get a nil entry and + // fall through to the normal path. + var entry *cacheEntry + if kind == snapshots.KindActive { + entry = s.lookupCache(ctx, opts...) + } + + // committed is set only once the cached layer is committed and we deliberately + // return ErrAlreadyExists; the committed dir must then be kept. Any real error + // (including an unexpected AlreadyExists from CreateSnapshot) leaves it false + // so the staged td/path is reclaimed. + var committed bool defer func() { - if err != nil { + if err != nil && !committed { if td != "" { if err1 := os.RemoveAll(td); err1 != nil { log.G(ctx).WithError(err1).Warn("failed to cleanup temp snapshot directory") @@ -527,7 +619,7 @@ func (s *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k }() snapshotDir := filepath.Join(s.root, "snapshots") - td, err = s.prepareDirectory(ctx, snapshotDir, kind) + td, err = s.prepareDirectory(ctx, snapshotDir, kind, entry) if err != nil { return nil, fmt.Errorf("failed to create prepare snapshot dir: %w", err) } @@ -598,11 +690,34 @@ func (s *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k return fmt.Errorf("failed to rename: %w", err) } td = "" + + // Commit the cached layer straight away as the target chainID. CommitActive + // replaces labels with those from opts (which carry snapshot.ref), which the + // metadata layer's Walk filter needs to resolve the backend target. + if entry != nil { + if _, err = storage.CommitActive(ctx, key, entry.target, snapshots.Usage{}, opts...); err != nil { + return fmt.Errorf("unable to commit active snapshot: %w", err) + } + } return nil }); err != nil { return nil, err } + // Cache hit committed successfully: signal the unpacker via ErrAlreadyExists to + // skip the layer download and conversion. (A concurrent pull that already + // committed the same target returned a plain AlreadyExists error above, which + // the metadata layer resolves the same way.) + if entry != nil { + log.G(ctx).WithFields(log.Fields{ + "key": key, + "chainID": entry.target, + "blob": entry.blob, + }).Debug("layer content cache hit, committed cached erofs blob") + committed = true + return nil, errdefs.ErrAlreadyExists + } + return s.mounts(snap, info) } @@ -610,6 +725,63 @@ func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...s return s.createSnapshot(ctx, snapshots.KindActive, key, parent, opts) } +// cacheEntry describes a resolved layer content cache entry: the target chainID +// to commit as and the absolute path of the cached blob to symlink (any +// dm-verity sidecar is derived from blob via dmverity.MetadataPath). +type cacheEntry struct { + target string + blob string +} + +// cacheBlobPath returns the expected path of the cached erofs blob for a diffID. +func (s *snapshotter) cacheBlobPath(diffID digest.Digest) string { + return erofsutils.CacheBlobPath(s.layerContentCache, diffID) +} + +// lookupCache resolves the layer content cache entry that can serve the layer +// being prepared, or nil on a miss. It gates on: the cache being configured, the +// Prepare being an image-layer extraction (carries the snapshot.ref and diff-id +// labels), and the diffID blob being present. Misses (cache disabled, +// non-extraction Prepare, missing entries, unreadable cache dirs such as a FUSE +// mount being down, malformed labels) all return nil so pulls keep working. The +// dm-verity sidecar is handled when the blob is materialized (prepareDirectory). +func (s *snapshotter) lookupCache(ctx context.Context, opts ...snapshots.Opt) *cacheEntry { + if s.layerContentCache == "" { + return nil + } + + var base snapshots.Info + for _, opt := range opts { + if err := opt(&base); err != nil { + return nil + } + } + + target := base.Labels[snapshots.LabelSnapshotRef] + diffIDStr := base.Labels[snapshots.LabelSnapshotDiffID] + if target == "" || diffIDStr == "" { + // Not an image-layer extraction, or no diffID to key on. + return nil + } + diffID, err := digest.Parse(diffIDStr) + if err != nil { + log.G(ctx).WithError(err).WithField("diffID", diffIDStr). + Warn("erofs layer cache: invalid diff-id label, treating as cache miss") + return nil + } + + blob := s.cacheBlobPath(diffID) + if _, err := os.Stat(blob); err != nil { + if !os.IsNotExist(err) { + log.G(ctx).WithError(err).WithField("blob", blob). + Warn("erofs layer cache: failed to stat cache blob, treating as cache miss") + } + return nil + } + + return &cacheEntry{target: target, blob: blob} +} + func (s *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { return s.createSnapshot(ctx, snapshots.KindView, key, parent, opts) } @@ -798,10 +970,20 @@ func (s *snapshotter) Remove(ctx context.Context, key string) (err error) { // The layer blob is only persisted for committed snapshots. if info.Kind == snapshots.KindCommitted { - // Clear IMMUTABLE_FL before removal, since this flag avoids it. - err = setImmutable(s.layerBlobPath(id), false) - if err != nil && !errdefs.IsNotImplemented(err) { - return fmt.Errorf("failed to clear IMMUTABLE_FL: %w", err) + layerBlob := s.layerBlobPath(id) + // A cache-hit snapshot's blob is a symlink into the operator-owned + // cache dir. Skip clearing IMMUTABLE_FL: setImmutable's os.Open would + // follow the link and ioctl the cache entry (which we don't own), and + // cache blobs were never made immutable by us in the first place. + // os.RemoveAll below unlinks the symlink without following it. + if fi, lerr := os.Lstat(layerBlob); lerr == nil && fi.Mode()&os.ModeSymlink != 0 { + log.G(ctx).WithField("id", id).Trace("erofs layer cache: skipping IMMUTABLE_FL clear for symlinked cache blob") + } else { + // Clear IMMUTABLE_FL before removal, since this flag avoids it. + err = setImmutable(layerBlob, false) + if err != nil && !errdefs.IsNotImplemented(err) { + return fmt.Errorf("failed to clear IMMUTABLE_FL: %w", err) + } } } _, _, err = storage.Remove(ctx, key) diff --git a/plugins/snapshots/erofs/erofs_linux_test.go b/plugins/snapshots/erofs/erofs_linux_test.go index 3c182d800d..6b6ff9b13e 100644 --- a/plugins/snapshots/erofs/erofs_linux_test.go +++ b/plugins/snapshots/erofs/erofs_linux_test.go @@ -26,6 +26,7 @@ import ( "testing" "time" + "github.com/containerd/errdefs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" bolt "go.etcd.io/bbolt" @@ -818,3 +819,258 @@ func TestMountFsMeta(t *testing.T) { }, m.Options) }) } + +// --- layer content cache tests --- + +const ( + cacheTestDiffID = "sha256:0000000000000000000000000000000000000000000000000000000000000001" + cacheTestChainID = "sha256:0000000000000000000000000000000000000000000000000000000000000002" +) + +// requireErofs skips a test unless the erofs kernel filesystem is available, +// which NewSnapshotter requires. The layer content cache tests don't mount, so +// they need neither root nor mkfs.erofs. +func requireErofs(t *testing.T) { + t.Helper() + if !FindErofs() { + t.Skip("check for erofs kernel support failed, skipping test") + } +} + +// writeCacheBlob writes a fake erofs blob into cacheDir keyed by diffID and +// returns its absolute path. The bytes need not be a valid erofs image: the +// snapshotter only symlinks the blob, so these tests exercise the +// Prepare/Commit/Remove logic rather than mounting. +func writeCacheBlob(t *testing.T, cacheDir string, diffID digest.Digest, data []byte) string { + t.Helper() + blob := erofsutils.CacheBlobPath(cacheDir, diffID) + require.NoError(t, os.MkdirAll(filepath.Dir(blob), 0755)) + require.NoError(t, os.WriteFile(blob, data, 0644)) + return blob +} + +// extractionOpt builds the snapshot options the unpacker attaches to an +// image-layer extraction Prepare (the snapshot.ref target and the diffID). +func extractionOpt(target string, diffID digest.Digest) snapshots.Opt { + return snapshots.WithLabels(map[string]string{ + snapshots.LabelSnapshotRef: target, + snapshots.LabelSnapshotDiffID: diffID.String(), + }) +} + +// snapshotID returns the backend snapshot ID for key. +func snapshotID(t *testing.T, ctx context.Context, s *snapshotter, key string) string { + t.Helper() + var id string + require.NoError(t, s.ms.WithTransaction(ctx, false, func(ctx context.Context) error { + var err error + id, _, _, err = storage.GetInfo(ctx, key) + return err + })) + return id +} + +// newCacheSnapshotter creates an erofs snapshotter rooted in a temp dir with the +// given options, skipping the test if erofs is unavailable and registering the +// snapshotter's cleanup. +func newCacheSnapshotter(t *testing.T, opts ...Opt) *snapshotter { + t.Helper() + requireErofs(t) + sn, err := NewSnapshotter(t.TempDir(), opts...) + require.NoError(t, err) + t.Cleanup(func() { sn.Close() }) + return sn.(*snapshotter) +} + +// prepareCacheHit runs an extraction Prepare for target/diffID and asserts it was +// served from the cache (committed and signaled via ErrAlreadyExists, no mounts). +func prepareCacheHit(t *testing.T, ctx context.Context, s *snapshotter, target string, diffID digest.Digest) { + t.Helper() + mounts, err := s.Prepare(ctx, "extract-1 "+target, "", extractionOpt(target, diffID)) + require.ErrorIs(t, err, errdefs.ErrAlreadyExists, "cache hit must signal the remote-snapshot protocol") + assert.Nil(t, mounts, "a cache hit returns no mounts") +} + +// TestCacheHit covers the happy path: an extraction Prepare whose diffID blob is +// in the cache commits the target chainID (right kind, parent, and snapshot.ref +// label), symlinks the blob, and returns ErrAlreadyExists. +func TestCacheHit(t *testing.T) { + ctx := namespaces.WithNamespace(context.Background(), "test") + + cacheDir := t.TempDir() + diffID := digest.Digest(cacheTestDiffID) + blob := writeCacheBlob(t, cacheDir, diffID, []byte("fake erofs blob")) + + s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir)) + + target := cacheTestChainID + prepareCacheHit(t, ctx, s, target, diffID) + + // The target chainID is committed, with the parent and snapshot.ref label the + // metadata layer's Walk filter needs to resolve the backend target. + info, err := s.Stat(ctx, target) + require.NoError(t, err, "committed snapshot must exist under the target chainID") + assert.Equal(t, snapshots.KindCommitted, info.Kind) + assert.Equal(t, "", info.Parent) + assert.Equal(t, target, info.Labels[snapshots.LabelSnapshotRef]) + + // layer.erofs is an absolute symlink into the operator-owned cache blob. + link := s.layerBlobPath(snapshotID(t, ctx, s, target)) + fi, err := os.Lstat(link) + require.NoError(t, err) + assert.NotZero(t, fi.Mode()&os.ModeSymlink, "layer.erofs should be a symlink") + dst, err := os.Readlink(link) + require.NoError(t, err) + assert.True(t, filepath.IsAbs(dst), "symlink target should be absolute") + assert.Equal(t, blob, dst) +} + +// TestCacheSidecar covers a hit in the default "auto" dm-verity mode where the +// cache entry has a sidecar: it must be copied into the snapshot dir as a plain +// regular file (not symlinked). +func TestCacheSidecar(t *testing.T) { + ctx := namespaces.WithNamespace(context.Background(), "test") + + cacheDir := t.TempDir() + diffID := digest.Digest(cacheTestDiffID) + blob := writeCacheBlob(t, cacheDir, diffID, []byte("fake erofs blob")) + require.NoError(t, os.WriteFile(dmverity.MetadataPath(blob), []byte(testDmverityMetadata), 0644)) + + // dmverity_mode defaults to "auto": use the sidecar if present. + s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir)) + + target := cacheTestChainID + prepareCacheHit(t, ctx, s, target, diffID) + + // The sidecar is copied in as a plain regular file (not a symlink) so mount-time + // metadata resolution is independent of the cache filesystem. + sidecar := dmverity.MetadataPath(s.layerBlobPath(snapshotID(t, ctx, s, target))) + fi, err := os.Lstat(sidecar) + require.NoError(t, err, "sidecar should be copied into the snapshot dir") + assert.Zero(t, fi.Mode()&os.ModeSymlink, "sidecar should be a regular file, not a symlink") + data, err := os.ReadFile(sidecar) + require.NoError(t, err) + assert.Equal(t, testDmverityMetadata, string(data)) +} + +// TestCacheMiss covers the cases that must NOT be served from the cache and +// instead create a normal active snapshot: cache disabled, blob absent, a +// label-less (container-rootfs) Prepare, and a View (which the KindActive gate +// excludes even with matching labels and a cached blob). +func TestCacheMiss(t *testing.T) { + ctx := namespaces.WithNamespace(context.Background(), "test") + diffID := digest.Digest(cacheTestDiffID) + target := cacheTestChainID + + // Each case must leave the extraction as a normal active snapshot: mounts are + // returned and the target chainID is not committed. + assertFellThrough := func(t *testing.T, s *snapshotter, mounts []mount.Mount, err error) { + t.Helper() + require.NoError(t, err) + assert.NotEmpty(t, mounts, "a miss must return normal active-snapshot mounts") + _, err = s.Stat(ctx, target) + assert.Error(t, err, "target chainID must not be committed on a miss") + } + + t.Run("cache disabled", func(t *testing.T) { + s := newCacheSnapshotter(t) // no cache configured + mounts, err := s.Prepare(ctx, "extract-1 "+target, "", extractionOpt(target, diffID)) + assertFellThrough(t, s, mounts, err) + }) + + t.Run("blob absent", func(t *testing.T) { + s := newCacheSnapshotter(t, WithLayerContentCache(t.TempDir())) + mounts, err := s.Prepare(ctx, "extract-1 "+target, "", extractionOpt(target, diffID)) + assertFellThrough(t, s, mounts, err) + }) + + t.Run("no extraction labels", func(t *testing.T) { + cacheDir := t.TempDir() + writeCacheBlob(t, cacheDir, diffID, []byte("blob")) + s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir)) + // A container-rootfs Prepare carries no snapshot.ref/diff-id labels. + mounts, err := s.Prepare(ctx, "container-rootfs", "") + require.NoError(t, err) + assert.NotEmpty(t, mounts) + }) + + t.Run("view is never short-circuited", func(t *testing.T) { + cacheDir := t.TempDir() + writeCacheBlob(t, cacheDir, diffID, []byte("blob")) + s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir)) + // Even with matching labels and a cached blob, a View must not commit. + mounts, err := s.View(ctx, "view-1", "", extractionOpt(target, diffID)) + assertFellThrough(t, s, mounts, err) + }) +} + +// TestCacheRemove covers removal of a cache-hit snapshot: it succeeds (the +// setImmutable guard skips the symlink), removes the snapshot dir/symlink, and +// leaves the operator-owned cache blob and sidecar untouched. +func TestCacheRemove(t *testing.T) { + ctx := namespaces.WithNamespace(context.Background(), "test") + + cacheDir := t.TempDir() + diffID := digest.Digest(cacheTestDiffID) + blob := writeCacheBlob(t, cacheDir, diffID, []byte("fake erofs blob")) + sidecar := dmverity.MetadataPath(blob) + require.NoError(t, os.WriteFile(sidecar, []byte(testDmverityMetadata), 0644)) + + s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir)) + + target := cacheTestChainID + prepareCacheHit(t, ctx, s, target, diffID) + + snapDir := filepath.Dir(s.layerBlobPath(snapshotID(t, ctx, s, target))) + + // Remove must succeed (the symlinked blob is skipped by the setImmutable guard, + // which would otherwise follow the link and ioctl the operator-owned blob). + require.NoError(t, s.Remove(ctx, target)) + + // The snapshot dir (and its symlink) is gone, but the cache is untouched. + _, err := os.Stat(snapDir) + assert.True(t, os.IsNotExist(err), "snapshot dir should be removed") + _, err = os.Stat(blob) + require.NoError(t, err, "cache blob must be untouched by Remove") + _, err = os.Stat(sidecar) + require.NoError(t, err, "cache sidecar must be untouched by Remove") +} + +// TestCacheDmverity covers dmverity_mode="on": a cache entry with a sidecar is +// committed (and the sidecar copied), while an entry missing its required +// sidecar is a hard error (not a hit, nothing committed). +func TestCacheDmverity(t *testing.T) { + if supported, err := dmverity.IsSupported(); err != nil || !supported { + t.Skip("dm-verity is not supported on this system") + } + ctx := namespaces.WithNamespace(context.Background(), "test") + diffID := digest.Digest(cacheTestDiffID) + target := cacheTestChainID + + t.Run("with sidecar commits and copies it", func(t *testing.T) { + cacheDir := t.TempDir() + blob := writeCacheBlob(t, cacheDir, diffID, []byte("fake erofs blob")) + require.NoError(t, os.WriteFile(dmverity.MetadataPath(blob), []byte(testDmverityMetadata), 0644)) + + s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir), WithDmverityMode("on")) + prepareCacheHit(t, ctx, s, target, diffID) + + _, err := os.Stat(dmverity.MetadataPath(s.layerBlobPath(snapshotID(t, ctx, s, target)))) + require.NoError(t, err, "sidecar must be present for a dmverity_mode=on hit") + }) + + t.Run("without sidecar fails the pull", func(t *testing.T) { + cacheDir := t.TempDir() + writeCacheBlob(t, cacheDir, diffID, []byte("fake erofs blob")) // no sidecar + + s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir), WithDmverityMode("on")) + + // dmverity_mode=on requires a sidecar; a cache entry without one is a hard + // error rather than a silent fallback. + _, err := s.Prepare(ctx, "extract-1 "+target, "", extractionOpt(target, diffID)) + require.Error(t, err) + assert.False(t, errdefs.IsAlreadyExists(err), "missing sidecar must not be treated as a hit") + _, err = s.Stat(ctx, target) + assert.Error(t, err, "no snapshot should be committed on failure") + }) +} diff --git a/plugins/snapshots/erofs/plugin/plugin.go b/plugins/snapshots/erofs/plugin/plugin.go index 834767f714..adf387b12f 100644 --- a/plugins/snapshots/erofs/plugin/plugin.go +++ b/plugins/snapshots/erofs/plugin/plugin.go @@ -55,6 +55,11 @@ type Config struct { // DmverityMode controls dm-verity behavior: "auto" (use if available), "on" (require), "off" (disable) // Linux only DmverityMode string `toml:"dmverity_mode"` + + // LayerContentCache is a directory of pre-converted, diffID-keyed erofs + // layer blobs. When set, layers already present in the cache are committed + // without being downloaded or converted. Empty disables the feature. + LayerContentCache string `toml:"layer_content_cache"` } func init() { @@ -100,6 +105,10 @@ func init() { opts = append(opts, erofs.WithDmverityMode(config.DmverityMode)) } + if config.LayerContentCache != "" { + opts = append(opts, erofs.WithLayerContentCache(config.LayerContentCache)) + } + // Don't bother supporting overlay's slow_chown, only RemapIDs ic.Meta.Capabilities = append(ic.Meta.Capabilities, capaOnlyRemapIDs) if ok, err := supportsIDMappedMounts(); err == nil && ok { @@ -108,7 +117,19 @@ func init() { } ic.Meta.Exports[plugins.SnapshotterRootDir] = root - ic.Meta.Capabilities = append(ic.Meta.Capabilities, "rebase") + // The "rebase" capability lets the unpacker unpack layers in parallel + // via a deferred commit: Prepare receives no parent and the real parent + // is applied at Commit time. The layer content cache is incompatible + // with that — it commits the layer during Prepare (returning + // ErrAlreadyExists), when the parent is not yet known in parallel mode, + // so the committed layer would be parentless and the chain would break. + // With the cache enabled we therefore unpack sequentially. Cache hits + // skip the download and conversion anyway, but a cache *miss* is then + // slower than a cold pull on an uncached node. + // TODO: keep "rebase" and defer the cache commit so misses stay parallel. + if config.LayerContentCache == "" { + ic.Meta.Capabilities = append(ic.Meta.Capabilities, "rebase") + } return erofs.NewSnapshotter(root, opts...) }, })