Support dmverity

Add a --dmverity flag to `ctr images build-erofs-cache`. When set, each cached
erofs blob is dm-verity formatted (the hash tree is appended in place) and a
.dmverity sidecar is written alongside it. This is required when the erofs
snapshotter runs with dmverity_mode=on, which rejects cache hits that lack a
sidecar; without it such layers would have to be formatted out-of-band.

Extract the differ's dm-verity formatting into a shared dmverity.FormatLayer so
the differ and the cache builder share one implementation; the differ's
formatDmverityLayer now delegates to it.

Signed-off-by: Maksym Pavlenko <pavlenko.maksym@gmail.com>

cmd/ctr/commands/images/build_erofs_cache.go

@@ -33,6 +33,7 @@ import (
 	"github.com/containerd/containerd/v2/core/content"
 	"github.com/containerd/containerd/v2/core/images"
 	"github.com/containerd/containerd/v2/core/images/converter/erofs"
+	"github.com/containerd/containerd/v2/internal/dmverity"
 	"github.com/containerd/containerd/v2/internal/erofsutils"
 )
 
@@ -41,10 +42,15 @@ var buildErofsCacheCommand = &cli.Command{
 	Usage:     "Build an erofs layer content cache from an image's layers",
 	ArgsUsage: "[flags] <image_ref> <cache_dir>",
 	Description: `Convert each layer of an already-pulled image into a directory of
-diffID-keyed erofs blobs (<cache_dir>/<algorithm>/<hex>.erofs) for the erofs
+diffID-keyed erofs blobs (<cache_dir>/<algorithm>/<xx>/<hex>.erofs, where <xx> is
+the first two characters of <hex>) for the erofs
 snapshotter's layer_content_cache. Layers are read from the content store; no
 converted image is produced. The directory can then be synced to the read-only
-location the fleet mounts. Requires mkfs.erofs.`,
+location the fleet mounts. Requires mkfs.erofs.
+
+Pass --dmverity to also generate a dm-verity hash tree and .dmverity sidecar for
+each blob (Linux only); this is required when the snapshotter runs with
+dmverity_mode=on, which rejects cache hits that lack a sidecar.`,
 	Flags: []cli.Flag{
 		&cli.StringFlag{
 			Name:  "compressors",
@@ -58,6 +64,10 @@ location the fleet mounts. Requires mkfs.erofs.`,
 			Name:  "platform",
 			Usage: "Cache layers for a specific platform (default: host platform)",
 		},
+		&cli.BoolFlag{
+			Name:  "dmverity",
+			Usage: "Generate a dm-verity hash tree and .dmverity sidecar for each blob (Linux only)",
+		},
 	},
 	Action: func(cliContext *cli.Context) error {
 		ref := cliContext.Args().Get(0)
@@ -94,9 +104,9 @@ location the fleet mounts. Requires mkfs.erofs.`,
 			return err
 		}
 
-		if err := buildCache(ctx, client.ContentStore(), img.Target, platform, cacheDir, opts...); err != nil {
+		if err := buildCache(ctx, client.ContentStore(), img.Target, platform, cacheDir, cliContext.Bool("dmverity"), opts...); err != nil {
 			if errdefs.IsNotFound(err) {
-				return fmt.Errorf("%w; fetch the image content first, e.g. `ctr content fetch %s`", err, ref)
+				return fmt.Errorf("fetch the image content first, e.g. `ctr content fetch %s`: %w", ref, err)
 			}
 			return err
 		}
@@ -111,9 +121,9 @@ location the fleet mounts. Requires mkfs.erofs.`,
 // the cache can be populated from an already-pulled image and then synced to the
 // read-only location the fleet mounts. Because the key is the source layer's
 // diffID, it matches what the runtime looks up when pulling the original image,
-// and layers shared across images converge on one blob. dm-verity sidecars are
-// not produced here (the converter does no dm-verity formatting).
-func buildCache(ctx context.Context, cs content.Store, image ocispec.Descriptor, platform platforms.MatchComparer, cacheDir string, opts ...erofs.ConvertOpt) error {
+// and layers shared across images converge on one blob. When withDmverity is set
+// a dm-verity hash tree and .dmverity sidecar are also produced per blob.
+func buildCache(ctx context.Context, cs content.Store, image ocispec.Descriptor, platform platforms.MatchComparer, cacheDir string, withDmverity bool, opts ...erofs.ConvertOpt) error {
 	manifest, err := images.Manifest(ctx, cs, image, platform)
 	if err != nil {
 		return fmt.Errorf("failed to resolve manifest: %w", err)
@@ -123,7 +133,7 @@ func buildCache(ctx context.Context, cs content.Store, image ocispec.Descriptor,
 		if !images.IsLayerType(layer.MediaType) || erofsutils.IsErofsMediaType(layer.MediaType) || images.IsNonDistributable(layer.MediaType) {
 			continue
 		}
-		if err := buildLayer(ctx, cs, layer, cacheDir, opts...); err != nil {
+		if err := buildLayer(ctx, cs, layer, cacheDir, withDmverity, opts...); err != nil {
 			// Preserve errdefs.IsNotFound so callers can add context-appropriate
 			// remediation (e.g. ctr suggesting `ctr content fetch`).
 			if errdefs.IsNotFound(err) {
@@ -138,8 +148,9 @@ func buildCache(ctx context.Context, cs content.Store, image ocispec.Descriptor,
 // buildLayer converts one layer into a temp file inside cacheDir, then
 // atomically renames it to its diffID-keyed name (same filesystem, so no extra
 // copy and no reliance on TMPDIR). An existing entry is overwritten, keeping the
-// operation idempotent across re-runs and shared layers.
-func buildLayer(ctx context.Context, cs content.Store, layer ocispec.Descriptor, cacheDir string, opts ...erofs.ConvertOpt) (retErr error) {
+// operation idempotent across re-runs and shared layers. When withDmverity is
+// set the blob is dm-verity formatted and its sidecar is moved into place too.
+func buildLayer(ctx context.Context, cs content.Store, layer ocispec.Descriptor, cacheDir string, withDmverity bool, opts ...erofs.ConvertOpt) (retErr error) {
 	if err := os.MkdirAll(cacheDir, 0755); err != nil {
 		return err
 	}
@@ -152,6 +163,7 @@ func buildLayer(ctx context.Context, cs content.Store, layer ocispec.Descriptor,
 	defer func() {
 		if retErr != nil {
 			os.Remove(tmpPath)
+			os.Remove(dmverity.MetadataPath(tmpPath))
 		}
 	}()
 
@@ -160,15 +172,62 @@ func buildLayer(ctx context.Context, cs content.Store, layer ocispec.Descriptor,
 		return err
 	}
 
+	if withDmverity {
+		// Appends the hash tree to the blob and writes tmpPath's .dmverity sidecar.
+		if err := dmverity.FormatLayer(ctx, tmpPath, nil); err != nil {
+			return err
+		}
+	}
+
 	// os.CreateTemp makes the file 0600; widen it so other users (e.g. a
 	// rootless containerd) can read the shared cache.
 	if err := os.Chmod(tmpPath, 0644); err != nil {
 		return err
 	}
 
-	dest := filepath.Join(cacheDir, diffID.Algorithm().String(), diffID.Encoded()+".erofs")
-	if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
+	dest := erofsutils.CacheBlobPath(cacheDir, diffID)
+	destDir := filepath.Dir(dest)
+	if err := os.MkdirAll(destDir, 0755); err != nil {
 		return err
 	}
-	return os.Rename(tmpPath, dest)
+
+	// fsync the temp files before renaming so a crash right after the rename can't
+	// leave a zero-length or torn cache entry. A missing sidecar (dm-verity
+	// disabled) is skipped.
+	for _, p := range []string{tmpPath, dmverity.MetadataPath(tmpPath)} {
+		if err := fsync(p); err != nil {
+			return err
+		}
+	}
+
+	// Move the sidecar into place before the blob: the snapshotter keys on the
+	// blob's presence, so the sidecar must already exist when the blob appears.
+	if withDmverity {
+		if err := os.Rename(dmverity.MetadataPath(tmpPath), dmverity.MetadataPath(dest)); err != nil {
+			return err
+		}
+	}
+	if err := os.Rename(tmpPath, dest); err != nil {
+		return err
+	}
+
+	// fsync the directory so the renames themselves survive a crash.
+	return fsync(destDir)
+}
+
+// fsync flushes the file (or directory) at path to disk. A path that does not
+// exist is treated as a no-op so callers can fsync an optional sidecar.
+func fsync(path string) error {
+	f, err := os.Open(path)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	if err := f.Sync(); err != nil {
+		f.Close()
+		return err
+	}
+	return f.Close()
 }

core/images/converter/erofs/erofs.go

@@ -204,7 +204,10 @@ func ConvertLayerToErofs(ctx context.Context, cs content.Store, desc ocispec.Des
 	mkfsArgs = append(mkfsArgs, copts.mkfsExtraOpts...)
 	mkfsArgs = erofsutils.AddDefaultMkfsOpts(mkfsArgs)
 
-	u := uuid.NewSHA1(uuid.NameSpaceURL, []byte("erofs:blobs/"+desc.Digest))
+	// Derive the erofs UUID from the uncompressed digest (diffID) so the blob is a
+	// deterministic function of the layer content, independent of how the source
+	// was compressed. This is also the key the layer content cache uses.
+	u := uuid.NewSHA1(uuid.NameSpaceURL, []byte("erofs:blobs/"+uncompressedDesc.Digest))
 	if err := erofsutils.ConvertTarErofs(ctx, sr, outPath, u.String(), mkfsArgs); err != nil {
 		return "", fmt.Errorf("failed to convert to EROFS: %w", err)
 	}

internal/dmverity/dmverity_linux.go

@@ -17,10 +17,16 @@
 package dmverity
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 
+	"github.com/containerd/log"
+	"github.com/google/uuid"
+
 	"github.com/containerd/containerd/v2/core/mount"
+	"github.com/containerd/containerd/v2/pkg/atomicfile"
 	"github.com/containerd/go-dmverity/pkg/utils"
 	"github.com/containerd/go-dmverity/pkg/verity"
 )
@@ -111,6 +117,111 @@ func Format(dataDevice, hashDevice string, opts *DmverityOptions) (string, error
 	return fmt.Sprintf("%x", rootDigest), nil
 }
 
+// FormatLayer appends a dm-verity hash tree to the erofs layer blob at
+// layerBlobPath (growing the file in place) and writes the "<blob>.dmverity"
+// sidecar holding the resulting root hash and superblock offset. It is a no-op
+// if the sidecar already exists. A nil opts uses DefaultDmverityOptions.
+func FormatLayer(ctx context.Context, layerBlobPath string, opts *DmverityOptions) error {
+	metadataPath := MetadataPath(layerBlobPath)
+	if _, err := os.Stat(metadataPath); err == nil {
+		log.G(ctx).WithField("path", layerBlobPath).Debug("Layer already formatted with dm-verity, skipping")
+		return nil
+	}
+
+	if opts == nil {
+		opts = DefaultDmverityOptions()
+	} else {
+		clone := *opts
+		opts = &clone
+	}
+
+	fileInfo, err := os.Stat(layerBlobPath)
+	if err != nil {
+		return fmt.Errorf("failed to stat layer blob: %w", err)
+	}
+
+	blockSize := int64(opts.DataBlockSize)
+	fileSize := fileInfo.Size()
+
+	// dm-verity requires the hash area to start at a block-aligned offset
+	dataBlocks := (fileSize + blockSize - 1) / blockSize
+	hashOffset := uint64(dataBlocks * blockSize)
+
+	opts.HashOffset = hashOffset
+	opts.DataBlocks = uint64(dataBlocks)
+
+	hashTreeSize, err := verity.GetHashTreeSize(&verity.Params{
+		HashName:      opts.HashAlgorithm,
+		DataBlockSize: opts.DataBlockSize,
+		HashBlockSize: opts.HashBlockSize,
+		DataBlocks:    opts.DataBlocks,
+		HashType:      opts.HashType,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to calculate hash tree size: %w", err)
+	}
+
+	// In superblock mode, Format() stores the superblock at hashOffset and the hash tree after it
+	superblockSize := uint64(0)
+	if !opts.NoSuperblock {
+		superblockSize = utils.AlignUp(uint64(verity.SuperblockSize), uint64(opts.HashBlockSize))
+	}
+	requiredSize := hashOffset + superblockSize + hashTreeSize
+	if err := os.Truncate(layerBlobPath, int64(requiredSize)); err != nil {
+		return fmt.Errorf("failed to pre-allocate space for hash tree: %w", err)
+	}
+
+	// Generate a random UUID for the superblock (required for superblock mode).
+	// The library's ReadSuperblock() validates that UUID is not nil/empty.
+	if opts.UUID == "" {
+		u, err := uuid.NewRandom()
+		if err != nil {
+			return fmt.Errorf("failed to generate superblock UUID: %w", err)
+		}
+		opts.UUID = u.String()
+	}
+
+	rootHash, err := Format(layerBlobPath, layerBlobPath, opts)
+	if err != nil {
+		return fmt.Errorf("failed to format dm-verity: %w", err)
+	}
+
+	// Save the ORIGINAL hashOffset (where the superblock is located), not the
+	// post-Format offset (which points past the superblock). Open() needs the
+	// superblock location to read device parameters.
+	metadata := DmverityMetadata{
+		RootHash:   rootHash,
+		HashOffset: hashOffset,
+	}
+	metadataBytes, err := json.MarshalIndent(metadata, "", "  ")
+	if err != nil {
+		return fmt.Errorf("failed to marshal dm-verity metadata: %w", err)
+	}
+	// Write the sidecar atomically (temp file, synced, renamed) so a reader never
+	// observes a partially written .dmverity.
+	f, err := atomicfile.New(metadataPath, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to create dm-verity metadata file: %w", err)
+	}
+	if _, err := f.Write(metadataBytes); err != nil {
+		f.Cancel()
+		return fmt.Errorf("failed to write dm-verity metadata: %w", err)
+	}
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("failed to write dm-verity metadata: %w", err)
+	}
+
+	log.G(ctx).WithFields(log.Fields{
+		"path":       layerBlobPath,
+		"size":       fileSize,
+		"blockSize":  opts.DataBlockSize,
+		"hashOffset": hashOffset,
+		"rootHash":   rootHash,
+	}).Debug("Successfully formatted dm-verity layer")
+
+	return nil
+}
+
 // Open creates a read-only device-mapper target for transparent integrity verification.
 // It supports both superblock and no-superblock modes:
 //

internal/dmverity/dmverity_other.go

@@ -18,7 +18,10 @@
 
 package dmverity
 
-import "fmt"
+import (
+	"context"
+	"fmt"
+)
 
 var errUnsupported = fmt.Errorf("dmverity is only supported on Linux systems")
 
@@ -30,6 +33,10 @@ func Format(_ string, _ string, _ *DmverityOptions) (string, error) {
 	return "", errUnsupported
 }
 
+func FormatLayer(_ context.Context, _ string, _ *DmverityOptions) error {
+	return errUnsupported
+}
+
 func Open(_ string, _ string, _ string, _ string, _ uint64, _ *DmverityOptions) (string, error) {
 	return "", errUnsupported
 }

plugins/diff/erofs/dmverity_linux.go

@@ -20,14 +20,6 @@ package erofs
 
 import (
 	"context"
-	"encoding/json"
-	"fmt"
-	"os"
-
-	"github.com/containerd/go-dmverity/pkg/utils"
-	"github.com/containerd/go-dmverity/pkg/verity"
-	"github.com/containerd/log"
-	"github.com/google/uuid"
 
 	"github.com/containerd/containerd/v2/internal/dmverity"
 )
@@ -53,84 +45,8 @@ func (s *erofsDiff) getDmverityOptions() *dmverity.DmverityOptions {
 	return opts
 }
 
-// formatDmverityLayer formats an EROFS layer with dm-verity hash tree
+// formatDmverityLayer formats an EROFS layer with a dm-verity hash tree using
+// the differ's configured options.
 func (s *erofsDiff) formatDmverityLayer(ctx context.Context, layerBlobPath string) error {
-	metadataPath := dmverity.MetadataPath(layerBlobPath)
-	if _, err := os.Stat(metadataPath); err == nil {
-		log.G(ctx).WithField("path", layerBlobPath).Debug("Layer already formatted with dm-verity, skipping")
-		return nil
-	}
-
-	fileInfo, err := os.Stat(layerBlobPath)
-	if err != nil {
-		return fmt.Errorf("failed to stat layer blob: %w", err)
-	}
-
-	opts := s.getDmverityOptions()
-	blockSize := int64(opts.DataBlockSize)
-	fileSize := fileInfo.Size()
-
-	// dm-verity requires the hash area to start at a block-aligned offset
-	dataBlocks := (fileSize + blockSize - 1) / blockSize
-	hashOffset := uint64(dataBlocks * blockSize)
-
-	opts.HashOffset = hashOffset
-	opts.DataBlocks = uint64(dataBlocks)
-
-	hashTreeSize, err := verity.GetHashTreeSize(&verity.Params{
-		HashName:      opts.HashAlgorithm,
-		DataBlockSize: opts.DataBlockSize,
-		HashBlockSize: opts.HashBlockSize,
-		DataBlocks:    opts.DataBlocks,
-		HashType:      opts.HashType,
-	})
-	if err != nil {
-		return fmt.Errorf("failed to calculate hash tree size: %w", err)
-	}
-
-	// In superblock mode, Format() stores the superblock at hashOffset and the hash tree after it
-	superblockSize := uint64(0)
-	if !opts.NoSuperblock {
-		superblockSize = utils.AlignUp(uint64(verity.SuperblockSize), uint64(opts.HashBlockSize))
-	}
-	requiredSize := hashOffset + superblockSize + hashTreeSize
-	if err := os.Truncate(layerBlobPath, int64(requiredSize)); err != nil {
-		return fmt.Errorf("failed to pre-allocate space for hash tree: %w", err)
-	}
-
-	// Generate a random UUID for the superblock (required for superblock mode)
-	// The library's ReadSuperblock() validates that UUID is not nil/empty
-	if opts.UUID == "" {
-		opts.UUID = uuid.New().String()
-	}
-
-	rootHash, err := dmverity.Format(layerBlobPath, layerBlobPath, opts)
-	if err != nil {
-		return fmt.Errorf("failed to format dm-verity: %w", err)
-	}
-
-	// Important: Save the ORIGINAL hashOffset (where superblock is located),
-	// not result.HashOffset (which points to where the hash tree starts after the superblock).
-	// Open() needs the superblock location to read device parameters.
-	metadata := dmverity.DmverityMetadata{
-		RootHash:   rootHash,
-		HashOffset: hashOffset,
-	}
-	metadataBytes, err := json.MarshalIndent(metadata, "", "  ")
-	if err != nil {
-		return fmt.Errorf("failed to marshal dm-verity metadata: %w", err)
-	}
-	if err := os.WriteFile(metadataPath, metadataBytes, 0644); err != nil {
-		return fmt.Errorf("failed to write dm-verity metadata: %w", err)
-	}
-
-	log.G(ctx).WithFields(log.Fields{
-		"path":       layerBlobPath,
-		"size":       fileSize,
-		"blockSize":  opts.DataBlockSize,
-		"hashOffset": hashOffset,
-		"rootHash":   rootHash,
-	}).Info("Successfully formatted dm-verity layer")
-
-	return nil
+	return dmverity.FormatLayer(ctx, layerBlobPath, s.getDmverityOptions())
 }