diff --git a/cmd/ctr/commands/images/build_erofs_cache.go b/cmd/ctr/commands/images/build_erofs_cache.go index 5cfb7ffd1..95220f8b8 100644 --- a/cmd/ctr/commands/images/build_erofs_cache.go +++ b/cmd/ctr/commands/images/build_erofs_cache.go @@ -33,6 +33,7 @@ import ( "github.com/containerd/containerd/v2/core/content" "github.com/containerd/containerd/v2/core/images" "github.com/containerd/containerd/v2/core/images/converter/erofs" + "github.com/containerd/containerd/v2/internal/dmverity" "github.com/containerd/containerd/v2/internal/erofsutils" ) @@ -41,10 +42,15 @@ var buildErofsCacheCommand = &cli.Command{ Usage: "Build an erofs layer content cache from an image's layers", ArgsUsage: "[flags] ", Description: `Convert each layer of an already-pulled image into a directory of -diffID-keyed erofs blobs (//.erofs) for the erofs +diffID-keyed erofs blobs (///.erofs, where is +the first two characters of ) for the erofs snapshotter's layer_content_cache. Layers are read from the content store; no converted image is produced. The directory can then be synced to the read-only -location the fleet mounts. Requires mkfs.erofs.`, +location the fleet mounts. Requires mkfs.erofs. + +Pass --dmverity to also generate a dm-verity hash tree and .dmverity sidecar for +each blob (Linux only); this is required when the snapshotter runs with +dmverity_mode=on, which rejects cache hits that lack a sidecar.`, Flags: []cli.Flag{ &cli.StringFlag{ Name: "compressors", @@ -58,6 +64,10 @@ location the fleet mounts. Requires mkfs.erofs.`, Name: "platform", Usage: "Cache layers for a specific platform (default: host platform)", }, + &cli.BoolFlag{ + Name: "dmverity", + Usage: "Generate a dm-verity hash tree and .dmverity sidecar for each blob (Linux only)", + }, }, Action: func(cliContext *cli.Context) error { ref := cliContext.Args().Get(0) @@ -94,9 +104,9 @@ location the fleet mounts. Requires mkfs.erofs.`, return err } - if err := buildCache(ctx, client.ContentStore(), img.Target, platform, cacheDir, opts...); err != nil { + if err := buildCache(ctx, client.ContentStore(), img.Target, platform, cacheDir, cliContext.Bool("dmverity"), opts...); err != nil { if errdefs.IsNotFound(err) { - return fmt.Errorf("%w; fetch the image content first, e.g. `ctr content fetch %s`", err, ref) + return fmt.Errorf("fetch the image content first, e.g. `ctr content fetch %s`: %w", ref, err) } return err } @@ -111,9 +121,9 @@ location the fleet mounts. Requires mkfs.erofs.`, // the cache can be populated from an already-pulled image and then synced to the // read-only location the fleet mounts. Because the key is the source layer's // diffID, it matches what the runtime looks up when pulling the original image, -// and layers shared across images converge on one blob. dm-verity sidecars are -// not produced here (the converter does no dm-verity formatting). -func buildCache(ctx context.Context, cs content.Store, image ocispec.Descriptor, platform platforms.MatchComparer, cacheDir string, opts ...erofs.ConvertOpt) error { +// and layers shared across images converge on one blob. When withDmverity is set +// a dm-verity hash tree and .dmverity sidecar are also produced per blob. +func buildCache(ctx context.Context, cs content.Store, image ocispec.Descriptor, platform platforms.MatchComparer, cacheDir string, withDmverity bool, opts ...erofs.ConvertOpt) error { manifest, err := images.Manifest(ctx, cs, image, platform) if err != nil { return fmt.Errorf("failed to resolve manifest: %w", err) @@ -123,7 +133,7 @@ func buildCache(ctx context.Context, cs content.Store, image ocispec.Descriptor, if !images.IsLayerType(layer.MediaType) || erofsutils.IsErofsMediaType(layer.MediaType) || images.IsNonDistributable(layer.MediaType) { continue } - if err := buildLayer(ctx, cs, layer, cacheDir, opts...); err != nil { + if err := buildLayer(ctx, cs, layer, cacheDir, withDmverity, opts...); err != nil { // Preserve errdefs.IsNotFound so callers can add context-appropriate // remediation (e.g. ctr suggesting `ctr content fetch`). if errdefs.IsNotFound(err) { @@ -138,8 +148,9 @@ func buildCache(ctx context.Context, cs content.Store, image ocispec.Descriptor, // buildLayer converts one layer into a temp file inside cacheDir, then // atomically renames it to its diffID-keyed name (same filesystem, so no extra // copy and no reliance on TMPDIR). An existing entry is overwritten, keeping the -// operation idempotent across re-runs and shared layers. -func buildLayer(ctx context.Context, cs content.Store, layer ocispec.Descriptor, cacheDir string, opts ...erofs.ConvertOpt) (retErr error) { +// operation idempotent across re-runs and shared layers. When withDmverity is +// set the blob is dm-verity formatted and its sidecar is moved into place too. +func buildLayer(ctx context.Context, cs content.Store, layer ocispec.Descriptor, cacheDir string, withDmverity bool, opts ...erofs.ConvertOpt) (retErr error) { if err := os.MkdirAll(cacheDir, 0755); err != nil { return err } @@ -152,6 +163,7 @@ func buildLayer(ctx context.Context, cs content.Store, layer ocispec.Descriptor, defer func() { if retErr != nil { os.Remove(tmpPath) + os.Remove(dmverity.MetadataPath(tmpPath)) } }() @@ -160,15 +172,62 @@ func buildLayer(ctx context.Context, cs content.Store, layer ocispec.Descriptor, return err } + if withDmverity { + // Appends the hash tree to the blob and writes tmpPath's .dmverity sidecar. + if err := dmverity.FormatLayer(ctx, tmpPath, nil); err != nil { + return err + } + } + // os.CreateTemp makes the file 0600; widen it so other users (e.g. a // rootless containerd) can read the shared cache. if err := os.Chmod(tmpPath, 0644); err != nil { return err } - dest := filepath.Join(cacheDir, diffID.Algorithm().String(), diffID.Encoded()+".erofs") - if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil { + dest := erofsutils.CacheBlobPath(cacheDir, diffID) + destDir := filepath.Dir(dest) + if err := os.MkdirAll(destDir, 0755); err != nil { return err } - return os.Rename(tmpPath, dest) + + // fsync the temp files before renaming so a crash right after the rename can't + // leave a zero-length or torn cache entry. A missing sidecar (dm-verity + // disabled) is skipped. + for _, p := range []string{tmpPath, dmverity.MetadataPath(tmpPath)} { + if err := fsync(p); err != nil { + return err + } + } + + // Move the sidecar into place before the blob: the snapshotter keys on the + // blob's presence, so the sidecar must already exist when the blob appears. + if withDmverity { + if err := os.Rename(dmverity.MetadataPath(tmpPath), dmverity.MetadataPath(dest)); err != nil { + return err + } + } + if err := os.Rename(tmpPath, dest); err != nil { + return err + } + + // fsync the directory so the renames themselves survive a crash. + return fsync(destDir) +} + +// fsync flushes the file (or directory) at path to disk. A path that does not +// exist is treated as a no-op so callers can fsync an optional sidecar. +func fsync(path string) error { + f, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if err := f.Sync(); err != nil { + f.Close() + return err + } + return f.Close() } diff --git a/core/images/converter/erofs/erofs.go b/core/images/converter/erofs/erofs.go index 41b3ba615..7f1f72a1f 100644 --- a/core/images/converter/erofs/erofs.go +++ b/core/images/converter/erofs/erofs.go @@ -204,7 +204,10 @@ func ConvertLayerToErofs(ctx context.Context, cs content.Store, desc ocispec.Des mkfsArgs = append(mkfsArgs, copts.mkfsExtraOpts...) mkfsArgs = erofsutils.AddDefaultMkfsOpts(mkfsArgs) - u := uuid.NewSHA1(uuid.NameSpaceURL, []byte("erofs:blobs/"+desc.Digest)) + // Derive the erofs UUID from the uncompressed digest (diffID) so the blob is a + // deterministic function of the layer content, independent of how the source + // was compressed. This is also the key the layer content cache uses. + u := uuid.NewSHA1(uuid.NameSpaceURL, []byte("erofs:blobs/"+uncompressedDesc.Digest)) if err := erofsutils.ConvertTarErofs(ctx, sr, outPath, u.String(), mkfsArgs); err != nil { return "", fmt.Errorf("failed to convert to EROFS: %w", err) } diff --git a/internal/dmverity/dmverity_linux.go b/internal/dmverity/dmverity_linux.go index ccaf98829..534bb9b2d 100644 --- a/internal/dmverity/dmverity_linux.go +++ b/internal/dmverity/dmverity_linux.go @@ -17,10 +17,16 @@ package dmverity import ( + "context" + "encoding/json" "fmt" "os" + "github.com/containerd/log" + "github.com/google/uuid" + "github.com/containerd/containerd/v2/core/mount" + "github.com/containerd/containerd/v2/pkg/atomicfile" "github.com/containerd/go-dmverity/pkg/utils" "github.com/containerd/go-dmverity/pkg/verity" ) @@ -111,6 +117,111 @@ func Format(dataDevice, hashDevice string, opts *DmverityOptions) (string, error return fmt.Sprintf("%x", rootDigest), nil } +// FormatLayer appends a dm-verity hash tree to the erofs layer blob at +// layerBlobPath (growing the file in place) and writes the ".dmverity" +// sidecar holding the resulting root hash and superblock offset. It is a no-op +// if the sidecar already exists. A nil opts uses DefaultDmverityOptions. +func FormatLayer(ctx context.Context, layerBlobPath string, opts *DmverityOptions) error { + metadataPath := MetadataPath(layerBlobPath) + if _, err := os.Stat(metadataPath); err == nil { + log.G(ctx).WithField("path", layerBlobPath).Debug("Layer already formatted with dm-verity, skipping") + return nil + } + + if opts == nil { + opts = DefaultDmverityOptions() + } else { + clone := *opts + opts = &clone + } + + fileInfo, err := os.Stat(layerBlobPath) + if err != nil { + return fmt.Errorf("failed to stat layer blob: %w", err) + } + + blockSize := int64(opts.DataBlockSize) + fileSize := fileInfo.Size() + + // dm-verity requires the hash area to start at a block-aligned offset + dataBlocks := (fileSize + blockSize - 1) / blockSize + hashOffset := uint64(dataBlocks * blockSize) + + opts.HashOffset = hashOffset + opts.DataBlocks = uint64(dataBlocks) + + hashTreeSize, err := verity.GetHashTreeSize(&verity.Params{ + HashName: opts.HashAlgorithm, + DataBlockSize: opts.DataBlockSize, + HashBlockSize: opts.HashBlockSize, + DataBlocks: opts.DataBlocks, + HashType: opts.HashType, + }) + if err != nil { + return fmt.Errorf("failed to calculate hash tree size: %w", err) + } + + // In superblock mode, Format() stores the superblock at hashOffset and the hash tree after it + superblockSize := uint64(0) + if !opts.NoSuperblock { + superblockSize = utils.AlignUp(uint64(verity.SuperblockSize), uint64(opts.HashBlockSize)) + } + requiredSize := hashOffset + superblockSize + hashTreeSize + if err := os.Truncate(layerBlobPath, int64(requiredSize)); err != nil { + return fmt.Errorf("failed to pre-allocate space for hash tree: %w", err) + } + + // Generate a random UUID for the superblock (required for superblock mode). + // The library's ReadSuperblock() validates that UUID is not nil/empty. + if opts.UUID == "" { + u, err := uuid.NewRandom() + if err != nil { + return fmt.Errorf("failed to generate superblock UUID: %w", err) + } + opts.UUID = u.String() + } + + rootHash, err := Format(layerBlobPath, layerBlobPath, opts) + if err != nil { + return fmt.Errorf("failed to format dm-verity: %w", err) + } + + // Save the ORIGINAL hashOffset (where the superblock is located), not the + // post-Format offset (which points past the superblock). Open() needs the + // superblock location to read device parameters. + metadata := DmverityMetadata{ + RootHash: rootHash, + HashOffset: hashOffset, + } + metadataBytes, err := json.MarshalIndent(metadata, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal dm-verity metadata: %w", err) + } + // Write the sidecar atomically (temp file, synced, renamed) so a reader never + // observes a partially written .dmverity. + f, err := atomicfile.New(metadataPath, 0644) + if err != nil { + return fmt.Errorf("failed to create dm-verity metadata file: %w", err) + } + if _, err := f.Write(metadataBytes); err != nil { + f.Cancel() + return fmt.Errorf("failed to write dm-verity metadata: %w", err) + } + if err := f.Close(); err != nil { + return fmt.Errorf("failed to write dm-verity metadata: %w", err) + } + + log.G(ctx).WithFields(log.Fields{ + "path": layerBlobPath, + "size": fileSize, + "blockSize": opts.DataBlockSize, + "hashOffset": hashOffset, + "rootHash": rootHash, + }).Debug("Successfully formatted dm-verity layer") + + return nil +} + // Open creates a read-only device-mapper target for transparent integrity verification. // It supports both superblock and no-superblock modes: // diff --git a/internal/dmverity/dmverity_other.go b/internal/dmverity/dmverity_other.go index 515edfff5..745a58767 100644 --- a/internal/dmverity/dmverity_other.go +++ b/internal/dmverity/dmverity_other.go @@ -18,7 +18,10 @@ package dmverity -import "fmt" +import ( + "context" + "fmt" +) var errUnsupported = fmt.Errorf("dmverity is only supported on Linux systems") @@ -30,6 +33,10 @@ func Format(_ string, _ string, _ *DmverityOptions) (string, error) { return "", errUnsupported } +func FormatLayer(_ context.Context, _ string, _ *DmverityOptions) error { + return errUnsupported +} + func Open(_ string, _ string, _ string, _ string, _ uint64, _ *DmverityOptions) (string, error) { return "", errUnsupported } diff --git a/plugins/diff/erofs/dmverity_linux.go b/plugins/diff/erofs/dmverity_linux.go index f59c3093d..53e7cb544 100644 --- a/plugins/diff/erofs/dmverity_linux.go +++ b/plugins/diff/erofs/dmverity_linux.go @@ -20,14 +20,6 @@ package erofs import ( "context" - "encoding/json" - "fmt" - "os" - - "github.com/containerd/go-dmverity/pkg/utils" - "github.com/containerd/go-dmverity/pkg/verity" - "github.com/containerd/log" - "github.com/google/uuid" "github.com/containerd/containerd/v2/internal/dmverity" ) @@ -53,84 +45,8 @@ func (s *erofsDiff) getDmverityOptions() *dmverity.DmverityOptions { return opts } -// formatDmverityLayer formats an EROFS layer with dm-verity hash tree +// formatDmverityLayer formats an EROFS layer with a dm-verity hash tree using +// the differ's configured options. func (s *erofsDiff) formatDmverityLayer(ctx context.Context, layerBlobPath string) error { - metadataPath := dmverity.MetadataPath(layerBlobPath) - if _, err := os.Stat(metadataPath); err == nil { - log.G(ctx).WithField("path", layerBlobPath).Debug("Layer already formatted with dm-verity, skipping") - return nil - } - - fileInfo, err := os.Stat(layerBlobPath) - if err != nil { - return fmt.Errorf("failed to stat layer blob: %w", err) - } - - opts := s.getDmverityOptions() - blockSize := int64(opts.DataBlockSize) - fileSize := fileInfo.Size() - - // dm-verity requires the hash area to start at a block-aligned offset - dataBlocks := (fileSize + blockSize - 1) / blockSize - hashOffset := uint64(dataBlocks * blockSize) - - opts.HashOffset = hashOffset - opts.DataBlocks = uint64(dataBlocks) - - hashTreeSize, err := verity.GetHashTreeSize(&verity.Params{ - HashName: opts.HashAlgorithm, - DataBlockSize: opts.DataBlockSize, - HashBlockSize: opts.HashBlockSize, - DataBlocks: opts.DataBlocks, - HashType: opts.HashType, - }) - if err != nil { - return fmt.Errorf("failed to calculate hash tree size: %w", err) - } - - // In superblock mode, Format() stores the superblock at hashOffset and the hash tree after it - superblockSize := uint64(0) - if !opts.NoSuperblock { - superblockSize = utils.AlignUp(uint64(verity.SuperblockSize), uint64(opts.HashBlockSize)) - } - requiredSize := hashOffset + superblockSize + hashTreeSize - if err := os.Truncate(layerBlobPath, int64(requiredSize)); err != nil { - return fmt.Errorf("failed to pre-allocate space for hash tree: %w", err) - } - - // Generate a random UUID for the superblock (required for superblock mode) - // The library's ReadSuperblock() validates that UUID is not nil/empty - if opts.UUID == "" { - opts.UUID = uuid.New().String() - } - - rootHash, err := dmverity.Format(layerBlobPath, layerBlobPath, opts) - if err != nil { - return fmt.Errorf("failed to format dm-verity: %w", err) - } - - // Important: Save the ORIGINAL hashOffset (where superblock is located), - // not result.HashOffset (which points to where the hash tree starts after the superblock). - // Open() needs the superblock location to read device parameters. - metadata := dmverity.DmverityMetadata{ - RootHash: rootHash, - HashOffset: hashOffset, - } - metadataBytes, err := json.MarshalIndent(metadata, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal dm-verity metadata: %w", err) - } - if err := os.WriteFile(metadataPath, metadataBytes, 0644); err != nil { - return fmt.Errorf("failed to write dm-verity metadata: %w", err) - } - - log.G(ctx).WithFields(log.Fields{ - "path": layerBlobPath, - "size": fileSize, - "blockSize": opts.DataBlockSize, - "hashOffset": hashOffset, - "rootHash": rootHash, - }).Info("Successfully formatted dm-verity layer") - - return nil + return dmverity.FormatLayer(ctx, layerBlobPath, s.getDmverityOptions()) }