Introduce EROFS differ

The EROFS differ only applies to EROFS layers which are marked by
a special file `.erofslayer` generated by the EROFS snapshotter.

Why it's needed?  Since we'd like to parse []mount.Mount directly
without actual mounting and convert OCI layers into EROFS blobs,
`.erofslayer` gives a hint that the active snapshotter supports
the output blob generated by the EROFS differ.

I'd suggest it could be read together with the next commit.

Signed-off-by: cardy.tang <zuniorone@gmail.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>

cmd/containerd/builtins/builtins_linux.go

@@ -20,6 +20,7 @@ import (
 	_ "github.com/containerd/containerd/api/types/runc/options"
 	_ "github.com/containerd/containerd/v2/core/metrics/cgroups"
 	_ "github.com/containerd/containerd/v2/core/metrics/cgroups/v2"
+	_ "github.com/containerd/containerd/v2/plugins/diff/erofs/plugin"
 	_ "github.com/containerd/containerd/v2/plugins/diff/walking/plugin"
 	_ "github.com/containerd/containerd/v2/plugins/snapshots/blockfile/plugin"
 	_ "github.com/containerd/containerd/v2/plugins/snapshots/native/plugin"

plugins/diff/erofs/differ_linux.go

@@ -0,0 +1,189 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package erofs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/core/diff"
+	"github.com/containerd/containerd/v2/core/images"
+	"github.com/containerd/containerd/v2/core/mount"
+	"github.com/containerd/errdefs"
+	"github.com/containerd/log"
+	digest "github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+var emptyDesc = ocispec.Descriptor{}
+
+type differ interface {
+	diff.Applier
+	diff.Comparer
+}
+
+// erofsDiff does erofs comparison and application
+type erofsDiff struct {
+	store         content.Store
+	mkfsExtraOpts []string
+}
+
+func NewErofsDiffer(store content.Store, mkfsExtraOpts []string) differ {
+	return &erofsDiff{
+		store:         store,
+		mkfsExtraOpts: mkfsExtraOpts,
+	}
+}
+
+// Compare creates a diff between the given mounts and uploads the result
+// to the content store.
+func (s erofsDiff) Compare(ctx context.Context, lower, upper []mount.Mount, opts ...diff.Opt) (d ocispec.Descriptor, err error) {
+	return emptyDesc, fmt.Errorf("erofsDiff does not implement Compare method: %w", errdefs.ErrNotImplemented)
+}
+
+func convertTarErofs(ctx context.Context, r io.Reader, layerPath string, mkfsExtraOpts []string) error {
+	args := append([]string{"--tar=f", "--aufs", "--quiet", "-Enoinline_data"}, mkfsExtraOpts...)
+	args = append(args, layerPath)
+	cmd := exec.CommandContext(ctx, "mkfs.erofs", args...)
+	cmd.Stdin = r
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("erofs apply failed: %s: %w", out, err)
+	}
+	log.G(ctx).Debugf("running %s %s %v", cmd.Path, cmd.Args, string(out))
+	return nil
+}
+
+// Get the snapshot layer directory in order to generate EROFS-formatted blobs;
+//
+// If mount[0].Type is `bind` or `erofs`, it just tries the source dir; Or if
+// mount[0].Type is `overlayfs`, it tries the parent of the upperdir;
+//
+// The candidate will be checked with ".erofslayer" to make sure this active
+// snapshot is really generated by the EROFS snapshotter instead of others.
+func mountsToLayer(mounts []mount.Mount) (string, error) {
+	var layer string
+	mnt := mounts[0]
+	if mnt.Type == "bind" || mnt.Type == "erofs" {
+		layer = filepath.Dir(mnt.Source)
+	} else if mnt.Type == "overlay" {
+		layer = ""
+		for _, o := range mnt.Options {
+			if strings.HasPrefix(o, "upperdir=") {
+				layer = filepath.Dir(strings.TrimPrefix(o, "upperdir="))
+			}
+		}
+		if layer == "" {
+			return "", fmt.Errorf("unsupported overlay layer for erofs differ: %w", errdefs.ErrNotImplemented)
+		}
+	} else {
+		return "", fmt.Errorf("invalid filesystem type for erofs differ: %w", errdefs.ErrNotImplemented)
+	}
+	// If the layer is not prepared by the EROFS snapshotter, fall back to the next differ
+	if _, err := os.Stat(filepath.Join(layer, ".erofslayer")); err != nil {
+		return "", fmt.Errorf("mount layer type must be erofs-layer: %w", errdefs.ErrNotImplemented)
+	}
+	return layer, nil
+}
+
+func (s erofsDiff) Apply(ctx context.Context, desc ocispec.Descriptor, mounts []mount.Mount, opts ...diff.ApplyOpt) (d ocispec.Descriptor, err error) {
+	t1 := time.Now()
+	defer func() {
+		if err == nil {
+			log.G(ctx).WithFields(log.Fields{
+				"d":      time.Since(t1),
+				"digest": desc.Digest,
+				"size":   desc.Size,
+				"media":  desc.MediaType,
+			}).Debugf("diff applied")
+		}
+	}()
+
+	if _, err := images.DiffCompression(ctx, desc.MediaType); err != nil {
+		return emptyDesc, fmt.Errorf("currently unsupported media type: %s", desc.MediaType)
+	}
+
+	var config diff.ApplyConfig
+	for _, o := range opts {
+		if err := o(ctx, desc, &config); err != nil {
+			return emptyDesc, fmt.Errorf("failed to apply config opt: %w", err)
+		}
+	}
+
+	layer, err := mountsToLayer(mounts)
+	if err != nil {
+		return emptyDesc, err
+	}
+
+	ra, err := s.store.ReaderAt(ctx, desc)
+	if err != nil {
+		return emptyDesc, fmt.Errorf("failed to get reader from content store: %w", err)
+	}
+	defer ra.Close()
+
+	processor := diff.NewProcessorChain(desc.MediaType, content.NewReader(ra))
+	for {
+		if processor, err = diff.GetProcessor(ctx, processor, config.ProcessorPayloads); err != nil {
+			return emptyDesc, fmt.Errorf("failed to get stream processor for %s: %w", desc.MediaType, err)
+		}
+		if processor.MediaType() == ocispec.MediaTypeImageLayer {
+			break
+		}
+	}
+	defer processor.Close()
+
+	digester := digest.Canonical.Digester()
+	rc := &readCounter{
+		r: io.TeeReader(processor, digester.Hash()),
+	}
+
+	layerBlobPath := path.Join(layer, "layer.erofs")
+	err = convertTarErofs(ctx, rc, layerBlobPath, s.mkfsExtraOpts)
+	if err != nil {
+		return emptyDesc, fmt.Errorf("failed to convert erofs: %w", err)
+	}
+
+	// Read any trailing data
+	if _, err := io.Copy(io.Discard, rc); err != nil {
+		return emptyDesc, err
+	}
+
+	return ocispec.Descriptor{
+		MediaType: ocispec.MediaTypeImageLayer,
+		Size:      rc.c,
+		Digest:    digester.Digest(),
+	}, nil
+}
+
+type readCounter struct {
+	r io.Reader
+	c int64
+}
+
+func (rc *readCounter) Read(p []byte) (n int, err error) {
+	n, err = rc.r.Read(p)
+	rc.c += int64(n)
+	return
+}

plugins/diff/erofs/plugin/plugin_linux.go

@@ -0,0 +1,63 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package plugin
+
+import (
+	"fmt"
+	"os/exec"
+
+	"github.com/containerd/containerd/v2/core/metadata"
+	"github.com/containerd/containerd/v2/plugins"
+	"github.com/containerd/containerd/v2/plugins/diff/erofs"
+	"github.com/containerd/platforms"
+	"github.com/containerd/plugin"
+	"github.com/containerd/plugin/registry"
+)
+
+// Config represents configuration for the erofs plugin.
+type Config struct {
+	// MkfsOptions are extra options used for the applier
+	MkfsOptions []string `toml:"mkfs_options"`
+}
+
+func init() {
+	registry.Register(&plugin.Registration{
+		Type: plugins.DiffPlugin,
+		ID:   "erofs",
+		Requires: []plugin.Type{
+			plugins.MetadataPlugin,
+		},
+		Config: &Config{},
+		InitFn: func(ic *plugin.InitContext) (interface{}, error) {
+			_, err := exec.LookPath("mkfs.erofs")
+			if err != nil {
+				return nil, fmt.Errorf("could not find mkfs.erofs: %v: %w", err, plugin.ErrSkipPlugin)
+			}
+
+			md, err := ic.GetSingle(plugins.MetadataPlugin)
+			if err != nil {
+				return nil, err
+			}
+
+			ic.Meta.Platforms = append(ic.Meta.Platforms, platforms.DefaultSpec())
+			cs := md.(*metadata.DB).ContentStore()
+			config := ic.Config.(*Config)
+
+			return erofs.NewErofsDiffer(cs, config.MkfsOptions), nil
+		},
+	})
+}