Merge pull request #48009 from dmcgowan/containerd-migration

Update containerd to default storage and add support for migration

.github/workflows/.test.yml

@@ -27,7 +27,7 @@ env:
   ITG_CLI_MATRIX_SIZE: 6
   DOCKER_EXPERIMENTAL: 1
   DOCKER_GRAPHDRIVER: ${{ inputs.storage == 'snapshotter' && 'overlayfs' || 'overlay2' }}
-  TEST_INTEGRATION_USE_SNAPSHOTTER: ${{ inputs.storage == 'snapshotter' && '1' || '' }}
+  TEST_INTEGRATION_USE_GRAPHDRIVER: ${{ inputs.storage == 'graphdriver' && '1' || '' }}
   SETUP_BUILDX_VERSION: edge
   SETUP_BUILDKIT_IMAGE: moby/buildkit:latest
 

.github/workflows/.windows.yml

@@ -364,10 +364,10 @@ jobs:
               "--exec-root=$env:TEMP\moby-exec", `
               "--pidfile=$env:TEMP\docker.pid", `
               "--register-service"
-          If ("${{ inputs.storage }}" -eq "snapshotter") {
+          If ("${{ inputs.storage }}" -eq "graphdriver") {
             # Make the env-var visible to the service-managed dockerd, as there's no CLI flag for this option.
-            & reg add "HKLM\SYSTEM\CurrentControlSet\Services\docker" /v Environment /t REG_MULTI_SZ /s '@' /d TEST_INTEGRATION_USE_SNAPSHOTTER=1 
-            echo "TEST_INTEGRATION_USE_SNAPSHOTTER=1" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
+            & reg add "HKLM\SYSTEM\CurrentControlSet\Services\docker" /v Environment /t REG_MULTI_SZ /s '@' /d TEST_INTEGRATION_USE_GRAPHDRIVER=1
+            echo "TEST_INTEGRATION_USE_GRAPHDRIVER=1" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
           }
           Write-Host "Starting service"
           Start-Service -Name docker

Makefile

@@ -54,7 +54,7 @@ DOCKER_ENVS := \
 	-e GITHUB_ACTIONS \
 	-e TEST_FORCE_VALIDATE \
 	-e TEST_INTEGRATION_DIR \
-	-e TEST_INTEGRATION_USE_SNAPSHOTTER \
+	-e TEST_INTEGRATION_USE_GRAPHDRIVER \
 	-e TEST_INTEGRATION_FAIL_FAST \
 	-e TEST_SKIP_INTEGRATION \
 	-e TEST_SKIP_INTEGRATION_CLI \

daemon/containerd/image_snapshot_unix.go

@@ -71,6 +71,7 @@ func (i *ImageService) copyAndUnremapRootFS(ctx context.Context, dst, src []moun
 				return fmt.Errorf("failed to copy: %w", err)
 			}
 
+			inos := make(map[uint64]struct{})
 			return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
 				if err != nil {
 					return err
@@ -80,11 +81,16 @@ func (i *ImageService) copyAndUnremapRootFS(ctx context.Context, dst, src []moun
 				if stat == nil {
 					return fmt.Errorf("cannot get underlying data for %s", path)
 				}
+				if _, ok := inos[stat.Ino]; ok {
+					// Inode already processed, skip
+					return nil
+				}
 
 				uid, gid, err := i.idMapping.ToContainer(int(stat.Uid), int(stat.Gid))
 				if err != nil {
 					return err
 				}
+				inos[stat.Ino] = struct{}{}
 
 				return chownWithCaps(path, uid, gid)
 			})

daemon/containerd/migration/migration.go

@@ -0,0 +1,317 @@
+package migration
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/core/images"
+	"github.com/containerd/containerd/v2/core/leases"
+	"github.com/containerd/containerd/v2/core/mount"
+	"github.com/containerd/containerd/v2/core/snapshots"
+	"github.com/containerd/containerd/v2/pkg/archive/compression"
+	"github.com/containerd/continuity/fs"
+	cerrdefs "github.com/containerd/errdefs"
+	"github.com/containerd/log"
+	"github.com/moby/moby/v2/daemon/internal/image"
+	"github.com/moby/moby/v2/daemon/internal/layer"
+	refstore "github.com/moby/moby/v2/daemon/internal/refstore"
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-spec/identity"
+	"github.com/opencontainers/image-spec/specs-go"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"golang.org/x/sync/errgroup"
+)
+
+type LayerMigrator struct {
+	layers  layer.Store
+	refs    refstore.Store
+	dis     image.Store
+	leases  leases.Manager
+	content content.Store
+	cis     images.Store
+}
+
+type Config struct {
+	ImageCount       int
+	LayerStore       layer.Store
+	ReferenceStore   refstore.Store
+	DockerImageStore image.Store
+	Leases           leases.Manager
+	Content          content.Store
+	ImageStore       images.Store
+}
+
+func NewLayerMigrator(config Config) *LayerMigrator {
+	return &LayerMigrator{
+		layers:  config.LayerStore,
+		refs:    config.ReferenceStore,
+		dis:     config.DockerImageStore,
+		leases:  config.Leases,
+		content: config.Content,
+		cis:     config.ImageStore,
+	}
+}
+
+// MigrateTocontainerd migrates containers from overlay2 to overlayfs or vfs to native
+func (lm *LayerMigrator) MigrateTocontainerd(ctx context.Context, snKey string, sn snapshots.Snapshotter) error {
+	if sn == nil {
+		return fmt.Errorf("no snapshotter to migrate to: %w", cerrdefs.ErrNotImplemented)
+	}
+
+	switch driver := lm.layers.DriverName(); driver {
+	case "overlay2":
+	case "vfs":
+	default:
+		return fmt.Errorf("%q not supported for migration: %w", driver, cerrdefs.ErrNotImplemented)
+	}
+
+	var (
+		// Zstd makes migration 10x faster
+		// TODO: make configurable
+		layerMediaType   = ocispec.MediaTypeImageLayerZstd
+		layerCompression = compression.Zstd
+	)
+
+	l, err := lm.leases.Create(ctx, leases.WithRandomID(), leases.WithExpiration(24*time.Hour))
+	if err != nil {
+		return err
+	}
+	defer func() {
+		lm.leases.Delete(ctx, l)
+	}()
+	ctx = leases.WithLease(ctx, l.ID)
+
+	for imgID, img := range lm.dis.Heads() {
+		diffids := img.RootFS.DiffIDs
+		if len(diffids) == 0 {
+			continue
+		}
+		var (
+			parent   string
+			manifest = ocispec.Manifest{
+				MediaType: ocispec.MediaTypeImageManifest,
+				Versioned: specs.Versioned{
+					SchemaVersion: 2,
+				},
+				Layers: make([]ocispec.Descriptor, len(diffids)),
+			}
+			ml        sync.Mutex
+			eg, egctx = errgroup.WithContext(ctx)
+		)
+		for i := range diffids {
+			chainID := identity.ChainID(diffids[:i+1])
+			l, err := lm.layers.Get(chainID)
+			if err != nil {
+				return fmt.Errorf("failed to get layer [%d] %q: %w", i, chainID, err)
+			}
+			layerIndex := i
+			eg.Go(func() error {
+				ctx := egctx
+				t1 := time.Now()
+				ts, err := l.TarStream()
+				if err != nil {
+					return err
+				}
+
+				desc := ocispec.Descriptor{
+					MediaType: layerMediaType,
+				}
+
+				cw, err := lm.content.Writer(ctx,
+					content.WithRef(fmt.Sprintf("ingest-%s", chainID)),
+					content.WithDescriptor(desc))
+				if err != nil {
+					return fmt.Errorf("failed to get content writer: %w", err)
+				}
+
+				dgstr := digest.Canonical.Digester()
+				cs, _ := compression.CompressStream(io.MultiWriter(cw, dgstr.Hash()), layerCompression)
+				_, err = io.Copy(cs, ts)
+				if err != nil {
+					return fmt.Errorf("failed to copy to compressed stream: %w", err)
+				}
+				cs.Close()
+
+				status, err := cw.Status()
+				if err != nil {
+					return err
+				}
+
+				desc.Size = status.Offset
+				desc.Digest = dgstr.Digest()
+
+				if err := cw.Commit(ctx, desc.Size, desc.Digest); err != nil && !cerrdefs.IsAlreadyExists(err) {
+					return err
+				}
+
+				log.G(ctx).WithFields(log.Fields{
+					"t":      time.Since(t1),
+					"size":   desc.Size,
+					"digest": desc.Digest,
+				}).Debug("Converted layer to content tar")
+
+				ml.Lock()
+				manifest.Layers[layerIndex] = desc
+				ml.Unlock()
+				return nil
+			})
+
+			metadata, err := l.Metadata()
+			if err != nil {
+				return err
+			}
+			src, ok := metadata["UpperDir"]
+			if !ok {
+				src, ok = metadata["SourceDir"]
+				if !ok {
+					log.G(ctx).WithField("metadata", metadata).WithField("driver", lm.layers.DriverName()).Debug("no source directory metadata")
+					return fmt.Errorf("graphdriver not supported: %w", cerrdefs.ErrNotImplemented)
+				}
+			}
+			log.G(ctx).WithField("metadata", metadata).Debugf("migrating %s from %s", chainID, src)
+
+			active := fmt.Sprintf("migration-%s", chainID)
+
+			key := chainID.String()
+
+			snapshotLabels := map[string]string{
+				"containerd.io/snapshot.ref": key,
+			}
+			mounts, err := sn.Prepare(ctx, active, parent, snapshots.WithLabels(snapshotLabels))
+			parent = key
+			if err != nil {
+				if cerrdefs.IsAlreadyExists(err) {
+					continue
+				}
+				return err
+			}
+
+			dst, err := extractSource(mounts)
+			if err != nil {
+				return err
+			}
+
+			t1 := time.Now()
+			if err := fs.CopyDir(dst, src); err != nil {
+				return err
+			}
+			log.G(ctx).WithFields(log.Fields{
+				"t":   time.Since(t1),
+				"key": key,
+			}).Debug("Copied layer to snapshot")
+
+			if err := sn.Commit(ctx, key, active); err != nil && !cerrdefs.IsAlreadyExists(err) {
+				return err
+			}
+		}
+
+		configBytes := img.RawJSON()
+		digest.FromBytes(configBytes)
+		manifest.Config = ocispec.Descriptor{
+			MediaType: ocispec.MediaTypeImageConfig,
+			Digest:    digest.FromBytes(configBytes),
+			Size:      int64(len(configBytes)),
+		}
+
+		configLabels := map[string]string{
+			fmt.Sprintf("containerd.io/gc.ref.snapshot.%s", snKey): parent,
+		}
+		if err = content.WriteBlob(ctx, lm.content, "config"+manifest.Config.Digest.String(), bytes.NewReader(configBytes), manifest.Config, content.WithLabels(configLabels)); err != nil && !cerrdefs.IsAlreadyExists(err) {
+			return err
+		}
+
+		if err := eg.Wait(); err != nil {
+			return err
+		}
+
+		manifestBytes, err := json.MarshalIndent(manifest, "", "   ")
+		if err != nil {
+			return err
+		}
+
+		manifestDesc := ocispec.Descriptor{
+			MediaType: manifest.MediaType,
+			Digest:    digest.FromBytes(manifestBytes),
+			Size:      int64(len(manifestBytes)),
+		}
+
+		manifestLabels := map[string]string{
+			"containerd.io/gc.ref.content.config": manifest.Config.Digest.String(),
+		}
+		for i := range manifest.Layers {
+			manifestLabels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = manifest.Layers[i].Digest.String()
+		}
+
+		if err = content.WriteBlob(ctx, lm.content, "manifest"+manifestDesc.Digest.String(), bytes.NewReader(manifestBytes), manifestDesc, content.WithLabels(manifestLabels)); err != nil && !cerrdefs.IsAlreadyExists(err) {
+			return err
+		}
+
+		childrenHandler := images.ChildrenHandler(lm.content)
+		childrenHandler = images.SetChildrenMappedLabels(lm.content, childrenHandler, nil)
+		if err = images.Walk(ctx, childrenHandler, manifestDesc); err != nil {
+			return err
+		}
+
+		var added bool
+		for _, named := range lm.refs.References(digest.Digest(imgID)) {
+			img := images.Image{
+				Name:   named.String(),
+				Target: manifestDesc,
+				// TODO: Any labels?
+			}
+			img, err = lm.cis.Create(ctx, img)
+			if err != nil && !cerrdefs.IsAlreadyExists(err) {
+				return err
+			} else if err != nil {
+				log.G(ctx).Infof("Tag already exists: %s", named)
+				continue
+			}
+
+			log.G(ctx).Infof("Migrated image %s to %s", img.Name, img.Target.Digest)
+			added = true
+		}
+
+		if !added {
+			img := images.Image{
+				Name:   "moby-dangling@" + manifestDesc.Digest.String(),
+				Target: manifestDesc,
+				// TODO: Any labels?
+			}
+			img, err = lm.cis.Create(ctx, img)
+			if err != nil && !cerrdefs.IsAlreadyExists(err) {
+				return err
+			} else if err == nil {
+				log.G(ctx).Infof("Migrated image %s to %s", img.Name, img.Target.Digest)
+			}
+		}
+	}
+
+	return nil
+}
+
+func extractSource(mounts []mount.Mount) (string, error) {
+	if len(mounts) != 1 {
+		return "", fmt.Errorf("cannot support snapshotters with multiple mount sources: %w", cerrdefs.ErrNotImplemented)
+	}
+	switch mounts[0].Type {
+	case "bind":
+		return mounts[0].Source, nil
+	case "overlay":
+		for _, option := range mounts[0].Options {
+			if strings.HasPrefix(option, "upperdir=") {
+				return option[9:], nil
+			}
+		}
+	default:
+		return "", fmt.Errorf("mount type %q not supported: %w", mounts[0].Type, cerrdefs.ErrNotImplemented)
+	}
+
+	return "", fmt.Errorf("mount is missing upper option: %w", cerrdefs.ErrNotImplemented)
+}

daemon/containerd/service.go

@@ -98,7 +98,15 @@ func (i *ImageService) CountImages(ctx context.Context) int {
 		return 0
 	}
 
-	return len(imgs)
+	uniqueImages := map[digest.Digest]struct{}{}
+	for _, i := range imgs {
+		dgst := i.Target().Digest
+		if _, ok := uniqueImages[dgst]; !ok {
+			uniqueImages[dgst] = struct{}{}
+		}
+	}
+
+	return len(uniqueImages)
 }
 
 // LayerStoreStatus returns the status for each layer store

daemon/daemon.go

@@ -10,6 +10,7 @@ import (
 	"crypto/sha256"
 	"encoding/binary"
 	"fmt"
+	"maps"
 	"net"
 	"net/netip"
 	"os"
@@ -28,6 +29,7 @@ import (
 	"github.com/containerd/log"
 	"github.com/distribution/reference"
 	dist "github.com/docker/distribution"
+	"github.com/docker/go-units"
 	"github.com/moby/buildkit/util/grpcerrors"
 	"github.com/moby/buildkit/util/tracing"
 	"github.com/moby/locker"
@@ -37,12 +39,27 @@ import (
 	registrytypes "github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/api/types/swarm"
 	volumetypes "github.com/moby/moby/api/types/volume"
+	"github.com/moby/sys/user"
+	"github.com/moby/sys/userns"
+	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+	"go.opentelemetry.io/otel"
+	"golang.org/x/sync/semaphore"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/backoff"
+	"google.golang.org/grpc/credentials/insecure"
+	"resenje.org/singleflight"
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+
 	"github.com/moby/moby/v2/daemon/builder"
 	executorpkg "github.com/moby/moby/v2/daemon/cluster/executor"
 	"github.com/moby/moby/v2/daemon/config"
 	"github.com/moby/moby/v2/daemon/container"
 	ctrd "github.com/moby/moby/v2/daemon/containerd"
+	"github.com/moby/moby/v2/daemon/containerd/migration"
 	"github.com/moby/moby/v2/daemon/events"
+	"github.com/moby/moby/v2/daemon/graphdriver"
 	_ "github.com/moby/moby/v2/daemon/graphdriver/register" // register graph drivers
 	"github.com/moby/moby/v2/daemon/images"
 	"github.com/moby/moby/v2/daemon/internal/distribution"
@@ -72,18 +89,6 @@ import (
 	"github.com/moby/moby/v2/pkg/authorization"
 	"github.com/moby/moby/v2/pkg/plugingetter"
 	"github.com/moby/moby/v2/pkg/sysinfo"
-	"github.com/moby/sys/user"
-	"github.com/moby/sys/userns"
-	"github.com/pkg/errors"
-	bolt "go.etcd.io/bbolt"
-	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
-	"go.opentelemetry.io/otel"
-	"golang.org/x/sync/semaphore"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/backoff"
-	"google.golang.org/grpc/credentials/insecure"
-	"resenje.org/singleflight"
-	"tags.cncf.io/container-device-interface/pkg/cdi"
 )
 
 type configStore struct {
@@ -203,15 +208,15 @@ func (daemon *Daemon) UsesSnapshotter() bool {
 	return daemon.usesSnapshotter
 }
 
-func (daemon *Daemon) restore(cfg *configStore) error {
+func (daemon *Daemon) loadContainers(ctx context.Context) (map[string]map[string]*container.Container, error) {
 	var mapLock sync.Mutex
-	containers := make(map[string]*container.Container)
+	driverContainers := make(map[string]map[string]*container.Container)
 
-	log.G(context.TODO()).Info("Loading containers: start.")
+	log.G(ctx).Info("Loading containers: start.")
 
 	dir, err := os.ReadDir(daemon.repository)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	// parallelLimit is the maximum number of parallel startup jobs that we
@@ -232,36 +237,46 @@ func (daemon *Daemon) restore(cfg *configStore) error {
 			_ = sem.Acquire(context.Background(), 1)
 			defer sem.Release(1)
 
-			logger := log.G(context.TODO()).WithField("container", id)
+			logger := log.G(ctx).WithField("container", id)
 
 			c, err := daemon.load(id)
 			if err != nil {
 				logger.WithError(err).Error("failed to load container")
 				return
 			}
-			if c.Driver != daemon.imageService.StorageDriver() {
-				// Ignore the container if it wasn't created with the current storage-driver
-				logger.Debugf("not restoring container because it was created with another storage driver (%s)", c.Driver)
-				return
-			}
-			rwlayer, err := daemon.imageService.GetLayerByID(c.ID)
-			if err != nil {
-				logger.WithError(err).Error("failed to load container mount")
-				return
-			}
-			c.RWLayer = rwlayer
-			logger.WithFields(log.Fields{
-				"running": c.IsRunning(),
-				"paused":  c.IsPaused(),
-			}).Debug("loaded container")
 
 			mapLock.Lock()
-			containers[c.ID] = c
+			if containers, ok := driverContainers[c.Driver]; !ok {
+				driverContainers[c.Driver] = map[string]*container.Container{
+					c.ID: c,
+				}
+			} else {
+				containers[c.ID] = c
+			}
 			mapLock.Unlock()
 		}(v.Name())
 	}
 	group.Wait()
 
+	return driverContainers, nil
+}
+
+func (daemon *Daemon) restore(ctx context.Context, cfg *configStore, containers map[string]*container.Container) error {
+	var mapLock sync.Mutex
+
+	log.G(ctx).Info("Restoring containers: start.")
+
+	// parallelLimit is the maximum number of parallel startup jobs that we
+	// allow (this is the limited used for all startup semaphores). The multipler
+	// (128) was chosen after some fairly significant benchmarking -- don't change
+	// it unless you've tested it significantly (this value is adjusted if
+	// RLIMIT_NOFILE is small to avoid EMFILE).
+	parallelLimit := adjustParallelLimit(len(containers), 128*runtime.NumCPU())
+
+	// Re-used for all parallel startup jobs.
+	var group sync.WaitGroup
+	sem := semaphore.NewWeighted(int64(parallelLimit))
+
 	removeContainers := make(map[string]*container.Container)
 	restartContainers := make(map[*container.Container]chan struct{})
 	activeSandboxes := make(map[string]any)
@@ -273,17 +288,28 @@ func (daemon *Daemon) restore(cfg *configStore) error {
 			_ = sem.Acquire(context.Background(), 1)
 			defer sem.Release(1)
 
-			logger := log.G(context.TODO()).WithField("container", c.ID)
+			logger := log.G(ctx).WithField("container", c.ID)
+
+			rwlayer, err := daemon.imageService.GetLayerByID(c.ID)
+			if err != nil {
+				logger.WithError(err).Error("failed to load container mount")
+				return
+			}
+			c.RWLayer = rwlayer
+			logger.WithFields(log.Fields{
+				"running": c.IsRunning(),
+				"paused":  c.IsPaused(),
+			}).Debug("loaded container")
 
 			if err := daemon.registerName(c); err != nil {
-				logger.WithError(err).Errorf("failed to register container name: %s", c.Name)
+				log.G(ctx).WithError(err).Errorf("failed to register container name: %s", c.Name)
 				mapLock.Lock()
 				delete(containers, c.ID)
 				mapLock.Unlock()
 				return
 			}
 			if err := daemon.register(context.TODO(), c); err != nil {
-				logger.WithError(err).Error("failed to register container")
+				log.G(ctx).WithError(err).Error("failed to register container")
 				mapLock.Lock()
 				delete(containers, c.ID)
 				mapLock.Unlock()
@@ -300,7 +326,7 @@ func (daemon *Daemon) restore(cfg *configStore) error {
 			_ = sem.Acquire(context.Background(), 1)
 			defer sem.Release(1)
 
-			baseLogger := log.G(context.TODO()).WithField("container", c.ID)
+			baseLogger := log.G(ctx).WithField("container", c.ID)
 
 			if c.HostConfig != nil {
 				// Migrate containers that don't have the default ("no") restart-policy set.
@@ -524,7 +550,7 @@ func (daemon *Daemon) restore(cfg *configStore) error {
 	//
 	// Note that we cannot initialize the network controller earlier, as it
 	// needs to know if there's active sandboxes (running containers).
-	if err = daemon.initNetworkController(&cfg.Config, activeSandboxes); err != nil {
+	if err := daemon.initNetworkController(&cfg.Config, activeSandboxes); err != nil {
 		return fmt.Errorf("Error initializing network controller: %v", err)
 	}
 
@@ -553,7 +579,7 @@ func (daemon *Daemon) restore(cfg *configStore) error {
 			_ = sem.Acquire(context.Background(), 1)
 
 			if err := daemon.registerLinks(c, c.HostConfig); err != nil {
-				log.G(context.TODO()).WithField("container", c.ID).WithError(err).Error("failed to register link for container")
+				log.G(ctx).WithField("container", c.ID).WithError(err).Error("failed to register link for container")
 			}
 
 			sem.Release(1)
@@ -567,7 +593,7 @@ func (daemon *Daemon) restore(cfg *configStore) error {
 		go func(c *container.Container, chNotify chan struct{}) {
 			_ = sem.Acquire(context.Background(), 1)
 
-			logger := log.G(context.TODO()).WithField("container", c.ID)
+			logger := log.G(ctx).WithField("container", c.ID)
 
 			logger.Debug("starting container")
 
@@ -607,7 +633,7 @@ func (daemon *Daemon) restore(cfg *configStore) error {
 			_ = sem.Acquire(context.Background(), 1)
 
 			if err := daemon.containerRm(&cfg.Config, cid, &backend.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err != nil {
-				log.G(context.TODO()).WithField("container", cid).WithError(err).Error("failed to remove container")
+				log.G(ctx).WithField("container", cid).WithError(err).Error("failed to remove container")
 			}
 
 			sem.Release(1)
@@ -638,7 +664,7 @@ func (daemon *Daemon) restore(cfg *configStore) error {
 			_ = sem.Acquire(context.Background(), 1)
 
 			if err := daemon.prepareMountPoints(c); err != nil {
-				log.G(context.TODO()).WithField("container", c.ID).WithError(err).Error("failed to prepare mountpoints for container")
+				log.G(ctx).WithField("container", c.ID).WithError(err).Error("failed to prepare mountpoints for container")
 			}
 
 			sem.Release(1)
@@ -647,7 +673,7 @@ func (daemon *Daemon) restore(cfg *configStore) error {
 	}
 	group.Wait()
 
-	log.G(context.TODO()).Info("Loading containers: done.")
+	log.G(ctx).Info("Loading containers: done.")
 
 	return nil
 }
@@ -833,11 +859,39 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 	}
 	d.configStore.Store(cfgStore)
 
-	// TEST_INTEGRATION_USE_SNAPSHOTTER is used for integration tests only.
-	if os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" {
-		d.usesSnapshotter = true
-	} else {
-		d.usesSnapshotter = config.Features["containerd-snapshotter"]
+	migrationThreshold := int64(-1)
+	isGraphDriver := func(driver string) (bool, error) {
+		return graphdriver.IsRegistered(driver), nil
+	}
+	if enabled, ok := config.Features["containerd-snapshotter"]; (ok && !enabled) || os.Getenv("TEST_INTEGRATION_USE_GRAPHDRIVER") != "" {
+		isGraphDriver = func(driver string) (bool, error) {
+			if driver == "" || graphdriver.IsRegistered(driver) {
+				return true, nil
+			}
+			return false, fmt.Errorf("graphdriver is explicitly enabled but %q is not registered, %v %v", driver, config.Features, os.Getenv("TEST_INTEGRATION_USE_GRAPHDRIVER"))
+		}
+	}
+	if config.Features["containerd-migration"] {
+		if ts := os.Getenv("DOCKER_MIGRATE_SNAPSHOTTER_THRESHOLD"); ts != "" {
+			v, err := units.FromHumanSize(ts)
+			if err == nil {
+				migrationThreshold = v
+			} else {
+				log.G(ctx).WithError(err).WithField("size", ts).Warn("Invalid migration threshold value, defaulting to 0")
+				migrationThreshold = 0
+			}
+
+		} else {
+			migrationThreshold = 0
+		}
+		if migrationThreshold > 0 {
+			log.G(ctx).WithField("max_size", migrationThreshold).Info("(Experimental) Migration to containerd is enabled, driver will be switched to snapshotter after migration is complete")
+		} else {
+			log.G(ctx).WithField("env", os.Environ()).Info("Migration to containerd is enabled, driver will be switched to snapshotter if there are no images or containers")
+		}
+	}
+	if config.Features["containerd-snapshotter"] {
+		log.G(ctx).Warn(`"containerd-snapshotter" is now the default and no longer needed to be set`)
 	}
 
 	// Ensure the daemon is properly shutdown if there is a failure during
@@ -1034,50 +1088,49 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 
 	d.linkIndex = newLinkIndex()
 
-	// On Windows we don't support the environment variable, or a user supplied graphdriver
+	containers, err := d.loadContainers(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	// On Windows we don't support the environment variable, or a user supplied graphdriver,
+	// but it is allowed when using snapshotters.
 	// Unix platforms however run a single graphdriver for all containers, and it can
 	// be set through an environment variable, a daemon start parameter, or chosen through
 	// initialization of the layerstore through driver priority order for example.
-	driverName := os.Getenv("DOCKER_DRIVER")
-	if isWindows && d.UsesSnapshotter() {
-		// Containerd WCOW snapshotter
-		driverName = "windows"
-	} else if isWindows {
-		// Docker WCOW graphdriver
-		driverName = "windowsfilter"
+	driverName := os.Getenv("DOCKER_GRAPHDRIVER")
+	if isWindows {
+		if driverName == "" {
+			driverName = cfgStore.GraphDriver
+		}
+		switch driverName {
+		case "windows":
+			// Docker WCOW snapshotters
+		case "windowsfilter":
+			// Docker WCOW graphdriver
+		case "":
+			// Use graph driver but enable migration
+			driverName = "windowsfilter"
+			if os.Getenv("TEST_INTEGRATION_USE_GRAPHDRIVER") == "" {
+				// Don't force migration if graph driver is explicit
+				migrationThreshold = 0
+			}
+		default:
+			log.G(ctx).Infof("Using non-default snapshotter %s", driverName)
+
+		}
 	} else if driverName != "" {
-		log.G(ctx).Infof("Setting the storage driver from the $DOCKER_DRIVER environment variable (%s)", driverName)
+		log.G(ctx).Infof("Setting the storage driver from the $DOCKER_GRAPHDRIVER environment variable (%s)", driverName)
 	} else {
 		driverName = cfgStore.GraphDriver
 	}
 
-	if d.UsesSnapshotter() {
-		if os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" {
-			log.G(ctx).Warn("Enabling containerd snapshotter through the $TEST_INTEGRATION_USE_SNAPSHOTTER environment variable. This should only be used for testing.")
-		}
-		log.G(ctx).Info("Starting daemon with containerd snapshotter integration enabled")
-
-		// FIXME(thaJeztah): implement automatic snapshotter-selection similar to graph-driver selection; see https://github.com/moby/moby/issues/44076
-		if driverName == "" {
-			driverName = defaults.DefaultSnapshotter
-		}
-
-		// Configure and validate the kernels security support. Note this is a Linux/FreeBSD
-		// operation only, so it is safe to pass *just* the runtime OS graphdriver.
-		if err := configureKernelSecuritySupport(&cfgStore.Config, driverName); err != nil {
-			return nil, err
-		}
-		d.imageService = ctrd.NewService(ctrd.ImageServiceConfig{
-			Client:          d.containerdClient,
-			Containers:      d.containers,
-			Snapshotter:     driverName,
-			RegistryHosts:   d.RegistryHosts,
-			Registry:        d.registryService,
-			EventsService:   d.EventsService,
-			IDMapping:       idMapping,
-			RefCountMounter: snapshotter.NewMounter(config.Root, driverName, idMapping),
-		})
-	} else {
+	var migrationConfig migration.Config
+	tryGraphDriver, err := isGraphDriver(driverName)
+	if err != nil {
+		return nil, err
+	}
+	if tryGraphDriver {
 		layerStore, err := layer.NewStoreFromOptions(layer.StoreOptions{
 			Root:               cfgStore.Root,
 			GraphDriver:        driverName,
@@ -1154,14 +1207,129 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 			}
 		}
 
-		// TODO: imageStore, distributionMetadataStore, and ReferenceStore are only
-		// used above to run migration. They could be initialized in ImageService
-		// if migration is called from daemon/images. layerStore might move as well.
-		d.imageService = images.NewImageService(imgSvcConfig)
+		// If no containers are present, check whether can migrate image service
+		if drv := layerStore.DriverName(); len(containers[drv]) == 0 && migrationThreshold >= 0 {
+			switch drv {
+			case "overlay2":
+				driverName = "overlayfs"
+			case "windowsfilter":
+				driverName = "windows"
+			case "vfs":
+				driverName = "native"
+			default:
+				migrationThreshold = -1
+				log.G(ctx).Infof("Not migrating to containerd snapshotter, no migration defined for graph driver %q", drv)
+			}
 
-		log.G(ctx).Debugf("Max Concurrent Downloads: %d", imgSvcConfig.MaxConcurrentDownloads)
-		log.G(ctx).Debugf("Max Concurrent Uploads: %d", imgSvcConfig.MaxConcurrentUploads)
-		log.G(ctx).Debugf("Max Download Attempts: %d", imgSvcConfig.MaxDownloadAttempts)
+			var totalSize int64
+			ic := imgSvcConfig.ImageStore.Len()
+			if migrationThreshold >= 0 && ic > 0 {
+				for _, img := range imgSvcConfig.ImageStore.Map() {
+					if layerID := img.RootFS.ChainID(); layerID != "" {
+						l, err := imgSvcConfig.LayerStore.Get(layerID)
+						if err != nil {
+							if errors.Is(err, layer.ErrLayerDoesNotExist) {
+								continue
+							}
+							return nil, err
+						}
+
+						// Just look at layer size for considering maximum size
+						totalSize += l.Size()
+						layer.ReleaseAndLog(imgSvcConfig.LayerStore, l)
+					}
+				}
+
+			}
+
+			if totalSize <= migrationThreshold {
+				log.G(ctx).WithField("total", totalSize).Infof("Enabling containerd snapshotter because migration set with no containers and %d images in graph driver", ic)
+				migrationConfig = migration.Config{
+					ImageCount:       ic,
+					LayerStore:       imgSvcConfig.LayerStore,
+					DockerImageStore: imgSvcConfig.ImageStore,
+					ReferenceStore:   imgSvcConfig.ReferenceStore,
+				}
+			} else if migrationThreshold >= 0 {
+				log.G(ctx).WithField("total", totalSize).Warnf("Not migrating to containerd snapshotter because still have %d images in graph driver", ic)
+				d.imageService = images.NewImageService(ctx, imgSvcConfig)
+			} else {
+				d.imageService = images.NewImageService(ctx, imgSvcConfig)
+			}
+		} else {
+			log.G(ctx).Debugf("Not attempting migration with %d containers and %d image threshold", len(containers[drv]), migrationThreshold)
+			d.imageService = images.NewImageService(ctx, imgSvcConfig)
+		}
+	}
+
+	if d.imageService == nil {
+		log.G(ctx).Info("Starting daemon with containerd snapshotter integration enabled")
+
+		resp, err := d.containerdClient.IntrospectionService().Plugins(ctx, `type=="io.containerd.snapshotter.v1"`)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get containerd plugins: %w", err)
+		}
+		if resp == nil || len(resp.Plugins) == 0 {
+			return nil, fmt.Errorf("failed to get containerd plugins response: %w", cerrdefs.ErrUnavailable)
+		}
+		availableDrivers := map[string]struct{}{}
+		for _, p := range resp.Plugins {
+			if p == nil || p.Type != "io.containerd.snapshotter.v1" {
+				continue
+			}
+			if p.InitErr == nil {
+				availableDrivers[p.ID] = struct{}{}
+			} else if (p.ID == driverName) || (driverName == "" && p.ID == defaults.DefaultSnapshotter) {
+				log.G(ctx).WithField("message", p.InitErr.Message).Warn("Preferred snapshotter not available in containerd")
+			}
+		}
+
+		if driverName == "" {
+			if _, ok := availableDrivers[defaults.DefaultSnapshotter]; ok {
+				driverName = defaults.DefaultSnapshotter
+			} else if _, ok := availableDrivers["native"]; ok {
+				driverName = "native"
+			} else {
+				log.G(ctx).WithField("available", maps.Keys(availableDrivers)).Debug("Preferred snapshotter not available in containerd")
+				return nil, fmt.Errorf("snapshotter selection failed, no drivers available: %w", cerrdefs.ErrUnavailable)
+			}
+		} else if _, ok := availableDrivers[driverName]; !ok {
+			return nil, fmt.Errorf("configured driver %q not available: %w", driverName, cerrdefs.ErrUnavailable)
+		}
+
+		// Configure and validate the kernels security support. Note this is a Linux/FreeBSD
+		// operation only, so it is safe to pass *just* the runtime OS graphdriver.
+		if err := configureKernelSecuritySupport(&cfgStore.Config, driverName); err != nil {
+			return nil, err
+		}
+		d.usesSnapshotter = true
+		d.imageService = ctrd.NewService(ctrd.ImageServiceConfig{
+			Client:          d.containerdClient,
+			Containers:      d.containers,
+			Snapshotter:     driverName,
+			RegistryHosts:   d.RegistryHosts,
+			Registry:        d.registryService,
+			EventsService:   d.EventsService,
+			IDMapping:       idMapping,
+			RefCountMounter: snapshotter.NewMounter(config.Root, driverName, idMapping),
+		})
+
+		if migrationConfig.ImageCount > 0 {
+			if d.imageService.CountImages(ctx) > 0 {
+				log.G(ctx).WithField("image_count", migrationConfig.ImageCount).Warnf("Images not migrated because images already exist in containerd %q", migrationConfig.LayerStore.DriverName())
+			} else {
+				migrationConfig.Leases = d.containerdClient.LeasesService()
+				migrationConfig.Content = d.containerdClient.ContentStore()
+				migrationConfig.ImageStore = d.containerdClient.ImageService()
+				m := migration.NewLayerMigrator(migrationConfig)
+				err := m.MigrateTocontainerd(ctx, driverName, d.containerdClient.SnapshotService(driverName))
+				if err != nil {
+					log.G(ctx).WithError(err).Errorf("Failed to migrate images to containerd, images in graph driver %q are no longer visible", migrationConfig.LayerStore.DriverName())
+				} else {
+					log.G(ctx).WithField("image_count", migrationConfig.ImageCount).Infof("Successfully migrated images from %q to containerd", migrationConfig.LayerStore.DriverName())
+				}
+			}
+		}
 	}
 
 	go d.execCommandGC()
@@ -1171,9 +1339,22 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		return nil, err
 	}
 
-	if err := d.restore(cfgStore); err != nil {
+	driverContainers, ok := containers[driverName]
+	// Log containers which are not loaded with current driver
+	if (!ok && len(containers) > 0) || len(containers) > 1 {
+		for driver, all := range containers {
+			if driver == driverName {
+				continue
+			}
+			for id := range all {
+				log.G(ctx).WithField("container", id).Debugf("not restoring container because it was created with another storage driver (%s)", driver)
+			}
+		}
+	}
+	if err := d.restore(ctx, cfgStore, driverContainers); err != nil {
 		return nil, err
 	}
+	// Wait for migration to complete
 	close(d.startupDone)
 
 	info, err := d.SystemInfo(ctx)

daemon/graphdriver/driver.go

@@ -121,6 +121,12 @@ func Register(name string, initFunc InitFunc) error {
 	return nil
 }
 
+// IsRegistered checks to see if the drive with the given name is registered
+func IsRegistered(name string) bool {
+	_, exists := drivers[name]
+	return exists
+}
+
 // getDriver initializes and returns the registered driver.
 func getDriver(name string, config Options) (Driver, error) {
 	if initFunc, exists := drivers[name]; exists {

daemon/graphdriver/vfs/driver.go

@@ -86,7 +86,16 @@ func (d *Driver) Status() [][2]string {
 
 // GetMetadata is used for implementing the graphdriver.ProtoDriver interface. VFS does not currently have any meta data.
 func (d *Driver) GetMetadata(id string) (map[string]string, error) {
-	return nil, nil
+	dir := d.dir(id)
+	if _, err := os.Stat(dir); err != nil {
+		return nil, err
+	}
+
+	metadata := map[string]string{
+		"SourceDir": dir,
+	}
+
+	return metadata, nil
 }
 
 // Cleanup is used to implement graphdriver.ProtoDriver. There is no cleanup required for this driver.

daemon/images/service.go

@@ -49,7 +49,11 @@ type ImageServiceConfig struct {
 }
 
 // NewImageService returns a new ImageService from a configuration
-func NewImageService(config ImageServiceConfig) *ImageService {
+func NewImageService(ctx context.Context, config ImageServiceConfig) *ImageService {
+	log.G(ctx).Debugf("Max Concurrent Downloads: %d", config.MaxConcurrentDownloads)
+	log.G(ctx).Debugf("Max Concurrent Uploads: %d", config.MaxConcurrentUploads)
+	log.G(ctx).Debugf("Max Download Attempts: %d", config.MaxDownloadAttempts)
+
 	return &ImageService{
 		containers:                config.ContainerStore,
 		distributionMetadataStore: config.DistributionMetadataStore,

daemon/reload_test.go

@@ -25,7 +25,7 @@ func muteLogs(t *testing.T) {
 func newDaemonForReloadT(t *testing.T, cfg *config.Config) *Daemon {
 	t.Helper()
 	daemon := &Daemon{
-		imageService: images.NewImageService(images.ImageServiceConfig{}),
+		imageService: images.NewImageService(context.TODO(), images.ImageServiceConfig{}),
 	}
 	var err error
 	daemon.registryService, err = registry.NewService(registry.ServiceOptions{})
@@ -63,7 +63,7 @@ func TestDaemonReloadLabels(t *testing.T) {
 
 func TestDaemonReloadMirrors(t *testing.T) {
 	daemon := &Daemon{
-		imageService: images.NewImageService(images.ImageServiceConfig{}),
+		imageService: images.NewImageService(context.TODO(), images.ImageServiceConfig{}),
 	}
 	muteLogs(t)
 
@@ -162,7 +162,7 @@ func TestDaemonReloadMirrors(t *testing.T) {
 
 func TestDaemonReloadInsecureRegistries(t *testing.T) {
 	daemon := &Daemon{
-		imageService: images.NewImageService(images.ImageServiceConfig{}),
+		imageService: images.NewImageService(context.TODO(), images.ImageServiceConfig{}),
 	}
 	muteLogs(t)
 

hack/dockerfile/etc/docker/daemon.json

@@ -3,8 +3,5 @@
 		"crun": {
 			"path": "/usr/local/bin/crun"
 		}
-	},
-	"features": {
-		"containerd-snapshotter": false
 	}
 }

hack/make/.integration-daemon-start

@@ -40,7 +40,7 @@ fi
 # intentionally open a couple bogus file descriptors to help test that they get scrubbed in containers
 exec 41>&1 42>&2
 
-export DOCKER_GRAPHDRIVER=${DOCKER_GRAPHDRIVER:-vfs}
+export DOCKER_GRAPHDRIVER=${DOCKER_GRAPHDRIVER:-native}
 export DOCKER_USERLANDPROXY=${DOCKER_USERLANDPROXY:-true}
 
 # example usage: DOCKER_STORAGE_OPTS="dm.basesize=20G,dm.loopdatasize=200G"

hack/make/.integration-test-helpers

@@ -210,7 +210,7 @@ test_env() {
 			PATH="$PATH" \
 			TEMP="$TEMP" \
 			TEST_CLIENT_BINARY="$TEST_CLIENT_BINARY" \
-			TEST_INTEGRATION_USE_SNAPSHOTTER="$TEST_INTEGRATION_USE_SNAPSHOTTER" \
+			TEST_INTEGRATION_USE_GRAPHDRIVER="$TEST_INTEGRATION_USE_GRAPHDRIVER" \
 			OTEL_EXPORTER_OTLP_ENDPOINT="$OTEL_EXPORTER_OTLP_ENDPOINT" \
 			OTEL_SERVICE_NAME="$OTEL_SERVICE_NAME" \
 			"$@"

hack/make/run

@@ -10,7 +10,7 @@ fi
 
 DOCKER_COMMAND="$(command -v dockerd)"
 
-DOCKER_GRAPHDRIVER=${DOCKER_GRAPHDRIVER:-vfs}
+DOCKER_GRAPHDRIVER=${DOCKER_GRAPHDRIVER:-native}
 DOCKER_USERLANDPROXY=${DOCKER_USERLANDPROXY:-true}
 
 # example usage: DOCKER_STORAGE_OPTS="dm.basesize=20G,dm.loopdatasize=200G"

hack/make/test-docker-py

@@ -15,7 +15,7 @@ source hack/make/.integration-test-helpers
 : "${PY_TEST_OPTIONS:=--junitxml=${DEST}/junit-report.xml}"
 
 # build --squash is not supported with containerd integration.
-if [ -n "$TEST_INTEGRATION_USE_SNAPSHOTTER" ]; then
+if [ -z "$TEST_INTEGRATION_USE_GRAPHDRIVER" ]; then
 	PY_TEST_OPTIONS="$PY_TEST_OPTIONS --deselect=tests/integration/api_build_test.py::BuildTest::test_build_squash"
 fi
 

integration/daemon/migration_test.go

@@ -0,0 +1,163 @@
+package daemon // import "github.com/docker/docker/integration/daemon"
+
+import (
+	"bytes"
+	"io"
+	"runtime"
+	"testing"
+
+	containertypes "github.com/moby/moby/api/types/container"
+	"github.com/moby/moby/api/types/image"
+	"github.com/moby/moby/client"
+	"github.com/moby/moby/v2/integration/internal/container"
+	"github.com/moby/moby/v2/testutil"
+	"github.com/moby/moby/v2/testutil/daemon"
+	"github.com/moby/moby/v2/testutil/fixtures/load"
+	"gotest.tools/v3/assert"
+	"gotest.tools/v3/skip"
+)
+
+func TestMigrateOverlaySnapshotter(t *testing.T) {
+	testMigrateSnapshotter(t, "overlay2", "overlayfs")
+}
+
+func TestMigrateNativeSnapshotter(t *testing.T) {
+	testMigrateSnapshotter(t, "vfs", "native")
+}
+
+func testMigrateSnapshotter(t *testing.T, graphdriver, snapshotter string) {
+	skip.If(t, runtime.GOOS != "linux")
+
+	t.Setenv("DOCKER_MIGRATE_SNAPSHOTTER_THRESHOLD", "200M")
+	t.Setenv("DOCKER_GRAPHDRIVER", "")
+
+	ctx := testutil.StartSpan(baseContext, t)
+
+	d := daemon.New(t)
+	defer d.Stop(t)
+
+	d.Start(t, "--iptables=false", "--ip6tables=false", "-s", graphdriver)
+	info := d.Info(t)
+	id := info.ID
+	assert.Check(t, id != "")
+	assert.Equal(t, info.Containers, 0)
+	assert.Equal(t, info.Images, 0)
+	assert.Equal(t, info.Driver, graphdriver)
+
+	load.FrozenImagesLinux(ctx, d.NewClientT(t), "busybox:latest")
+
+	info = d.Info(t)
+	allImages := info.Images
+	assert.Check(t, allImages > 0)
+
+	apiClient := d.NewClientT(t)
+
+	containerID := container.Run(ctx, t, apiClient, func(c *container.TestContainerConfig) {
+		c.Name = "Migration-1-" + snapshotter
+		c.Config.Image = "busybox:latest"
+		c.Config.Cmd = []string{"top"}
+	})
+
+	d.Stop(t)
+
+	// Start with migration feature but with a container which will prevent migration
+	d.Start(t, "--iptables=false", "--ip6tables=false", "-s", graphdriver, "--feature", "containerd-migration")
+	info = d.Info(t)
+	assert.Equal(t, info.ID, id)
+	assert.Equal(t, info.Driver, graphdriver)
+	assert.Equal(t, info.Containers, 1)
+	assert.Equal(t, info.Images, allImages)
+	container.Remove(ctx, t, apiClient, containerID, containertypes.RemoveOptions{
+		Force: true,
+	})
+
+	d.Stop(t)
+
+	d.Start(t, "--iptables=false", "--ip6tables=false", "-s", graphdriver, "--feature", "containerd-migration")
+	info = d.Info(t)
+	assert.Equal(t, info.ID, id)
+	assert.Equal(t, info.Containers, 0)
+	assert.Equal(t, info.Driver, snapshotter, "expected migrate to switch from %s to %s", graphdriver, snapshotter)
+	assert.Equal(t, info.Images, allImages)
+
+	result := container.RunAttach(ctx, t, apiClient, func(c *container.TestContainerConfig) {
+		c.Name = "Migration-2-" + snapshotter
+		c.Config.Image = "busybox:latest"
+		c.Config.Cmd = []string{"echo", "hello"}
+	})
+	assert.Equal(t, result.ExitCode, 0)
+	container.Remove(ctx, t, apiClient, result.ContainerID, containertypes.RemoveOptions{})
+}
+
+func TestMigrateSaveLoad(t *testing.T) {
+	skip.If(t, runtime.GOOS != "linux")
+
+	t.Setenv("DOCKER_MIGRATE_SNAPSHOTTER_THRESHOLD", "200M")
+	t.Setenv("DOCKER_GRAPHDRIVER", "")
+
+	var (
+		ctx         = testutil.StartSpan(baseContext, t)
+		d           = daemon.New(t)
+		graphdriver = "overlay2"
+		snapshotter = "overlayfs"
+	)
+	defer d.Stop(t)
+
+	d.Start(t, "--iptables=false", "--ip6tables=false", "-s", graphdriver)
+	info := d.Info(t)
+	id := info.ID
+	assert.Check(t, id != "")
+	assert.Equal(t, info.Containers, 0)
+	assert.Equal(t, info.Images, 0)
+	assert.Equal(t, info.Driver, graphdriver)
+
+	load.FrozenImagesLinux(ctx, d.NewClientT(t), "busybox:latest")
+
+	info = d.Info(t)
+	allImages := info.Images
+	assert.Check(t, allImages > 0)
+
+	d.Stop(t)
+
+	d.Start(t, "--iptables=false", "--ip6tables=false", "-s", graphdriver, "--feature", "containerd-migration")
+	info = d.Info(t)
+	assert.Equal(t, info.ID, id)
+	assert.Equal(t, info.Containers, 0)
+	assert.Equal(t, info.Driver, snapshotter, "expected migrate to switch from %s to %s", graphdriver, snapshotter)
+	assert.Equal(t, info.Images, allImages)
+
+	apiClient := d.NewClientT(t)
+
+	// Save image to buffer
+	rdr, err := apiClient.ImageSave(ctx, []string{"busybox:latest"})
+	assert.NilError(t, err)
+	buf := bytes.NewBuffer(nil)
+	io.Copy(buf, rdr)
+	rdr.Close()
+
+	// Delete all images
+	list, err := apiClient.ImageList(ctx, image.ListOptions{})
+	assert.NilError(t, err)
+	for _, i := range list {
+		_, err = apiClient.ImageRemove(ctx, i.ID, image.RemoveOptions{Force: true})
+		assert.NilError(t, err)
+	}
+
+	// Check zero images
+	info = d.Info(t)
+	assert.Equal(t, info.Images, 0)
+
+	// Import
+	lr, err := apiClient.ImageLoad(ctx, bytes.NewReader(buf.Bytes()), client.ImageLoadWithQuiet(true))
+	assert.NilError(t, err)
+	io.Copy(io.Discard, lr.Body)
+	lr.Body.Close()
+
+	result := container.RunAttach(ctx, t, apiClient, func(c *container.TestContainerConfig) {
+		c.Name = "Migration-save-load-" + snapshotter
+		c.Config.Image = "busybox:latest"
+		c.Config.Cmd = []string{"echo", "hello"}
+	})
+	assert.Equal(t, result.ExitCode, 0)
+	container.Remove(ctx, t, apiClient, result.ContainerID, containertypes.RemoveOptions{})
+}

integration/volume/mount_test.go

@@ -147,7 +147,7 @@ func TestRunMountImage(t *testing.T) {
 		{name: "image_remove_force", cmd: []string{"cat", "/image/foo"}, expected: "barbar"},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
-			testImage := setupTestImage(t, ctx, apiClient, tc.name, testEnv.UsingSnapshotter())
+			testImage := setupTestImage(t, ctx, apiClient, tc.name)
 			if testImage != "" {
 				defer apiClient.ImageRemove(ctx, testImage, image.RemoveOptions{Force: true})
 			}
@@ -292,26 +292,19 @@ func setupTestVolume(t *testing.T, apiClient client.APIClient) string {
 	return volumeName
 }
 
-func setupTestImage(t *testing.T, ctx context.Context, apiClient client.APIClient, test string, snapshotter bool) string {
+func setupTestImage(t *testing.T, ctx context.Context, apiClient client.APIClient, test string) string {
 	imgName := "test-image"
 
 	if test == "image_tag" {
 		imgName += ":foo"
 	}
 
-	var symlink string
-	if snapshotter {
-		symlink = "../../../../rootfs"
-	} else {
-		symlink = "../../../../../docker"
-	}
-
 	//nolint:dupword // ignore "Duplicate words (subdir) found (dupword)"
 	dockerfile := `
 		FROM busybox as symlink
 		RUN mkdir /hack \
 			&& ln -s "../subdir" /hack/good \
-			&& ln -s "` + symlink + `" /hack/bad
+			&& ln -s "../../../../../docker" /hack/bad
 		#--
 		FROM scratch
 		COPY foo /

testutil/environment/environment.go

@@ -189,10 +189,9 @@ func (e *Execution) IsUserNamespaceInKernel() bool {
 }
 
 // UsingSnapshotter returns whether containerd snapshotters are used for the
-// tests by checking if the "TEST_INTEGRATION_USE_SNAPSHOTTER" is set to a
-// non-empty value.
+// tests by checking if the "TEST_INTEGRATION_USE_GRAPHDRIVER" is empty
 func (e *Execution) UsingSnapshotter() bool {
-	return os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != ""
+	return os.Getenv("TEST_INTEGRATION_USE_GRAPHDRIVER") == ""
 }
 
 // HasExistingImage checks whether there is an image with the given reference.