Merge pull request #53946 from vvoland/c8d-dropins

daemon/containerd: Support configuration drop-ins

daemon/command/daemon.go

@@ -1167,6 +1167,40 @@ func nopWaitFunc(time.Duration) error {
 	return nil
 }
 
+// containerdDropInDir is the directory, relative to the directory of the
+// daemon configuration file, with configuration files for embedded containerd.
+const containerdDropInDir = "containerd.d"
+
+// containerdConfigDropInDir returns the absolute path of the directory with
+// configuration files for embedded containerd, or an empty string if the
+// daemon has no configuration file.
+func (cli *daemonCLI) containerdConfigDropInDir() (string, error) {
+	if *cli.configFile == "" {
+		return "", nil
+	}
+	configDir, err := filepath.Abs(filepath.Dir(*cli.configFile))
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(configDir, containerdDropInDir), nil
+}
+
+// warnUnusedContainerdDropIns warns if containerd configuration drop-ins are
+// present while dockerd does not use embedded containerd, so they are ignored.
+func (cli *daemonCLI) warnUnusedContainerdDropIns(ctx context.Context) {
+	dir, err := cli.containerdConfigDropInDir()
+	if err != nil || dir == "" {
+		return
+	}
+	entries, _ := os.ReadDir(dir)
+	if !slices.ContainsFunc(entries, func(entry os.DirEntry) bool {
+		return !entry.IsDir() && strings.HasSuffix(entry.Name(), ".toml")
+	}) {
+		return
+	}
+	log.G(ctx).WithField("dir", dir).Warn("ignoring containerd config drop-ins; they are only supported with embedded containerd")
+}
+
 func (cli *daemonCLI) initializeContainerd(ctx context.Context) (func(time.Duration) error, error) {
 	systemContainerdAddr, ok, err := systemContainerdRunning(honorXDG)
 	if err != nil {
@@ -1175,6 +1209,7 @@ func (cli *daemonCLI) initializeContainerd(ctx context.Context) (func(time.Durat
 	if ok {
 		// Use the externally managed system containerd detected at this address.
 		cli.Config.ContainerdAddr = systemContainerdAddr
+		cli.warnUnusedContainerdDropIns(ctx)
 		return nopWaitFunc, nil
 	}
 
@@ -1192,6 +1227,7 @@ func (cli *daemonCLI) initializeContainerd(ctx context.Context) (func(time.Durat
 		return nil, errors.Wrap(err, "failed to start containerd")
 	}
 	cli.Config.ContainerdAddr = r.Address()
+	cli.warnUnusedContainerdDropIns(ctx)
 
 	// Try to wait for containerd to shutdown
 	return r.WaitTimeout, nil

daemon/command/daemon_embedded_containerd.go

@@ -21,10 +21,15 @@ func (cli *daemonCLI) initEmbeddedContainerd(ctx context.Context) (func(time.Dur
 		return nil, err
 	}
 	log.G(ctx).Warn("Running with experimental embedded-containerd mode")
+	dropInDir, err := cli.containerdConfigDropInDir()
+	if err != nil {
+		return nil, err
+	}
 	d, err := embedded.Start(
 		ctx,
 		rootDir,
 		filepath.Join(cli.Config.ExecRoot, "containerd"),
+		dropInDir,
 	)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to start embedded containerd")

daemon/command/daemon_unix.go

@@ -137,6 +137,7 @@ func (cli *daemonCLI) initContainerd(ctx context.Context) (func(time.Duration) e
 	}
 	if cli.Config.ContainerdAddr != "" {
 		// use system containerd at the given address.
+		cli.warnUnusedContainerdDropIns(ctx)
 		return nopWaitFunc, nil
 	}
 

daemon/command/daemon_windows.go

@@ -135,11 +135,13 @@ func (cli *daemonCLI) initContainerd(ctx context.Context) (func(time.Duration) e
 	}
 	if cli.Config.ContainerdAddr != "" {
 		// use system containerd at the given address.
+		cli.warnUnusedContainerdDropIns(ctx)
 		return nopWaitFunc, nil
 	}
 
 	if cli.Config.DefaultRuntime == "" || cli.Config.DefaultRuntime == config.WindowsV1RuntimeName {
 		// Legacy non-containerd runtime is used
+		cli.warnUnusedContainerdDropIns(ctx)
 		return nopWaitFunc, nil
 	}
 

daemon/internal/containerd/server/embedded/containerd.go

@@ -7,12 +7,16 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"maps"
 	"net"
 	"os"
 	"path/filepath"
+	"slices"
+	"strings"
 	"sync"
 	"sync/atomic"
 
+	srvconfig "github.com/containerd/containerd/v2/cmd/containerd/server/config"
 	"github.com/containerd/containerd/v2/defaults"
 	"github.com/containerd/containerd/v2/pkg/namespaces"
 	"github.com/containerd/containerd/v2/pkg/sys"
@@ -40,6 +44,10 @@ type serverConfig struct {
 	ttrpcAddress       string
 	maxRecvMessageSize int
 	maxSendMessageSize int
+
+	// containerd is the configuration loaded from drop-in files.
+	// Only its plugins, disabled_plugins, and required_plugins settings apply.
+	containerd srvconfig.Config
 }
 
 type containerdServer struct {
@@ -57,7 +65,7 @@ type containerdServer struct {
 // Command-owned behavior such as process mutation, proxy plugins, TCP/TLS, and
 // Prometheus registration is intentionally not part of the embedded server.
 func newServer(ctx context.Context, cfg *serverConfig) (*containerdServer, error) {
-	registrations := registry.Graph(nil)
+	registrations := registry.Graph(srvconfig.V2DisabledFilter(cfg.containerd.DisabledPlugins))
 	return newServerWithRegistrations(ctx, cfg, registrations)
 }
 
@@ -100,6 +108,10 @@ func newServerWithRegistrations(ctx context.Context, cfg *serverConfig, registra
 	}()
 
 	initialized := plugin.NewPluginSet()
+	required := make(map[string]struct{}, len(cfg.containerd.RequiredPlugins))
+	for _, id := range cfg.containerd.RequiredPlugins {
+		required[id] = struct{}{}
+	}
 
 	var grpcServices []grpcService
 	var ttrpcServices []ttrpcService
@@ -114,6 +126,14 @@ func newServerWithRegistrations(ctx context.Context, cfg *serverConfig, registra
 			plugins.PropertyTTRPCAddress: cfg.ttrpcAddress,
 		})
 
+		if registration.Config != nil {
+			pluginConfig, err := cfg.containerd.Decode(ctx, id, registration.Config)
+			if err != nil {
+				return nil, fmt.Errorf("decoding config for plugin %q: %w", id, err)
+			}
+			registration.Config = pluginConfig
+		}
+
 		// Disable cgroups Prometheus metrics for the embedded server to
 		// avoid exposing them through dockerd's process-global Prometheus
 		// registry.
@@ -137,6 +157,9 @@ func newServerWithRegistrations(ctx context.Context, cfg *serverConfig, registra
 			if mustSucceed.Load() {
 				return nil, fmt.Errorf("plugin %q failed after registering readiness: %w", id, err)
 			}
+			if _, ok := required[id]; ok {
+				return nil, fmt.Errorf("loading required plugin %q: %w", id, err)
+			}
 			fields := log.Fields{"error": err, "id": id, "type": registration.Type}
 			if plugin.IsSkipPlugin(err) {
 				log.G(ctx).WithFields(fields).Info("skip loading plugin")
@@ -145,6 +168,7 @@ func newServerWithRegistrations(ctx context.Context, cfg *serverConfig, registra
 			}
 			continue
 		}
+		delete(required, id)
 
 		if service, ok := instance.(grpcService); ok {
 			grpcServices = append(grpcServices, service)
@@ -154,6 +178,9 @@ func newServerWithRegistrations(ctx context.Context, cfg *serverConfig, registra
 		}
 		srv.plugins = append(srv.plugins, result)
 	}
+	if len(required) != 0 {
+		return nil, fmt.Errorf("required plugins not loaded: %s", strings.Join(slices.Sorted(maps.Keys(required)), ", "))
+	}
 
 	for _, service := range grpcServices {
 		if err := service.Register(srv.grpcServer); err != nil {

daemon/internal/containerd/server/embedded/dropin.go

@@ -0,0 +1,105 @@
+//go:build (linux || windows) && !no_embedded_containerd
+
+package embedded
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"iter"
+	"maps"
+	"os"
+	"path/filepath"
+	"reflect"
+	"slices"
+	"strings"
+
+	srvconfig "github.com/containerd/containerd/v2/cmd/containerd/server/config"
+	"github.com/containerd/containerd/v2/plugins"
+	"github.com/containerd/containerd/v2/version"
+	"github.com/containerd/log"
+	"github.com/containerd/plugin"
+	"github.com/containerd/plugin/registry"
+	"github.com/pelletier/go-toml/v2"
+)
+
+// loadConfigDropIns loads the containerd configuration drop-in files in dir.
+func loadConfigDropIns(ctx context.Context, dir string) (srvconfig.Config, error) {
+	files, err := configDropInFiles(dir)
+	if err != nil {
+		return srvconfig.Config{}, err
+	}
+
+	registrations := func() iter.Seq[plugin.Registration] {
+		return slices.Values(registry.Graph(nil))
+	}
+	cfg := srvconfig.Config{Version: version.ConfigVersion}
+	for _, path := range files {
+		if err := srvconfig.LoadConfigWithPlugins(ctx, path, registrations, &cfg); err != nil {
+			return srvconfig.Config{}, fmt.Errorf("failed to load containerd config drop-in %q: %w", path, err)
+		}
+	}
+
+	// dockerd configures the embedded server itself, and does not implement
+	// containerd's process and server settings.
+	ignored := cfg
+	ignored.Version = 0
+	ignored.Plugins = nil
+	ignored.DisabledPlugins = nil
+	ignored.RequiredPlugins = nil
+	// gRPC, TTRPC, debug, and metrics settings are server plugins, which the
+	// embedded server does not register.
+	serverSettings := slices.ContainsFunc(slices.Collect(maps.Keys(cfg.Plugins)), func(id string) bool {
+		return strings.HasPrefix(id, string(plugins.ServerPlugin)+".")
+	})
+	if serverSettings || !reflect.ValueOf(ignored).IsZero() {
+		log.G(ctx).WithField("dir", dir).Warn("embedded containerd ignores server and process settings in config drop-ins; only plugin, disabled_plugins, and required_plugins settings apply")
+	}
+	return cfg, nil
+}
+
+// configDropInFiles returns the containerd configuration files (*.toml) in
+// dir, in lexical order.
+// A missing directory has no files.
+// Files with a configuration version older than 4 or non-empty imports are
+// rejected.
+func configDropInFiles(dir string) ([]string, error) {
+	entries, err := os.ReadDir(dir)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to find containerd config drop-ins: %w", err)
+	}
+	var files []string
+	for _, entry := range entries {
+		if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".toml") {
+			continue
+		}
+		path := filepath.Join(dir, entry.Name())
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read containerd config drop-in %q: %w", path, err)
+		}
+		var dropIn struct {
+			Version int      `toml:"version"`
+			Imports []string `toml:"imports"`
+		}
+		if err := toml.Unmarshal(data, &dropIn); err != nil {
+			return nil, fmt.Errorf("failed to parse containerd config drop-in %q: %w", path, err)
+		}
+		// Drop-ins have no legacy files to support, so require the configuration
+		// version introduced in containerd 2.3 instead of relying on migrations
+		// of older versions, including a missing version being treated as 1.
+		if dropIn.Version < 4 {
+			return nil, fmt.Errorf("containerd config drop-in %q must set version 4 or later", path)
+		}
+		// Keep the merge order defined only by the lexical order of the files
+		// in the directory.
+		if len(dropIn.Imports) != 0 {
+			return nil, fmt.Errorf("containerd config drop-in %q must not contain imports", path)
+		}
+		files = append(files, path)
+	}
+	return files, nil
+}

daemon/internal/containerd/server/embedded/dropin_test.go

@@ -0,0 +1,109 @@
+//go:build (linux || windows) && !no_embedded_containerd
+
+package embedded
+
+import (
+	"os"
+	"path/filepath"
+	"strconv"
+	"testing"
+
+	srvconfig "github.com/containerd/containerd/v2/cmd/containerd/server/config"
+	"github.com/containerd/plugin"
+	"gotest.tools/v3/assert"
+	is "gotest.tools/v3/assert/cmp"
+)
+
+func TestConfigDropInsConfigurePlugins(t *testing.T) {
+	t.Parallel()
+
+	// Brackets must not be interpreted as a glob pattern.
+	dir := filepath.Join(t.TempDir(), "docker[prod]")
+	err := os.Mkdir(dir, 0o755)
+	assert.NilError(t, err)
+	err = os.WriteFile(filepath.Join(dir, "10-custom.toml"), []byte(`
+version = 4
+
+[plugins."io.moby.test.v1.configured"]
+  value = "from-drop-in"
+`), 0o644)
+	assert.NilError(t, err)
+
+	containerdConfig, err := loadConfigDropIns(t.Context(), dir)
+	assert.NilError(t, err)
+
+	type testConfig struct {
+		Value string `toml:"value"`
+	}
+	var got any
+	cfg := &serverConfig{
+		root:       filepath.Join(t.TempDir(), "root"),
+		state:      filepath.Join(t.TempDir(), "state"),
+		containerd: containerdConfig,
+	}
+	server, err := newServerWithRegistrations(t.Context(), cfg, []plugin.Registration{{
+		Type:   "io.moby.test.v1",
+		ID:     "configured",
+		Config: &testConfig{Value: "default"},
+		InitFn: func(initContext *plugin.InitContext) (any, error) {
+			got = initContext.Config
+			return struct{}{}, nil
+		},
+	}})
+	assert.NilError(t, err)
+	server.Stop()
+	assert.Check(t, is.DeepEqual(got, &testConfig{Value: "from-drop-in"}))
+}
+
+func TestConfigDropInsRequireMissingPlugin(t *testing.T) {
+	t.Parallel()
+
+	cfg := &serverConfig{
+		root:  filepath.Join(t.TempDir(), "root"),
+		state: filepath.Join(t.TempDir(), "state"),
+		containerd: srvconfig.Config{
+			RequiredPlugins: []string{"io.moby.test.v1.missing"},
+		},
+	}
+	server, err := newServerWithRegistrations(t.Context(), cfg, nil)
+	assert.Check(t, is.Nil(server))
+	assert.Check(t, is.ErrorContains(err, "required plugins not loaded: io.moby.test.v1.missing"))
+}
+
+func TestConfigDropInsRejectInvalidFiles(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name    string
+		content string
+		expErr  string
+	}{
+		{
+			name: "imports",
+			content: `
+version = 4
+imports = ["nested.conf"]
+`,
+			expErr: "must not contain imports",
+		},
+		{
+			name:    "legacy version",
+			content: `version = 3`,
+			expErr:  "must set version 4 or later",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			dir := t.TempDir()
+			dropInFile := filepath.Join(dir, "10-custom.toml")
+			err := os.WriteFile(dropInFile, []byte(tc.content), 0o644)
+			assert.NilError(t, err)
+
+			_, err = loadConfigDropIns(t.Context(), dir)
+			assert.ErrorContains(t, err, tc.expErr)
+			assert.ErrorContains(t, err, strconv.Quote(dropInFile))
+		})
+	}
+}

daemon/internal/containerd/server/embedded/embedded_test.go

@@ -21,7 +21,7 @@ func TestPluginGraphResolves(t *testing.T) {
 	ctx, cancel := context.WithCancel(t.Context())
 	defer cancel()
 
-	d, err := Start(ctx, t.TempDir(), t.TempDir())
+	d, err := Start(ctx, t.TempDir(), t.TempDir(), "")
 	if err != nil {
 		if strings.Contains(err.Error(), "not found") || strings.Contains(err.Error(), "no plugins registered") {
 			t.Fatalf("embedded containerd plugin graph is incomplete: %v", err)

daemon/internal/containerd/server/embedded/server.go

@@ -12,6 +12,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	srvconfig "github.com/containerd/containerd/v2/cmd/containerd/server/config"
 	"github.com/containerd/containerd/v2/defaults"
 	"github.com/containerd/containerd/v2/version"
 	"github.com/containerd/log"
@@ -50,12 +51,24 @@ import (
 // Start initializes and runs the embedded containerd server, using rootDir for
 // persistent state (bolt DB, content store) and the daemon subdirectory under
 // stateDir for runtime state.
+// If configDropInDir is not empty, the plugins, disabled_plugins, and
+// required_plugins settings of the containerd configuration files (*.toml)
+// in that directory apply to the server, excluding server plugin settings.
 // See the package doc for the transport layout.
 // Plugins self-register via the blank imports above and in the platform-specific
 // files.
-func Start(ctx context.Context, rootDir, stateDir string) (*Daemon, error) {
+func Start(ctx context.Context, rootDir, stateDir, configDropInDir string) (*Daemon, error) {
 	setContainerdVersion()
 
+	var containerdConfig srvconfig.Config
+	if configDropInDir != "" {
+		var err error
+		containerdConfig, err = loadConfigDropIns(ctx, configDropInDir)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	address := defaultAddress(stateDir)
 	cfg := &serverConfig{
 		root:               rootDir,
@@ -64,6 +77,7 @@ func Start(ctx context.Context, rootDir, stateDir string) (*Daemon, error) {
 		ttrpcAddress:       address + ".ttrpc",
 		maxRecvMessageSize: defaults.DefaultMaxRecvMsgSize,
 		maxSendMessageSize: defaults.DefaultMaxSendMsgSize,
+		containerd:         containerdConfig,
 	}
 
 	// Create the root and state directories with the permissions containerd

man/dockerd.8.md

@@ -201,7 +201,18 @@ Bridge networks will accept packets with this firewall mark/mask.
   */etc/docker/daemon.json*.
 
 **--containerd**=""
-  Path to containerd socket.
+  Path to containerd socket. If not set, and no system containerd is running,
+  the daemon starts and manages its own containerd.
+  With the *embedded-containerd* feature, the daemon loads containerd
+  configuration files (*\*.toml*) from the *containerd.d* directory next to
+  the daemon configuration file (by default */etc/docker/containerd.d*).
+  Drop-in files must set *version* 4 or later, and must not contain non-empty
+  *imports* lists.
+  Only the *plugins*, *disabled_plugins*, and *required_plugins* settings of
+  these files apply, excluding server settings such as gRPC, TTRPC, debug, and
+  metrics.
+  The daemon ignores these files, and logs a warning, when it does not use
+  embedded containerd.
 
 **--data-root**=""
   Path to the directory used to store persisted Docker data such as