diff --git a/daemon/command/daemon.go b/daemon/command/daemon.go index b26bf371b3..0911096972 100644 --- a/daemon/command/daemon.go +++ b/daemon/command/daemon.go @@ -1167,6 +1167,40 @@ func nopWaitFunc(time.Duration) error { return nil } +// containerdDropInDir is the directory, relative to the directory of the +// daemon configuration file, with configuration files for embedded containerd. +const containerdDropInDir = "containerd.d" + +// containerdConfigDropInDir returns the absolute path of the directory with +// configuration files for embedded containerd, or an empty string if the +// daemon has no configuration file. +func (cli *daemonCLI) containerdConfigDropInDir() (string, error) { + if *cli.configFile == "" { + return "", nil + } + configDir, err := filepath.Abs(filepath.Dir(*cli.configFile)) + if err != nil { + return "", err + } + return filepath.Join(configDir, containerdDropInDir), nil +} + +// warnUnusedContainerdDropIns warns if containerd configuration drop-ins are +// present while dockerd does not use embedded containerd, so they are ignored. +func (cli *daemonCLI) warnUnusedContainerdDropIns(ctx context.Context) { + dir, err := cli.containerdConfigDropInDir() + if err != nil || dir == "" { + return + } + entries, _ := os.ReadDir(dir) + if !slices.ContainsFunc(entries, func(entry os.DirEntry) bool { + return !entry.IsDir() && strings.HasSuffix(entry.Name(), ".toml") + }) { + return + } + log.G(ctx).WithField("dir", dir).Warn("ignoring containerd config drop-ins; they are only supported with embedded containerd") +} + func (cli *daemonCLI) initializeContainerd(ctx context.Context) (func(time.Duration) error, error) { systemContainerdAddr, ok, err := systemContainerdRunning(honorXDG) if err != nil { @@ -1175,6 +1209,7 @@ func (cli *daemonCLI) initializeContainerd(ctx context.Context) (func(time.Durat if ok { // Use the externally managed system containerd detected at this address. cli.Config.ContainerdAddr = systemContainerdAddr + cli.warnUnusedContainerdDropIns(ctx) return nopWaitFunc, nil } @@ -1192,6 +1227,7 @@ func (cli *daemonCLI) initializeContainerd(ctx context.Context) (func(time.Durat return nil, errors.Wrap(err, "failed to start containerd") } cli.Config.ContainerdAddr = r.Address() + cli.warnUnusedContainerdDropIns(ctx) // Try to wait for containerd to shutdown return r.WaitTimeout, nil diff --git a/daemon/command/daemon_embedded_containerd.go b/daemon/command/daemon_embedded_containerd.go index cea9a97bf7..7d4563cecb 100644 --- a/daemon/command/daemon_embedded_containerd.go +++ b/daemon/command/daemon_embedded_containerd.go @@ -21,10 +21,15 @@ func (cli *daemonCLI) initEmbeddedContainerd(ctx context.Context) (func(time.Dur return nil, err } log.G(ctx).Warn("Running with experimental embedded-containerd mode") + dropInDir, err := cli.containerdConfigDropInDir() + if err != nil { + return nil, err + } d, err := embedded.Start( ctx, rootDir, filepath.Join(cli.Config.ExecRoot, "containerd"), + dropInDir, ) if err != nil { return nil, errors.Wrap(err, "failed to start embedded containerd") diff --git a/daemon/command/daemon_unix.go b/daemon/command/daemon_unix.go index 9d369162df..94322f6767 100644 --- a/daemon/command/daemon_unix.go +++ b/daemon/command/daemon_unix.go @@ -137,6 +137,7 @@ func (cli *daemonCLI) initContainerd(ctx context.Context) (func(time.Duration) e } if cli.Config.ContainerdAddr != "" { // use system containerd at the given address. + cli.warnUnusedContainerdDropIns(ctx) return nopWaitFunc, nil } diff --git a/daemon/command/daemon_windows.go b/daemon/command/daemon_windows.go index bbb35292cb..9ccd8ba464 100644 --- a/daemon/command/daemon_windows.go +++ b/daemon/command/daemon_windows.go @@ -135,11 +135,13 @@ func (cli *daemonCLI) initContainerd(ctx context.Context) (func(time.Duration) e } if cli.Config.ContainerdAddr != "" { // use system containerd at the given address. + cli.warnUnusedContainerdDropIns(ctx) return nopWaitFunc, nil } if cli.Config.DefaultRuntime == "" || cli.Config.DefaultRuntime == config.WindowsV1RuntimeName { // Legacy non-containerd runtime is used + cli.warnUnusedContainerdDropIns(ctx) return nopWaitFunc, nil } diff --git a/daemon/internal/containerd/server/embedded/containerd.go b/daemon/internal/containerd/server/embedded/containerd.go index 25f838022a..d9fc24cca5 100644 --- a/daemon/internal/containerd/server/embedded/containerd.go +++ b/daemon/internal/containerd/server/embedded/containerd.go @@ -7,12 +7,16 @@ import ( "errors" "fmt" "io" + "maps" "net" "os" "path/filepath" + "slices" + "strings" "sync" "sync/atomic" + srvconfig "github.com/containerd/containerd/v2/cmd/containerd/server/config" "github.com/containerd/containerd/v2/defaults" "github.com/containerd/containerd/v2/pkg/namespaces" "github.com/containerd/containerd/v2/pkg/sys" @@ -40,6 +44,10 @@ type serverConfig struct { ttrpcAddress string maxRecvMessageSize int maxSendMessageSize int + + // containerd is the configuration loaded from drop-in files. + // Only its plugins, disabled_plugins, and required_plugins settings apply. + containerd srvconfig.Config } type containerdServer struct { @@ -57,7 +65,7 @@ type containerdServer struct { // Command-owned behavior such as process mutation, proxy plugins, TCP/TLS, and // Prometheus registration is intentionally not part of the embedded server. func newServer(ctx context.Context, cfg *serverConfig) (*containerdServer, error) { - registrations := registry.Graph(nil) + registrations := registry.Graph(srvconfig.V2DisabledFilter(cfg.containerd.DisabledPlugins)) return newServerWithRegistrations(ctx, cfg, registrations) } @@ -100,6 +108,10 @@ func newServerWithRegistrations(ctx context.Context, cfg *serverConfig, registra }() initialized := plugin.NewPluginSet() + required := make(map[string]struct{}, len(cfg.containerd.RequiredPlugins)) + for _, id := range cfg.containerd.RequiredPlugins { + required[id] = struct{}{} + } var grpcServices []grpcService var ttrpcServices []ttrpcService @@ -114,6 +126,14 @@ func newServerWithRegistrations(ctx context.Context, cfg *serverConfig, registra plugins.PropertyTTRPCAddress: cfg.ttrpcAddress, }) + if registration.Config != nil { + pluginConfig, err := cfg.containerd.Decode(ctx, id, registration.Config) + if err != nil { + return nil, fmt.Errorf("decoding config for plugin %q: %w", id, err) + } + registration.Config = pluginConfig + } + // Disable cgroups Prometheus metrics for the embedded server to // avoid exposing them through dockerd's process-global Prometheus // registry. @@ -137,6 +157,9 @@ func newServerWithRegistrations(ctx context.Context, cfg *serverConfig, registra if mustSucceed.Load() { return nil, fmt.Errorf("plugin %q failed after registering readiness: %w", id, err) } + if _, ok := required[id]; ok { + return nil, fmt.Errorf("loading required plugin %q: %w", id, err) + } fields := log.Fields{"error": err, "id": id, "type": registration.Type} if plugin.IsSkipPlugin(err) { log.G(ctx).WithFields(fields).Info("skip loading plugin") @@ -145,6 +168,7 @@ func newServerWithRegistrations(ctx context.Context, cfg *serverConfig, registra } continue } + delete(required, id) if service, ok := instance.(grpcService); ok { grpcServices = append(grpcServices, service) @@ -154,6 +178,9 @@ func newServerWithRegistrations(ctx context.Context, cfg *serverConfig, registra } srv.plugins = append(srv.plugins, result) } + if len(required) != 0 { + return nil, fmt.Errorf("required plugins not loaded: %s", strings.Join(slices.Sorted(maps.Keys(required)), ", ")) + } for _, service := range grpcServices { if err := service.Register(srv.grpcServer); err != nil { diff --git a/daemon/internal/containerd/server/embedded/dropin.go b/daemon/internal/containerd/server/embedded/dropin.go new file mode 100644 index 0000000000..e39c44f7ee --- /dev/null +++ b/daemon/internal/containerd/server/embedded/dropin.go @@ -0,0 +1,105 @@ +//go:build (linux || windows) && !no_embedded_containerd + +package embedded + +import ( + "context" + "errors" + "fmt" + "iter" + "maps" + "os" + "path/filepath" + "reflect" + "slices" + "strings" + + srvconfig "github.com/containerd/containerd/v2/cmd/containerd/server/config" + "github.com/containerd/containerd/v2/plugins" + "github.com/containerd/containerd/v2/version" + "github.com/containerd/log" + "github.com/containerd/plugin" + "github.com/containerd/plugin/registry" + "github.com/pelletier/go-toml/v2" +) + +// loadConfigDropIns loads the containerd configuration drop-in files in dir. +func loadConfigDropIns(ctx context.Context, dir string) (srvconfig.Config, error) { + files, err := configDropInFiles(dir) + if err != nil { + return srvconfig.Config{}, err + } + + registrations := func() iter.Seq[plugin.Registration] { + return slices.Values(registry.Graph(nil)) + } + cfg := srvconfig.Config{Version: version.ConfigVersion} + for _, path := range files { + if err := srvconfig.LoadConfigWithPlugins(ctx, path, registrations, &cfg); err != nil { + return srvconfig.Config{}, fmt.Errorf("failed to load containerd config drop-in %q: %w", path, err) + } + } + + // dockerd configures the embedded server itself, and does not implement + // containerd's process and server settings. + ignored := cfg + ignored.Version = 0 + ignored.Plugins = nil + ignored.DisabledPlugins = nil + ignored.RequiredPlugins = nil + // gRPC, TTRPC, debug, and metrics settings are server plugins, which the + // embedded server does not register. + serverSettings := slices.ContainsFunc(slices.Collect(maps.Keys(cfg.Plugins)), func(id string) bool { + return strings.HasPrefix(id, string(plugins.ServerPlugin)+".") + }) + if serverSettings || !reflect.ValueOf(ignored).IsZero() { + log.G(ctx).WithField("dir", dir).Warn("embedded containerd ignores server and process settings in config drop-ins; only plugin, disabled_plugins, and required_plugins settings apply") + } + return cfg, nil +} + +// configDropInFiles returns the containerd configuration files (*.toml) in +// dir, in lexical order. +// A missing directory has no files. +// Files with a configuration version older than 4 or non-empty imports are +// rejected. +func configDropInFiles(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("failed to find containerd config drop-ins: %w", err) + } + var files []string + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".toml") { + continue + } + path := filepath.Join(dir, entry.Name()) + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read containerd config drop-in %q: %w", path, err) + } + var dropIn struct { + Version int `toml:"version"` + Imports []string `toml:"imports"` + } + if err := toml.Unmarshal(data, &dropIn); err != nil { + return nil, fmt.Errorf("failed to parse containerd config drop-in %q: %w", path, err) + } + // Drop-ins have no legacy files to support, so require the configuration + // version introduced in containerd 2.3 instead of relying on migrations + // of older versions, including a missing version being treated as 1. + if dropIn.Version < 4 { + return nil, fmt.Errorf("containerd config drop-in %q must set version 4 or later", path) + } + // Keep the merge order defined only by the lexical order of the files + // in the directory. + if len(dropIn.Imports) != 0 { + return nil, fmt.Errorf("containerd config drop-in %q must not contain imports", path) + } + files = append(files, path) + } + return files, nil +} diff --git a/daemon/internal/containerd/server/embedded/dropin_test.go b/daemon/internal/containerd/server/embedded/dropin_test.go new file mode 100644 index 0000000000..51a1379013 --- /dev/null +++ b/daemon/internal/containerd/server/embedded/dropin_test.go @@ -0,0 +1,109 @@ +//go:build (linux || windows) && !no_embedded_containerd + +package embedded + +import ( + "os" + "path/filepath" + "strconv" + "testing" + + srvconfig "github.com/containerd/containerd/v2/cmd/containerd/server/config" + "github.com/containerd/plugin" + "gotest.tools/v3/assert" + is "gotest.tools/v3/assert/cmp" +) + +func TestConfigDropInsConfigurePlugins(t *testing.T) { + t.Parallel() + + // Brackets must not be interpreted as a glob pattern. + dir := filepath.Join(t.TempDir(), "docker[prod]") + err := os.Mkdir(dir, 0o755) + assert.NilError(t, err) + err = os.WriteFile(filepath.Join(dir, "10-custom.toml"), []byte(` +version = 4 + +[plugins."io.moby.test.v1.configured"] + value = "from-drop-in" +`), 0o644) + assert.NilError(t, err) + + containerdConfig, err := loadConfigDropIns(t.Context(), dir) + assert.NilError(t, err) + + type testConfig struct { + Value string `toml:"value"` + } + var got any + cfg := &serverConfig{ + root: filepath.Join(t.TempDir(), "root"), + state: filepath.Join(t.TempDir(), "state"), + containerd: containerdConfig, + } + server, err := newServerWithRegistrations(t.Context(), cfg, []plugin.Registration{{ + Type: "io.moby.test.v1", + ID: "configured", + Config: &testConfig{Value: "default"}, + InitFn: func(initContext *plugin.InitContext) (any, error) { + got = initContext.Config + return struct{}{}, nil + }, + }}) + assert.NilError(t, err) + server.Stop() + assert.Check(t, is.DeepEqual(got, &testConfig{Value: "from-drop-in"})) +} + +func TestConfigDropInsRequireMissingPlugin(t *testing.T) { + t.Parallel() + + cfg := &serverConfig{ + root: filepath.Join(t.TempDir(), "root"), + state: filepath.Join(t.TempDir(), "state"), + containerd: srvconfig.Config{ + RequiredPlugins: []string{"io.moby.test.v1.missing"}, + }, + } + server, err := newServerWithRegistrations(t.Context(), cfg, nil) + assert.Check(t, is.Nil(server)) + assert.Check(t, is.ErrorContains(err, "required plugins not loaded: io.moby.test.v1.missing")) +} + +func TestConfigDropInsRejectInvalidFiles(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + content string + expErr string + }{ + { + name: "imports", + content: ` +version = 4 +imports = ["nested.conf"] +`, + expErr: "must not contain imports", + }, + { + name: "legacy version", + content: `version = 3`, + expErr: "must set version 4 or later", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + dropInFile := filepath.Join(dir, "10-custom.toml") + err := os.WriteFile(dropInFile, []byte(tc.content), 0o644) + assert.NilError(t, err) + + _, err = loadConfigDropIns(t.Context(), dir) + assert.ErrorContains(t, err, tc.expErr) + assert.ErrorContains(t, err, strconv.Quote(dropInFile)) + }) + } +} diff --git a/daemon/internal/containerd/server/embedded/embedded_test.go b/daemon/internal/containerd/server/embedded/embedded_test.go index 4466c9020e..962731da5a 100644 --- a/daemon/internal/containerd/server/embedded/embedded_test.go +++ b/daemon/internal/containerd/server/embedded/embedded_test.go @@ -21,7 +21,7 @@ func TestPluginGraphResolves(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() - d, err := Start(ctx, t.TempDir(), t.TempDir()) + d, err := Start(ctx, t.TempDir(), t.TempDir(), "") if err != nil { if strings.Contains(err.Error(), "not found") || strings.Contains(err.Error(), "no plugins registered") { t.Fatalf("embedded containerd plugin graph is incomplete: %v", err) diff --git a/daemon/internal/containerd/server/embedded/server.go b/daemon/internal/containerd/server/embedded/server.go index ae102f3886..d9e6d6dfbe 100644 --- a/daemon/internal/containerd/server/embedded/server.go +++ b/daemon/internal/containerd/server/embedded/server.go @@ -12,6 +12,7 @@ import ( "sync/atomic" "time" + srvconfig "github.com/containerd/containerd/v2/cmd/containerd/server/config" "github.com/containerd/containerd/v2/defaults" "github.com/containerd/containerd/v2/version" "github.com/containerd/log" @@ -50,12 +51,24 @@ import ( // Start initializes and runs the embedded containerd server, using rootDir for // persistent state (bolt DB, content store) and the daemon subdirectory under // stateDir for runtime state. +// If configDropInDir is not empty, the plugins, disabled_plugins, and +// required_plugins settings of the containerd configuration files (*.toml) +// in that directory apply to the server, excluding server plugin settings. // See the package doc for the transport layout. // Plugins self-register via the blank imports above and in the platform-specific // files. -func Start(ctx context.Context, rootDir, stateDir string) (*Daemon, error) { +func Start(ctx context.Context, rootDir, stateDir, configDropInDir string) (*Daemon, error) { setContainerdVersion() + var containerdConfig srvconfig.Config + if configDropInDir != "" { + var err error + containerdConfig, err = loadConfigDropIns(ctx, configDropInDir) + if err != nil { + return nil, err + } + } + address := defaultAddress(stateDir) cfg := &serverConfig{ root: rootDir, @@ -64,6 +77,7 @@ func Start(ctx context.Context, rootDir, stateDir string) (*Daemon, error) { ttrpcAddress: address + ".ttrpc", maxRecvMessageSize: defaults.DefaultMaxRecvMsgSize, maxSendMessageSize: defaults.DefaultMaxSendMsgSize, + containerd: containerdConfig, } // Create the root and state directories with the permissions containerd diff --git a/man/dockerd.8.md b/man/dockerd.8.md index 5aefaaaa89..ad697bce4b 100644 --- a/man/dockerd.8.md +++ b/man/dockerd.8.md @@ -201,7 +201,18 @@ Bridge networks will accept packets with this firewall mark/mask. */etc/docker/daemon.json*. **--containerd**="" - Path to containerd socket. + Path to containerd socket. If not set, and no system containerd is running, + the daemon starts and manages its own containerd. + With the *embedded-containerd* feature, the daemon loads containerd + configuration files (*\*.toml*) from the *containerd.d* directory next to + the daemon configuration file (by default */etc/docker/containerd.d*). + Drop-in files must set *version* 4 or later, and must not contain non-empty + *imports* lists. + Only the *plugins*, *disabled_plugins*, and *required_plugins* settings of + these files apply, excluding server settings such as gRPC, TTRPC, debug, and + metrics. + The daemon ignores these files, and logs a warning, when it does not use + embedded containerd. **--data-root**="" Path to the directory used to store persisted Docker data such as