daemon: Create the extension API socket on first use

Every daemon with an extension host opened an extra full-access API
socket at startup, even when no extension used the API.

The socket is now created when an extension first connects to the API.
The daemonapi client looks up the endpoint on its first connection, so
NewClient no longer takes a context.

The socket is served separately from the configured hosts, returns 503
until the daemon is ready, and closes after extensions shut down.
It is accessible only to the daemon user, or to Administrators and
LocalSystem on Windows.

Signed-off-by: Paweł Gronowski <pawel.gronowski@docker.com>
This commit is contained in:
Paweł Gronowski committed 2026-11-04 19:48:07 +01:00
1 parent fca6d1f16d
commit be285dcad8
14 files changed
+527 -43

No files matched your search

+3 -9
View File
@@ -360,16 +360,10 @@ func (cli *daemonCLI) start(ctx context.Context) (retErr error) {
if err != nil {
return err
}
// Serve the API to extensions on a private socket.
extLs, err := d.ExtensionAPIListener()
if err != nil {
return err
}
if extLs != nil {
lss = append(lss, extLs)
}
httpServer.Protocols = &p
httpServer.Handler = newHTTPHandler(ctx, gs, extProxy, apiServer.CreateMux(ctx, routers...))
handler := newHTTPHandler(ctx, gs, extProxy, apiServer.CreateMux(ctx, routers...))
httpServer.Handler = handler
d.StartAPISockets(handler)
go d.ProcessClusterNotifications(ctx, c.GetWatchStream())
+8
View File
@@ -38,6 +38,7 @@ import (
networktypes "github.com/moby/moby/api/types/network"
registrytypes "github.com/moby/moby/api/types/registry"
"github.com/moby/moby/api/types/swarm"
"github.com/moby/moby/v2/daemon/internal/apisocket"
"github.com/moby/moby/v2/daemon/internal/nri"
"github.com/moby/sys/user"
"github.com/moby/sys/userns"
@@ -136,6 +137,7 @@ type Daemon struct {
nri *nri.NRI
pluginManager *plugin.Manager
extensionHost *host.Host
apiSockets *apisocket.Manager
linkIndex *linkIndex
containerdClient *containerd.Client
containerd libcontainerdtypes.Client
@@ -933,6 +935,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
d := &Daemon{
PluginStore: pluginStore,
startupDone: make(chan struct{}),
apiSockets: apisocket.NewManager(),
}
cfgStore := &configStore{
Config: *config,
@@ -1556,6 +1559,11 @@ func (daemon *Daemon) Shutdown(ctx context.Context) error {
log.G(ctx).WithError(err).Error("failed to shut down extensions")
}
}
// Close the API sockets only after extensions have shut down, so they can
// still use the API while shutting down.
if err := daemon.apiSockets.Close(); err != nil {
log.G(ctx).WithError(err).Error("failed to close API sockets")
}
// Keep mounts and networking running on daemon shutdown if
// we are to keep containers running and restore them.
+23 -18
View File
@@ -5,7 +5,7 @@ import (
"errors"
"fmt"
"io/fs"
"net"
"net/http"
"os"
"path/filepath"
"slices"
@@ -31,6 +31,12 @@ func (daemon *Daemon) newExtensionHost(ctx context.Context) (*host.Host, error)
if err := os.Chmod(runtimeDir, 0o700); err != nil {
return nil, fmt.Errorf("set permissions on extension runtime directory: %w", err)
}
// A crashed daemon leaves its API socket behind. Opening the socket
// replaces it, but a daemon whose extensions never use the API would
// otherwise keep the dead socket around.
if err := os.Remove(extensionAPISocket(&cfg)); err != nil && !errors.Is(err, fs.ErrNotExist) {
log.G(ctx).WithError(err).Warn("failed to remove stale extension API socket")
}
defaults := slices.Concat(builtinExtensions, daemon.daemonExtensions())
exts, err := selectBuiltinExtensions(cfg.Extensions, defaults, daemon.optionalExtensions())
if err != nil {
@@ -55,30 +61,29 @@ func extensionRuntimeDir(cfg *config.Config) string {
return filepath.Join(cfg.ExecRoot, "extensions")
}
// extensionAPISocket is the path of the API socket served for extensions.
// extensionAPISocket is the path of the API socket for extensions.
func extensionAPISocket(cfg *config.Config) string {
return filepath.Join(extensionRuntimeDir(cfg), "api.sock")
}
// ExtensionAPIListener listens on the API socket that the daemon.api point
// gives to extensions.
// It is separate from the configured hosts, which may be TLS-only TCP or
// socket-activated and so unusable by extensions.
// It returns nil if the daemon has no extension host.
func (daemon *Daemon) ExtensionAPIListener() (net.Listener, error) {
if daemon.extensionHost == nil {
return nil, nil
}
// extensionAPIEndpoint provides the daemon.api point.
// It opens the API socket for extensions when an extension first asks for the
// endpoint, so daemons without such extensions expose no extra API endpoint.
// The socket is separate from the configured hosts, which may be TLS-only TCP
// or socket-activated and so unusable by extensions.
func (daemon *Daemon) extensionAPIEndpoint(ctx context.Context) (string, error) {
cfg := daemon.Config()
path := extensionAPISocket(&cfg)
if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) {
return nil, fmt.Errorf("remove stale extension API socket: %w", err)
if err := daemon.apiSockets.Open(ctx, path); err != nil {
return "", err
}
ls, err := net.Listen("unix", path)
if err != nil {
return nil, fmt.Errorf("listen on extension API socket: %w", err)
}
return ls, nil
return "unix://" + path, nil
}
// StartAPISockets serves h on the API sockets the daemon creates for its own
// consumers, such as extensions.
func (daemon *Daemon) StartAPISockets(h http.Handler) {
daemon.apiSockets.Start(h)
}
// extensionConfig converts daemon.json's extensions.configs entries to host form.
+1 -1
View File
@@ -43,7 +43,7 @@ func (daemon *Daemon) daemonExtensions() []extensions.Extension {
return []extensions.Extension{
extensionconfig.New(&cfg),
newMonolithExtension(daemon),
extensionapi.New("unix://" + extensionAPISocket(&cfg)),
extensionapi.New(daemon.extensionAPIEndpoint),
}
}
+13
View File
@@ -0,0 +1,13 @@
//go:build !windows
package apisocket
import (
"net"
"github.com/docker/go-connections/sockets"
)
func listen(path string) (net.Listener, error) {
return sockets.NewUnixSocketWithOpts(path, sockets.WithChmod(0o600))
}
@@ -0,0 +1,11 @@
package apisocket
import (
"net"
"github.com/docker/go-connections/sockets"
)
func listen(path string) (net.Listener, error) {
return sockets.NewUnixSocketWithOpts(path, sockets.WithBasePermissions())
}
+161
View File
@@ -0,0 +1,161 @@
// Package apisocket serves the Engine API on unix sockets that the daemon
// creates for its own consumers, separately from the configured hosts.
package apisocket
import (
"context"
"errors"
"fmt"
"net"
"net/http"
"sync"
"sync/atomic"
"time"
"github.com/containerd/log"
)
// Manager owns the daemon-created API sockets and serves them all with one
// server.
type Manager struct {
handler atomic.Pointer[http.Handler]
srv *http.Server
mu sync.Mutex
sockets map[string]*listener
closed bool
}
// NewManager returns a Manager with no sockets.
func NewManager() *Manager {
m := &Manager{sockets: make(map[string]*listener)}
var p http.Protocols
p.SetHTTP1(true)
p.SetUnencryptedHTTP2(true)
m.srv = &http.Server{
Handler: http.HandlerFunc(m.serveHTTP),
Protocols: &p,
ReadHeaderTimeout: 5 * time.Minute, // Same as the configured hosts.
}
return m
}
// Open creates a unix socket at path and serves the API on it.
// On Unix, the socket is accessible only to its owner.
// Until [Manager.Start], requests fail with 503 Service Unavailable, so a
// socket can be handed out before the API is ready.
// Opening a path that is already open does nothing.
func (m *Manager) Open(ctx context.Context, path string) error {
m.mu.Lock()
defer m.mu.Unlock()
if m.closed {
return errors.New("API socket manager is closed")
}
if _, ok := m.sockets[path]; ok {
return nil
}
ls, err := listen(path)
if err != nil {
return fmt.Errorf("listen on API socket %q: %w", path, err)
}
ul, ok := ls.(*net.UnixListener)
if !ok {
_ = ls.Close()
return fmt.Errorf("listen on API socket %q: unexpected listener type %T", path, ls)
}
l := &listener{UnixListener: ul, conns: make(map[*conn]struct{})}
m.sockets[path] = l
log.G(ctx).Infof("API listen on %s", path)
go func() {
if err := m.srv.Serve(l); err != nil && !errors.Is(err, http.ErrServerClosed) {
log.G(ctx).WithFields(log.Fields{
"error": err,
"listener": path,
}).Error("ServeAPI error")
}
}()
return nil
}
// Start serves h on all sockets, including ones opened later.
func (m *Manager) Start(h http.Handler) {
m.handler.Store(&h)
}
func (m *Manager) serveHTTP(w http.ResponseWriter, r *http.Request) {
h := m.handler.Load()
if h == nil {
http.Error(w, "the daemon is not ready to serve API requests", http.StatusServiceUnavailable)
return
}
(*h).ServeHTTP(w, r)
}
// Close stops serving all sockets, closes all their connections, including
// hijacked ones, and removes the socket files.
// Later calls to [Manager.Open] fail.
func (m *Manager) Close() error {
m.mu.Lock()
defer m.mu.Unlock()
m.closed = true
err := m.srv.Close()
for _, l := range m.sockets {
// Serve may not track the listener yet; closing it here removes the
// socket file either way.
_ = l.Close()
l.closeConns()
}
clear(m.sockets)
return err
}
// listener tracks its connections, because the server does not track
// hijacked connections such as attach and exec streams.
type listener struct {
*net.UnixListener
mu sync.Mutex
conns map[*conn]struct{}
}
func (l *listener) Accept() (net.Conn, error) {
uc, err := l.AcceptUnix()
if err != nil {
return nil, err
}
c := &conn{UnixConn: uc, l: l}
l.mu.Lock()
l.conns[c] = struct{}{}
l.mu.Unlock()
return c, nil
}
func (l *listener) closeConns() {
l.mu.Lock()
conns := make([]*conn, 0, len(l.conns))
for c := range l.conns {
conns = append(conns, c)
}
l.mu.Unlock()
for _, c := range conns {
_ = c.Close()
}
}
// conn embeds *net.UnixConn so that CloseWrite stays available for
// half-closing hijacked streams.
type conn struct {
*net.UnixConn
l *listener
}
func (c *conn) Close() error {
c.l.mu.Lock()
delete(c.l.conns, c)
c.l.mu.Unlock()
return c.UnixConn.Close()
}
@@ -0,0 +1,89 @@
package apisocket
import (
"errors"
"fmt"
"net"
"os"
"path/filepath"
"runtime"
"strconv"
"syscall"
"testing"
"time"
"github.com/moby/sys/reexec"
"gotest.tools/v3/assert"
"gotest.tools/v3/skip"
)
const (
socketConnectCommand = "apisocket-test-connect"
unprivilegedUID = 65534
)
func TestMain(m *testing.M) {
reexec.Register(socketConnectCommand, func() {
if uid := os.Geteuid(); uid != unprivilegedUID {
fmt.Fprintf(os.Stderr, "connection probe has unexpected UID %d\n", uid)
os.Exit(1)
}
c, err := net.DialTimeout("unix", os.Args[1], 5*time.Second) // #nosec G704 -- reexec dials a test-owned Unix socket.
if c != nil {
defer c.Close()
}
if os.Args[2] == "true" && err == nil {
return
}
if os.Args[2] == "false" && errors.Is(err, os.ErrPermission) {
return
}
fmt.Fprintf(os.Stderr, "unexpected connection result (allow %s): %v\n", os.Args[2], err)
os.Exit(1)
})
if reexec.Init() {
return
}
os.Exit(m.Run())
}
func TestManagerRejectsOtherUsers(t *testing.T) {
t.Parallel()
skip.If(t, os.Geteuid() != 0, "requires root to connect as another user")
// Keep every parent traversable so only the socket can deny access.
dir, err := os.MkdirTemp("/tmp", "apisocket")
assert.NilError(t, err)
t.Cleanup(func() { _ = os.RemoveAll(dir) })
assert.NilError(t, os.Chmod(dir, 0o755))
path := filepath.Join(dir, "api.sock")
m := NewManager()
t.Cleanup(func() { assert.NilError(t, m.Close()) })
assert.NilError(t, m.Open(t.Context(), path))
checkConnection := func(gid uint32, allow bool) {
t.Helper()
// Keep the creating thread alive for reexec's parent-death signal.
runtime.LockOSThread()
defer runtime.UnlockOSThread()
cmd := reexec.Command(socketConnectCommand, path, strconv.FormatBool(allow))
cmd.SysProcAttr.Credential = &syscall.Credential{Uid: unprivilegedUID, Gid: gid}
out, err := cmd.CombinedOutput()
assert.NilError(t, err, "connection probe with GID %d (allow %t): %s", gid, allow, out)
}
sameGroup := uint32(os.Getegid())
checkConnection(sameGroup, false)
checkConnection(sameGroup+1, false)
// Prove both probes can reach the listener when socket access is allowed.
assert.NilError(t, os.Chmod(path, 0o666))
checkConnection(sameGroup, true)
checkConnection(sameGroup+1, true)
}
+79
View File
@@ -0,0 +1,79 @@
package apisocket
import (
"bufio"
"context"
"errors"
"io"
"io/fs"
"net"
"net/http"
"os"
"path/filepath"
"testing"
"time"
"gotest.tools/v3/assert"
is "gotest.tools/v3/assert/cmp"
)
func TestManagerLifecycle(t *testing.T) {
// t.TempDir paths can exceed the unix socket path limit on macOS.
dir, err := os.MkdirTemp("", "apisocket")
assert.NilError(t, err)
t.Cleanup(func() { _ = os.RemoveAll(dir) })
path := filepath.Join(dir, "api.sock")
ctx := t.Context()
m := NewManager()
assert.NilError(t, m.Open(ctx, path))
assert.NilError(t, m.Open(ctx, path), "opening an open socket again must be a no-op")
c := &http.Client{Transport: &http.Transport{
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", path)
},
}}
t.Cleanup(c.CloseIdleConnections)
status := func() int {
t.Helper()
resp, err := c.Get("http://api/_ping")
assert.NilError(t, err)
_ = resp.Body.Close()
return resp.StatusCode
}
assert.Check(t, is.Equal(status(), http.StatusServiceUnavailable), "requests before Start must fail")
m.Start(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/hijack" {
w.WriteHeader(http.StatusNoContent)
return
}
// Keep the hijacked connection open, like an attach stream.
hc, _, err := http.NewResponseController(w).Hijack()
if err == nil {
_, _ = hc.Write([]byte("hijacked\n"))
}
}))
assert.Check(t, is.Equal(status(), http.StatusNoContent))
hijacked, err := net.Dial("unix", path)
assert.NilError(t, err)
defer hijacked.Close()
_, err = hijacked.Write([]byte("GET /hijack HTTP/1.1\r\nHost: api\r\n\r\n"))
assert.NilError(t, err)
br := bufio.NewReader(hijacked)
line, err := br.ReadString('\n')
assert.NilError(t, err)
assert.Check(t, is.Equal(line, "hijacked\n"))
assert.NilError(t, m.Close())
_, err = os.Stat(path)
assert.Check(t, errors.Is(err, fs.ErrNotExist), "Close must remove the socket file")
assert.Check(t, m.Open(ctx, path) != nil, "Open after Close must fail")
assert.NilError(t, hijacked.SetReadDeadline(time.Now().Add(10*time.Second)))
_, err = br.ReadByte()
assert.Check(t, is.ErrorIs(err, io.EOF), "Close must close hijacked connections")
}
+10 -5
View File
@@ -13,8 +13,9 @@ import (
daemonapiv0 "github.com/moby/moby/v2/extpoints/daemonapi/v0"
)
// New declares a provider for the API endpoint the daemon serves at host.
func New(host string) extensions.Extension {
// New declares a provider for the API endpoint returned by host.
// host is called each time an extension resolves the endpoint.
func New(host func(context.Context) (string, error)) extensions.Extension {
return extensions.New(extensions.Declaration{
ID: "org.mobyproject.daemon.api.v0",
Providers: []extensions.Provider{
@@ -24,9 +25,13 @@ func New(host string) extensions.Extension {
}
type endpoint struct {
host string
host func(context.Context) (string, error)
}
func (e endpoint) Get(context.Context, *daemonapiv0.GetRequest) (*daemonapiv0.GetResponse, error) {
return &daemonapiv0.GetResponse{Host: e.host}, nil
func (e endpoint) Get(ctx context.Context, _ *daemonapiv0.GetRequest) (*daemonapiv0.GetResponse, error) {
host, err := e.host(ctx)
if err != nil {
return nil, err
}
return &daemonapiv0.GetResponse{Host: host}, nil
}
+59 -8
View File
@@ -3,23 +3,74 @@ package daemonapiv0
import (
"context"
"fmt"
"net"
"strings"
"github.com/moby/moby/client"
)
// NewClient returns an Engine API client connected to the endpoint api
// reports.
// placeholderHost only selects the plain unix transport; the client dials the
// endpoint the provider reports.
// The client includes its host in connection errors, so the name points at
// where the real endpoint comes from rather than looking like a real path.
const placeholderHost = "unix:///resolved-by-daemonapi-v0.sock"
// NewClient returns an Engine API client for the endpoint api reports.
// It is a temporary shortcut; prefer a specific extension point when one
// covers what the extension needs.
// Resolve api with Point.Single during initialization, after declaring
// Point.Dependency().
// The client asks api for the endpoint when it first connects, so the daemon
// creates the socket only once an extension actually uses the API.
// opts are applied after the endpoint, so they may override it.
//
// The daemon serves the API only after every extension has initialized, so do
// not make API calls from Init; they would block daemon startup.
// The daemon serves the API only after it finishes initializing, so API calls
// made before then, including from Init, fail.
// The caller owns the returned client and must close it.
func NewClient(ctx context.Context, api DaemonAPI, opts ...client.Opt) (*client.Client, error) {
resp, err := api.Get(ctx, &GetRequest{})
func NewClient(api DaemonAPI, opts ...client.Opt) (*client.Client, error) {
d := &dialer{api: api, lock: make(chan struct{}, 1)}
return client.New(append([]client.Opt{
client.WithHost(placeholderHost),
client.WithDialContext(d.dial),
}, opts...)...)
}
type dialer struct {
api DaemonAPI
// lock is a one-slot semaphore rather than a mutex so that dials waiting
// on a slow Get still honor their own context.
lock chan struct{}
path string
}
func (d *dialer) dial(ctx context.Context, _, _ string) (net.Conn, error) {
path, err := d.socketPath(ctx)
if err != nil {
return nil, fmt.Errorf("get daemon API endpoint: %w", err)
return nil, err
}
return client.New(append([]client.Opt{client.WithHost(resp.Host)}, opts...)...)
var nd net.Dialer
return nd.DialContext(ctx, "unix", path)
}
func (d *dialer) socketPath(ctx context.Context) (string, error) {
select {
case d.lock <- struct{}{}:
case <-ctx.Done():
return "", ctx.Err()
}
defer func() { <-d.lock }()
if d.path != "" {
return d.path, nil
}
resp, err := d.api.Get(ctx, &GetRequest{})
if err != nil {
return "", fmt.Errorf("get daemon API endpoint: %w", err)
}
path, ok := strings.CutPrefix(resp.Host, "unix://")
if !ok {
return "", fmt.Errorf("unsupported daemon API endpoint %q", resp.Host)
}
d.path = path
return path, nil
}
+60
View File
@@ -0,0 +1,60 @@
package daemonapiv0_test
import (
"context"
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sync/atomic"
"testing"
"github.com/moby/moby/client"
daemonapiv0 "github.com/moby/moby/v2/extpoints/daemonapi/v0"
"gotest.tools/v3/assert"
is "gotest.tools/v3/assert/cmp"
)
type fakeAPI struct {
host string
calls atomic.Int32
}
func (f *fakeAPI) Get(context.Context, *daemonapiv0.GetRequest) (*daemonapiv0.GetResponse, error) {
f.calls.Add(1)
return &daemonapiv0.GetResponse{Host: f.host}, nil
}
func TestNewClientGetsEndpointOnFirstConnect(t *testing.T) {
t.Parallel()
// t.TempDir paths can exceed the unix socket path limit on macOS.
dir, err := os.MkdirTemp("", "daemonapi")
assert.NilError(t, err)
t.Cleanup(func() { _ = os.RemoveAll(dir) })
sock := filepath.Join(dir, "api.sock")
l, err := net.Listen("unix", sock)
assert.NilError(t, err)
srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
// Force a new connection per request so the second Ping dials again.
w.Header().Set("Connection", "close")
}))
_ = srv.Listener.Close()
srv.Listener = l
srv.Start()
t.Cleanup(srv.Close)
api := &fakeAPI{host: "unix://" + sock}
c, err := daemonapiv0.NewClient(api)
assert.NilError(t, err)
t.Cleanup(func() { _ = c.Close() })
assert.Check(t, is.Equal(api.calls.Load(), int32(0)))
for range 2 {
_, err = c.Ping(t.Context(), client.PingOptions{})
assert.NilError(t, err)
}
assert.Check(t, is.Equal(api.calls.Load(), int32(1)))
}
+8
View File
@@ -27,6 +27,14 @@ type DaemonAPI interface {
// The daemon registers its builtin provider for extension dependencies,
// including out-of-process extensions, but does not publish it on the public
// daemon socket.
//
// The endpoint is a unix socket that only the daemon user can access; on
// Windows, only Administrators and LocalSystem.
// The daemon creates it on the first Get call, which a [NewClient] client
// makes when it first connects.
// Extensions run as the daemon user, so every extension can reach the socket
// and gets unrestricted Engine API access, whether or not it declares a
// dependency on this point.
var Point = extensions.DefineSinglePoint[DaemonAPI]("org.mobyproject.extension.daemon.api.v0")
// GetRequest requests the daemon API endpoint.
+2 -2
View File
@@ -44,12 +44,12 @@ func (e *extension) Declaration() extensions.Declaration {
Dependencies: []extensions.Dependency{
daemonapiv0.Point.Dependency(),
},
Init: func(ctx context.Context, _ extensions.Config, resolver extensions.Resolver) error {
Init: func(_ context.Context, _ extensions.Config, resolver extensions.Resolver) error {
provider, err := daemonapiv0.Point.Single(resolver)
if err != nil {
return err
}
api, err := daemonapiv0.NewClient(ctx, provider)
api, err := daemonapiv0.NewClient(provider)
if err != nil {
return err
}