From 4e246efcd117ea01526c0c7a1c50846f947bade0 Mon Sep 17 00:00:00 2001 From: Albin Kerouanton Date: Wed, 2 Jul 2025 14:59:26 +0200 Subject: [PATCH] libnet/d/bridge: mv portmapper to libnet/pms/{nat,routed} Signed-off-by: Albin Kerouanton --- daemon/daemon.go | 3 +- daemon/daemon_unix.go | 31 +- daemon/daemon_windows.go | 2 +- daemon/libnetwork/config/config.go | 20 + .../libnetwork/drivers/bridge/bridge_linux.go | 71 +--- .../drivers/bridge/bridge_linux_test.go | 26 +- .../drivers/bridge/port_mapping_linux.go | 342 +++--------------- .../drivers/bridge/port_mapping_linux_test.go | 324 +++++++++-------- daemon/libnetwork/drivers_linux.go | 29 ++ daemon/libnetwork/portmapperapi/api.go | 16 +- daemon/libnetwork/portmapperapi/api_test.go | 2 +- .../portmappers/nat/mapper_linux.go | 325 +++++++++++++++++ .../portmappers/nat/mapper_linux_test.go | 36 ++ .../portmappers/routed/mapper_linux.go | 60 +++ 14 files changed, 731 insertions(+), 556 deletions(-) create mode 100644 daemon/libnetwork/portmappers/nat/mapper_linux.go create mode 100644 daemon/libnetwork/portmappers/nat/mapper_linux_test.go create mode 100644 daemon/libnetwork/portmappers/routed/mapper_linux.go diff --git a/daemon/daemon.go b/daemon/daemon.go index 3f2cd9f421..fd938928ac 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -1458,9 +1458,10 @@ func (daemon *Daemon) networkOptions(conf *config.Config, pg plugingetter.Plugin nwconfig.OptionLabels(conf.Labels), nwconfig.OptionNetworkControlPlaneMTU(conf.NetworkControlPlaneMTU), nwconfig.OptionFirewallBackend(conf.FirewallBackend), - driverOptions(conf), } + options = append(options, networkPlatformOptions(conf)...) + defaultAddressPools := ipamutils.GetLocalScopeDefaultNetworks() if len(conf.NetworkConfig.DefaultAddressPools.Value()) > 0 { defaultAddressPools = conf.NetworkConfig.DefaultAddressPools.Value() diff --git a/daemon/daemon_unix.go b/daemon/daemon_unix.go index a1f2a41c3f..978fb29c2f 100644 --- a/daemon/daemon_unix.go +++ b/daemon/daemon_unix.go @@ -924,20 +924,23 @@ func setHostGatewayIP(controller *libnetwork.Controller, config *config.Config) } } -func driverOptions(config *config.Config) nwconfig.Option { - return nwconfig.OptionDriverConfig("bridge", options.Generic{ - netlabel.GenericData: options.Generic{ - "EnableIPForwarding": config.BridgeConfig.EnableIPForward, - "DisableFilterForwardDrop": config.BridgeConfig.DisableFilterForwardDrop, - "EnableIPTables": config.BridgeConfig.EnableIPTables, - "EnableIP6Tables": config.BridgeConfig.EnableIP6Tables, - "EnableUserlandProxy": config.EnableUserlandProxy, - "UserlandProxyPath": config.UserlandProxyPath, - "Hairpin": !config.EnableUserlandProxy || config.UserlandProxyPath == "", - "AllowDirectRouting": config.BridgeConfig.AllowDirectRouting, - "Rootless": config.Rootless, - }, - }) +// networkPlatformOptions returns a slice of platform-specific libnetwork +// options. +func networkPlatformOptions(conf *config.Config) []nwconfig.Option { + return []nwconfig.Option{ + nwconfig.OptionRootless(conf.Rootless), + nwconfig.OptionUserlandProxy(conf.EnableUserlandProxy, conf.UserlandProxyPath), + nwconfig.OptionDriverConfig("bridge", options.Generic{ + netlabel.GenericData: options.Generic{ + "EnableIPForwarding": conf.BridgeConfig.EnableIPForward, + "DisableFilterForwardDrop": conf.BridgeConfig.DisableFilterForwardDrop, + "EnableIPTables": conf.BridgeConfig.EnableIPTables, + "EnableIP6Tables": conf.BridgeConfig.EnableIP6Tables, + "Hairpin": !conf.EnableUserlandProxy || conf.UserlandProxyPath == "", + "AllowDirectRouting": conf.BridgeConfig.AllowDirectRouting, + }, + }), + } } type defBrOptsV4 struct { diff --git a/daemon/daemon_windows.go b/daemon/daemon_windows.go index 62ccdfc8d0..a20aeb2888 100644 --- a/daemon/daemon_windows.go +++ b/daemon/daemon_windows.go @@ -523,7 +523,7 @@ func (daemon *Daemon) conditionalUnmountOnCleanup(container *container.Container return daemon.Unmount(container) } -func driverOptions(_ *config.Config) nwconfig.Option { +func networkPlatformOptions(_ *config.Config) []nwconfig.Option { return nil } diff --git a/daemon/libnetwork/config/config.go b/daemon/libnetwork/config/config.go index ff6b51f86b..d384098319 100644 --- a/daemon/libnetwork/config/config.go +++ b/daemon/libnetwork/config/config.go @@ -43,6 +43,9 @@ type Config struct { ActiveSandboxes map[string]any PluginGetter plugingetter.PluginGetter FirewallBackend string + Rootless bool + EnableUserlandProxy bool + UserlandProxyPath string } // New creates a new Config and initializes it with the given Options. @@ -162,3 +165,20 @@ func OptionFirewallBackend(val string) Option { c.FirewallBackend = val } } + +// OptionRootless returns an option setter that indicates whether the daemon is +// running in rootless mode. +func OptionRootless(rootless bool) Option { + return func(c *Config) { + c.Rootless = rootless + } +} + +// OptionUserlandProxy returns an option setter that indicates whether the +// userland proxy is enabled, and sets the path to the proxy binary. +func OptionUserlandProxy(enabled bool, proxyPath string) Option { + return func(c *Config) { + c.EnableUserlandProxy = enabled + c.UserlandProxyPath = proxyPath + } +} diff --git a/daemon/libnetwork/drivers/bridge/bridge_linux.go b/daemon/libnetwork/drivers/bridge/bridge_linux.go index cc3dc657f9..a5c037b30b 100644 --- a/daemon/libnetwork/drivers/bridge/bridge_linux.go +++ b/daemon/libnetwork/drivers/bridge/bridge_linux.go @@ -24,7 +24,6 @@ import ( "github.com/docker/docker/daemon/libnetwork/drvregistry" "github.com/docker/docker/daemon/libnetwork/internal/netiputil" "github.com/docker/docker/daemon/libnetwork/internal/nftables" - "github.com/docker/docker/daemon/libnetwork/internal/rlkclient" "github.com/docker/docker/daemon/libnetwork/iptables" "github.com/docker/docker/daemon/libnetwork/netlabel" "github.com/docker/docker/daemon/libnetwork/netutils" @@ -51,7 +50,6 @@ const ( vethPrefix = "veth" vethLen = len(vethPrefix) + 7 defaultContainerVethPrefix = "eth" - maxAllocatePortAttempts = 10 ) const ( @@ -74,13 +72,10 @@ type configuration struct { DisableFilterForwardDrop bool EnableIPTables bool EnableIP6Tables bool - EnableUserlandProxy bool - UserlandProxyPath string // Hairpin indicates whether packets sent from a container to a host port // published by another container on the same bridge network should be // hairpinned. Hairpin bool - Rootless bool AllowDirectRouting bool } @@ -161,25 +156,14 @@ type bridgeNetwork struct { sync.Mutex } -type portDriverClient interface { - ChildHostIP(hostIP netip.Addr) netip.Addr - AddPort(ctx context.Context, proto string, hostIP, childIP netip.Addr, hostPort int) (func() error, error) -} - -// Allow unit tests to supply a dummy RootlessKit port driver client. -var newPortDriverClient = func(ctx context.Context) (portDriverClient, error) { - return rlkclient.NewPortDriverClient(ctx) -} - type driver struct { - config configuration - networks map[string]*bridgeNetwork - store *datastore.Store - nlh nlwrap.Handle - portDriverClient portDriverClient - configNetwork sync.Mutex - firewaller firewaller.Firewaller - portmappers *drvregistry.PortMappers + config configuration + networks map[string]*bridgeNetwork + store *datastore.Store + nlh nlwrap.Handle + configNetwork sync.Mutex + firewaller firewaller.Firewaller + portmappers *drvregistry.PortMappers sync.Mutex } @@ -476,15 +460,6 @@ func (n *bridgeNetwork) gwMode(v firewaller.IPVersion) gwMode { return n.config.GwModeIPv6 } -func (n *bridgeNetwork) userlandProxyPath() string { - n.Lock() - defer n.Unlock() - if n.driver == nil { - return "" - } - return n.driver.userlandProxyPath() -} - func (n *bridgeNetwork) hairpin() bool { n.Lock() defer n.Unlock() @@ -494,13 +469,13 @@ func (n *bridgeNetwork) hairpin() bool { return n.driver.config.Hairpin } -func (n *bridgeNetwork) getPortDriverClient() portDriverClient { +func (n *bridgeNetwork) portMappers() *drvregistry.PortMappers { n.Lock() defer n.Unlock() if n.driver == nil { return nil } - return n.driver.getPortDriverClient() + return n.driver.portmappers } func (n *bridgeNetwork) getEndpoint(eid string) (*bridgeEndpoint, error) { @@ -546,17 +521,7 @@ func (d *driver) configure(option map[string]interface{}) error { return err } - var pdc portDriverClient - if config.Rootless { - var err error - pdc, err = newPortDriverClient(context.TODO()) - if err != nil { - return err - } - } - d.Lock() - d.portDriverClient = pdc d.config = config d.Unlock() @@ -604,22 +569,6 @@ func (d *driver) getNetwork(id string) (*bridgeNetwork, error) { return nil, types.NotFoundErrorf("network not found: %s", id) } -func (d *driver) userlandProxyPath() string { - d.Lock() - defer d.Unlock() - - if d.config.EnableUserlandProxy { - return d.config.UserlandProxyPath - } - return "" -} - -func (d *driver) getPortDriverClient() portDriverClient { - d.Lock() - defer d.Unlock() - return d.portDriverClient -} - func parseNetworkGenericOptions(data interface{}) (*networkConfiguration, error) { var ( err error @@ -1639,7 +1588,7 @@ func (ep *bridgeEndpoint) trimPortBindings(ctx context.Context, n *bridgeNetwork return nil, nil } - if err := releasePortBindings(toDrop, n.firewallerNetwork); err != nil { + if err := n.unmapPBs(ctx, toDrop); err != nil { log.G(ctx).WithFields(log.Fields{ "error": err, "gw4": pbmReq.ipv4, diff --git a/daemon/libnetwork/drivers/bridge/bridge_linux_test.go b/daemon/libnetwork/drivers/bridge/bridge_linux_test.go index b7640e8378..f2daa3c744 100644 --- a/daemon/libnetwork/drivers/bridge/bridge_linux_test.go +++ b/daemon/libnetwork/drivers/bridge/bridge_linux_test.go @@ -8,7 +8,6 @@ import ( "maps" "net" "net/netip" - "os/exec" "slices" "strconv" "testing" @@ -796,21 +795,16 @@ func testQueryEndpointInfo(t *testing.T, ulPxyEnabled bool) { defer netnsutils.SetupTestOSContext(t)() useStubFirewaller(t) - d := newDriver(storeutils.NewTempStore(t), &drvregistry.PortMappers{}) + pms := drvregistry.PortMappers{} + pm := &stubPortMapper{} + err := pms.Register("nat", pm) + assert.NilError(t, err) + + d := newDriver(storeutils.NewTempStore(t), &pms) portallocator.Get().ReleaseAll() - var proxyBinary string - var err error - if ulPxyEnabled { - proxyBinary, err = exec.LookPath("docker-proxy") - if err != nil { - t.Fatalf("failed to lookup userland-proxy binary: %v", err) - } - } config := &configuration{ - EnableIPTables: true, - EnableUserlandProxy: ulPxyEnabled, - UserlandProxyPath: proxyBinary, + EnableIPTables: true, } genericOption := make(map[string]interface{}) genericOption[netlabel.GenericData] = config @@ -865,15 +859,15 @@ func testQueryEndpointInfo(t *testing.T, ulPxyEnabled bool) { if !ok { t.Fatal("Endpoint operational data does not contain port mapping data") } - pm, ok := pmd.([]types.PortBinding) + pbs, ok := pmd.([]types.PortBinding) if !ok { t.Fatal("Unexpected format for port mapping in endpoint operational data") } - if len(ep.portMapping) != len(pm) { + if len(ep.portMapping) != len(pbs) { t.Fatal("Incomplete data for port mapping in endpoint operational data") } for i, pb := range ep.portMapping { - if !comparePortBinding(&pb.PortBinding, &pm[i]) { + if !comparePortBinding(&pb.PortBinding, &pbs[i]) { t.Fatal("Unexpected data for port mapping in endpoint operational data") } } diff --git a/daemon/libnetwork/drivers/bridge/port_mapping_linux.go b/daemon/libnetwork/drivers/bridge/port_mapping_linux.go index 191f889b84..34839de3da 100644 --- a/daemon/libnetwork/drivers/bridge/port_mapping_linux.go +++ b/daemon/libnetwork/drivers/bridge/port_mapping_linux.go @@ -8,25 +8,15 @@ import ( "errors" "fmt" "net" - "net/netip" - "os" "slices" - "strconv" - "syscall" "github.com/containerd/log" - "github.com/docker/docker/daemon/libnetwork/drivers/bridge/internal/firewaller" - "github.com/docker/docker/daemon/libnetwork/internal/rlkclient" "github.com/docker/docker/daemon/libnetwork/netutils" - "github.com/docker/docker/daemon/libnetwork/portallocator" - "github.com/docker/docker/daemon/libnetwork/portmapper" "github.com/docker/docker/daemon/libnetwork/portmapperapi" "github.com/docker/docker/daemon/libnetwork/types" + "github.com/docker/docker/internal/sliceutil" ) -// Allow unit tests to supply a dummy StartProxy. -var startProxy = portmapper.StartProxy - // addPortMappings takes cfg, the configuration for port mappings, selects host // ports when ranges are given, binds host ports to check they're available and // reserve them, starts docker-proxy if required, and sets up iptables @@ -50,20 +40,23 @@ func (n *bridgeNetwork) addPortMappings( defHostIP = addr4 } + pms := n.portMappers() + bindings := make([]portmapperapi.PortBinding, 0, len(cfg)*2) defer func() { if retErr != nil { - if err := releasePortBindings(bindings, n.firewallerNetwork); err != nil { - log.G(ctx).Warnf("Release port bindings: %s", err.Error()) + if err := n.unmapPBs(ctx, bindings); err != nil { + log.G(ctx).WithFields(log.Fields{ + "bindings": bindings, + "error": err, + "origErr": retErr, + }).Warn("Failed to unmap port bindings after error") } } }() bindingReqs := n.sortAndNormPBs(ctx, ep, cfg, defHostIP, pbmReq) - proxyPath := n.userlandProxyPath() - pdc := n.getPortDriverClient() - // toBind accumulates port bindings that should be allocated the same host port // (if required by NAT config). If the host address is unspecified, and defHostIP // is 0.0.0.0, one iteration of the loop may generate bindings for v4 and v6. If @@ -76,52 +69,30 @@ func (n *bridgeNetwork) addPortMappings( var toBind []portmapperapi.PortBindingReq for i, c := range bindingReqs { toBind = append(toBind, c) - if i < len(bindingReqs)-1 && c.DisableNAT == bindingReqs[i+1].DisableNAT && needSamePort(c, bindingReqs[i+1]) { + if i < len(bindingReqs)-1 && c.Mapper == bindingReqs[i+1].Mapper && needSamePort(c, bindingReqs[i+1]) { // This port binding matches the next, apart from host IP. So, continue // collecting bindings, then allocate the same host port for all addresses. continue } - var newB []portmapperapi.PortBinding - var err error - if c.DisableNAT { - newB, err = setupForwardedPorts(ctx, toBind, n.firewallerNetwork) - } else { - newB, err = bindHostPorts(ctx, toBind, proxyPath, pdc, n.firewallerNetwork) - } + pm, err := pms.Get(c.Mapper) if err != nil { return nil, err } - bindings = append(bindings, newB...) + + newB, err := pm.MapPorts(ctx, toBind, n.firewallerNetwork) + if err != nil { + return nil, err + } + bindings = append(bindings, sliceutil.Map(newB, func(b portmapperapi.PortBinding) portmapperapi.PortBinding { + b.Mapper = c.Mapper + return b + })...) // Reset toBind now the ports are bound. toBind = toBind[:0] } - // Start userland proxy processes. - if proxyPath != "" { - for i := range bindings { - if bindings[i].BoundSocket == nil || bindings[i].RootlesskitUnsupported || bindings[i].StopProxy != nil { - continue - } - var err error - bindings[i].StopProxy, err = startProxy( - bindings[i].ChildPortBinding(), proxyPath, bindings[i].BoundSocket, - ) - if err != nil { - return nil, fmt.Errorf("failed to start userland proxy for port mapping %s: %w", - bindings[i].PortBinding, err) - } - if err := bindings[i].BoundSocket.Close(); err != nil { - log.G(ctx).WithFields(log.Fields{ - "error": err, - "mapping": bindings[i].PortBinding, - }).Warnf("failed to close proxy socket") - } - bindings[i].BoundSocket = nil - } - } - return bindings, nil } @@ -283,7 +254,10 @@ func configurePortBindingIPv4( // Unmap the addresses if they're IPv4-mapped IPv6. bnd.HostIP = bnd.HostIP.To4() bnd.IP = containerIPv4.To4() - bnd.DisableNAT = disableNAT + bnd.Mapper = "nat" + if disableNAT { + bnd.Mapper = "routed" + } return bnd, true } @@ -340,236 +314,13 @@ func configurePortBindingIPv6( } bnd.IP = containerIP - bnd.DisableNAT = disableNAT + bnd.Mapper = "nat" + if disableNAT { + bnd.Mapper = "routed" + } return bnd, true } -func setChildHostIP(pdc portDriverClient, req portmapperapi.PortBindingReq) portmapperapi.PortBindingReq { - if pdc == nil { - req.ChildHostIP = req.HostIP - return req - } - hip, _ := netip.AddrFromSlice(req.HostIP) - req.ChildHostIP = pdc.ChildHostIP(hip).AsSlice() - return req -} - -// setupForwardedPorts sets up firewall rules to allow direct remote access to -// the container's ports in cfg. -func setupForwardedPorts(ctx context.Context, cfg []portmapperapi.PortBindingReq, fwn firewaller.Network) ([]portmapperapi.PortBinding, error) { - if len(cfg) == 0 { - return nil, nil - } - - res := make([]portmapperapi.PortBinding, 0, len(cfg)) - bindings := make([]types.PortBinding, 0, len(cfg)) - for _, c := range cfg { - pb := portmapperapi.PortBinding{PortBinding: c.GetCopy()} - if pb.HostPort != 0 || pb.HostPortEnd != 0 { - log.G(ctx).WithFields(log.Fields{"mapping": pb}).Infof( - "Host port ignored, because NAT is disabled") - pb.HostPort = 0 - pb.HostPortEnd = 0 - } - res = append(res, pb) - bindings = append(bindings, pb.PortBinding) - } - - if err := fwn.AddPorts(ctx, bindings); err != nil { - return nil, err - } - - return res, nil -} - -// bindHostPorts allocates and binds host ports for the given cfg. The -// caller is responsible for ensuring that all entries in cfg map the same proto, -// container port, and host port range (their host addresses must differ). -func bindHostPorts( - ctx context.Context, - cfg []portmapperapi.PortBindingReq, - proxyPath string, - pdc portDriverClient, - fwn firewaller.Network, -) ([]portmapperapi.PortBinding, error) { - if len(cfg) == 0 { - return nil, nil - } - // Ensure that all of cfg's entries have the same proto and ports. - proto, port, hostPort, hostPortEnd := cfg[0].Proto, cfg[0].Port, cfg[0].HostPort, cfg[0].HostPortEnd - for _, c := range cfg[1:] { - if c.Proto != proto || c.Port != port || c.HostPort != hostPort || c.HostPortEnd != hostPortEnd { - return nil, types.InternalErrorf("port binding mismatch %d/%s:%d-%d, %d/%s:%d-%d", - port, proto, hostPort, hostPortEnd, - port, c.Proto, c.HostPort, c.HostPortEnd) - } - } - - // Try up to maxAllocatePortAttempts times to get a port that's not already allocated. - var err error - for i := 0; i < maxAllocatePortAttempts; i++ { - var b []portmapperapi.PortBinding - b, err = attemptBindHostPorts(ctx, cfg, proto, hostPort, hostPortEnd, proxyPath, pdc, fwn) - if err == nil { - return b, nil - } - // There is no point in immediately retrying to map an explicitly chosen port. - if hostPort != 0 && hostPort == hostPortEnd { - log.G(ctx).WithError(err).Warnf("Failed to allocate and map port") - break - } - log.G(ctx).WithFields(log.Fields{ - "error": err, - "attempt": i + 1, - }).Warn("Failed to allocate and map port") - } - return nil, err -} - -// attemptBindHostPorts allocates host ports for each NAT port mapping, and -// reserves those ports by binding them. -// -// If the allocator doesn't have an available port in the required range, or the -// port can't be bound (perhaps because another process has already bound it), -// all resources are released and an error is returned. When ports are -// successfully reserved, a PortBinding is returned for each mapping. -func attemptBindHostPorts( - ctx context.Context, - cfg []portmapperapi.PortBindingReq, - proto types.Protocol, - hostPortStart, hostPortEnd uint16, - proxyPath string, - pdc portDriverClient, - fwn firewaller.Network, -) (_ []portmapperapi.PortBinding, retErr error) { - var err error - var port int - - addrs := make([]net.IP, 0, len(cfg)) - for i := range cfg { - cfg[i] = setChildHostIP(pdc, cfg[i]) - addrs = append(addrs, cfg[i].ChildHostIP) - } - - pa := portallocator.NewOSAllocator() - port, socks, err := pa.RequestPortsInRange(addrs, proto, int(hostPortStart), int(hostPortEnd)) - if err != nil { - return nil, err - } - defer func() { - if retErr != nil { - pa.ReleasePorts(addrs, proto, port) - } - }() - - if len(socks) != len(cfg) { - for _, sock := range socks { - if err := sock.Close(); err != nil { - log.G(ctx).WithError(err).Warn("Failed to close socket") - } - } - return nil, types.InternalErrorf("port allocator returned %d sockets for %d port bindings", len(socks), len(cfg)) - } - - res := make([]portmapperapi.PortBinding, 0, len(cfg)) - defer func() { - if retErr != nil { - if err := releasePortBindings(res, fwn); err != nil { - log.G(ctx).WithError(err).Warn("Failed to release port bindings") - } - } - }() - - for i := range cfg { - pb := portmapperapi.PortBinding{ - PortBinding: cfg[i].PortBinding.GetCopy(), - BoundSocket: socks[i], - ChildHostIP: cfg[i].ChildHostIP, - } - pb.PortBinding.HostPort = uint16(port) - pb.PortBinding.HostPortEnd = pb.HostPort - res = append(res, pb) - } - - if err := configPortDriver(ctx, res, pdc); err != nil { - return nil, err - } - if err := fwn.AddPorts(ctx, mergeChildHostIPs(res)); err != nil { - return nil, err - } - // Now the firewall rules are set up, it's safe to listen on the socket. (Listening - // earlier could result in dropped connections if the proxy becomes unreachable due - // to NAT rules sending packets directly to the container.) - // - // If not starting the proxy, nothing will ever accept a connection on the - // socket. Listen here anyway because SO_REUSEADDR is set, so bind() won't notice - // the problem if a port's bound to both INADDR_ANY and a specific address. (Also - // so the binding shows up in "netstat -at".) - if err := listenBoundPorts(res, proxyPath); err != nil { - return nil, err - } - return res, nil -} - -// configPortDriver passes the port binding's details to rootlesskit, and updates the -// port binding with callbacks to remove the rootlesskit config (or marks the binding as -// unsupported by rootlesskit). -func configPortDriver(ctx context.Context, pbs []portmapperapi.PortBinding, pdc portDriverClient) error { - for i := range pbs { - b := pbs[i] - if pdc != nil && b.HostPort != 0 { - var err error - hip, ok := netip.AddrFromSlice(b.HostIP) - if !ok { - return fmt.Errorf("invalid host IP address in %s", b) - } - chip, ok := netip.AddrFromSlice(b.ChildHostIP) - if !ok { - return fmt.Errorf("invalid child host IP address %s in %s", b.ChildHostIP, b) - } - pbs[i].PortDriverRemove, err = pdc.AddPort(ctx, b.Proto.String(), hip, chip, int(b.HostPort)) - if err != nil { - var pErr *rlkclient.ProtocolUnsupportedError - if errors.As(err, &pErr) { - log.G(ctx).WithFields(log.Fields{ - "error": pErr, - }).Warnf("discarding request for %q", net.JoinHostPort(hip.String(), strconv.Itoa(int(b.HostPort)))) - pbs[i].RootlesskitUnsupported = true - continue - } - return err - } - } - } - return nil -} - -func listenBoundPorts(pbs []portmapperapi.PortBinding, proxyPath string) error { - for i := range pbs { - if pbs[i].BoundSocket == nil || pbs[i].RootlesskitUnsupported || pbs[i].Proto == types.UDP { - continue - } - rc, err := pbs[i].BoundSocket.SyscallConn() - if err != nil { - return fmt.Errorf("raw conn not available on %s socket: %w", pbs[i].Proto, err) - } - if errC := rc.Control(func(fd uintptr) { - somaxconn := 0 - // SCTP sockets do not support somaxconn=0 - if proxyPath != "" || pbs[i].Proto == types.SCTP { - somaxconn = -1 // silently capped to "/proc/sys/net/core/somaxconn" - } - err = syscall.Listen(int(fd), somaxconn) - }); errC != nil { - return fmt.Errorf("failed to Control %s socket: %w", pbs[i].Proto, err) - } - if err != nil { - return fmt.Errorf("failed to listen on %s socket: %w", pbs[i].Proto, err) - } - } - return nil -} - // releasePorts attempts to release all port bindings, does not stop on failure func (n *bridgeNetwork) releasePorts(ep *bridgeEndpoint) error { n.Lock() @@ -578,36 +329,25 @@ func (n *bridgeNetwork) releasePorts(ep *bridgeEndpoint) error { ep.portBindingState = portBindingMode{} n.Unlock() - return releasePortBindings(pbs, n.firewallerNetwork) + return n.unmapPBs(context.TODO(), pbs) } -func releasePortBindings(pbs []portmapperapi.PortBinding, fwn firewaller.Network) error { +func (n *bridgeNetwork) unmapPBs(ctx context.Context, bindings []portmapperapi.PortBinding) error { + pms := n.portMappers() + var errs []error - for _, pb := range pbs { - if pb.BoundSocket != nil { - if err := pb.BoundSocket.Close(); err != nil { - errs = append(errs, fmt.Errorf("failed to close socket for port mapping %s: %w", pb, err)) - } + for _, b := range bindings { + pm, err := pms.Get(b.Mapper) + if err != nil { + errs = append(errs, fmt.Errorf("unmapping port binding %s: %w", b.PortBinding, err)) + continue } - if pb.PortDriverRemove != nil { - if err := pb.PortDriverRemove(); err != nil { - errs = append(errs, err) - } - } - if pb.StopProxy != nil { - if err := pb.StopProxy(); err != nil && !errors.Is(err, os.ErrProcessDone) { - errs = append(errs, fmt.Errorf("failed to stop userland proxy for port mapping %s: %w", pb, err)) - } - } - } - if err := fwn.DelPorts(context.TODO(), mergeChildHostIPs(pbs)); err != nil { - errs = append(errs, err) - } - for _, pb := range pbs { - if pb.HostPort > 0 { - portallocator.Get().ReleasePort(pb.ChildHostIP, pb.Proto.String(), int(pb.HostPort)) + + if err := pm.UnmapPorts(ctx, []portmapperapi.PortBinding{b}, n.firewallerNetwork); err != nil { + errs = append(errs, fmt.Errorf("unmapping port binding %s: %w", b.PortBinding, err)) } } + return errors.Join(errs...) } diff --git a/daemon/libnetwork/drivers/bridge/port_mapping_linux_test.go b/daemon/libnetwork/drivers/bridge/port_mapping_linux_test.go index bf0d1133e7..6f5ef8b2b7 100644 --- a/daemon/libnetwork/drivers/bridge/port_mapping_linux_test.go +++ b/daemon/libnetwork/drivers/bridge/port_mapping_linux_test.go @@ -1,3 +1,6 @@ +// FIXME(thaJeztah): remove once we are a module; the go:build directive prevents go from downgrading language version to go1.16: +//go:build go1.23 + package bridge import ( @@ -7,6 +10,7 @@ import ( "net" "net/netip" "os" + "slices" "strconv" "strings" "syscall" @@ -19,7 +23,10 @@ import ( "github.com/docker/docker/daemon/libnetwork/ns" "github.com/docker/docker/daemon/libnetwork/portallocator" "github.com/docker/docker/daemon/libnetwork/portmapperapi" + "github.com/docker/docker/daemon/libnetwork/portmappers/nat" + "github.com/docker/docker/daemon/libnetwork/portmappers/routed" "github.com/docker/docker/daemon/libnetwork/types" + "github.com/docker/docker/internal/sliceutil" "github.com/docker/docker/internal/testutils/netnsutils" "github.com/docker/docker/internal/testutils/storeutils" "github.com/sirupsen/logrus" @@ -32,7 +39,12 @@ func TestPortMappingConfig(t *testing.T) { defer netnsutils.SetupTestOSContext(t)() useStubFirewaller(t) - d := newDriver(storeutils.NewTempStore(t), &drvregistry.PortMappers{}) + pms := drvregistry.PortMappers{} + pm := &stubPortMapper{} + err := pms.Register("nat", pm) + assert.NilError(t, err) + + d := newDriver(storeutils.NewTempStore(t), &pms) config := &configuration{ EnableIPTables: true, @@ -61,7 +73,7 @@ func TestPortMappingConfig(t *testing.T) { } ipdList4 := getIPv4Data(t) - err := d.CreateNetwork(context.Background(), "dummy", netOptions, nil, ipdList4, getIPv6Data(t)) + err = d.CreateNetwork(context.Background(), "dummy", netOptions, nil, ipdList4, getIPv6Data(t)) if err != nil { t.Fatalf("Failed to create bridge: %v", err) } @@ -117,7 +129,12 @@ func TestPortMappingV6Config(t *testing.T) { t.Fatalf("Could not bring loopback iface up: %v", err) } - d := newDriver(storeutils.NewTempStore(t), &drvregistry.PortMappers{}) + pms := drvregistry.PortMappers{} + pm := &stubPortMapper{} + err := pms.Register("nat", pm) + assert.NilError(t, err) + + d := newDriver(storeutils.NewTempStore(t), &pms) config := &configuration{ EnableIPTables: true, @@ -147,7 +164,7 @@ func TestPortMappingV6Config(t *testing.T) { ipdList4 := getIPv4Data(t) ipdList6 := getIPv6Data(t) - err := d.CreateNetwork(context.Background(), "dummy", netOptions, nil, ipdList4, ipdList6) + err = d.CreateNetwork(context.Background(), "dummy", netOptions, nil, ipdList4, ipdList6) if err != nil { t.Fatalf("Failed to create bridge: %v", err) } @@ -196,30 +213,6 @@ func loopbackUp() error { return nlHandle.LinkSetUp(iface) } -func TestBindHostPortsError(t *testing.T) { - cfg := []portmapperapi.PortBindingReq{ - { - PortBinding: types.PortBinding{ - Proto: types.TCP, - Port: 80, - HostPort: 8080, - HostPortEnd: 8080, - }, - }, - { - PortBinding: types.PortBinding{ - Proto: types.TCP, - Port: 80, - HostPort: 8080, - HostPortEnd: 8081, - }, - }, - } - pbs, err := bindHostPorts(context.Background(), cfg, "", nil, nil) - assert.Check(t, is.Error(err, "port binding mismatch 80/tcp:8080-8080, 80/tcp:8080-8081")) - assert.Check(t, is.Nil(pbs)) -} - func newIPNet(t *testing.T, cidr string) *net.IPNet { t.Helper() ip, ipNet, err := net.ParseCIDR(cidr) @@ -242,7 +235,7 @@ func TestAddPortMappings(t *testing.T) { gwMode6 gwMode cfg []portmapperapi.PortBindingReq defHostIP net.IP - proxyPath string + enableProxy bool hairpin bool busyPortIPv4 int rootless bool @@ -267,7 +260,7 @@ func TestAddPortMappings(t *testing.T) { {PortBinding: types.PortBinding{Proto: types.TCP, Port: 22}}, {PortBinding: types.PortBinding{Proto: types.TCP, Port: 80}}, }, - proxyPath: "/dummy/path/to/proxy", + enableProxy: true, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 22, HostIP: net.IPv4zero, HostPort: firstEphemPort}, {Proto: types.TCP, IP: ctrIP6.IP, Port: 22, HostIP: net.IPv6zero, HostPort: firstEphemPort}, @@ -276,24 +269,24 @@ func TestAddPortMappings(t *testing.T) { }, }, { - name: "specific host port", - epAddrV4: ctrIP4, - epAddrV6: ctrIP6, - cfg: []portmapperapi.PortBindingReq{{PortBinding: types.PortBinding{Proto: types.TCP, Port: 80, HostPort: 8080}}}, - proxyPath: "/dummy/path/to/proxy", + name: "specific host port", + epAddrV4: ctrIP4, + epAddrV6: ctrIP6, + cfg: []portmapperapi.PortBindingReq{{PortBinding: types.PortBinding{Proto: types.TCP, Port: 80, HostPort: 8080}}}, + enableProxy: true, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: net.IPv4zero, HostPort: 8080, HostPortEnd: 8080}, {Proto: types.TCP, IP: ctrIP6.IP, Port: 80, HostIP: net.IPv6zero, HostPort: 8080, HostPortEnd: 8080}, }, }, { - name: "nat explicitly enabled", - epAddrV4: ctrIP4, - epAddrV6: ctrIP6, - cfg: []portmapperapi.PortBindingReq{{PortBinding: types.PortBinding{Proto: types.TCP, Port: 80, HostPort: 8080}}}, - gwMode4: gwModeNAT, - gwMode6: gwModeNAT, - proxyPath: "/dummy/path/to/proxy", + name: "nat explicitly enabled", + epAddrV4: ctrIP4, + epAddrV6: ctrIP6, + cfg: []portmapperapi.PortBindingReq{{PortBinding: types.PortBinding{Proto: types.TCP, Port: 80, HostPort: 8080}}}, + gwMode4: gwModeNAT, + gwMode6: gwModeNAT, + enableProxy: true, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: net.IPv4zero, HostPort: 8080, HostPortEnd: 8080}, {Proto: types.TCP, IP: ctrIP6.IP, Port: 80, HostIP: net.IPv6zero, HostPort: 8080, HostPortEnd: 8080}, @@ -304,27 +297,27 @@ func TestAddPortMappings(t *testing.T) { epAddrV4: ctrIP4, epAddrV6: ctrIP6, cfg: []portmapperapi.PortBindingReq{{PortBinding: types.PortBinding{Proto: types.TCP, Port: 80, HostPort: 8080}}}, - proxyPath: "/dummy/path/to/proxy", + enableProxy: true, busyPortIPv4: 8080, expErr: "failed to bind host port 0.0.0.0:8080/tcp: address already in use", }, { - name: "ipv4 mapped container address with specific host port", - epAddrV4: ctrIP4Mapped, - epAddrV6: ctrIP6, - cfg: []portmapperapi.PortBindingReq{{PortBinding: types.PortBinding{Proto: types.TCP, Port: 80, HostPort: 8080}}}, - proxyPath: "/dummy/path/to/proxy", + name: "ipv4 mapped container address with specific host port", + epAddrV4: ctrIP4Mapped, + epAddrV6: ctrIP6, + cfg: []portmapperapi.PortBindingReq{{PortBinding: types.PortBinding{Proto: types.TCP, Port: 80, HostPort: 8080}}}, + enableProxy: true, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: net.IPv4zero, HostPort: 8080, HostPortEnd: 8080}, {Proto: types.TCP, IP: ctrIP6.IP, Port: 80, HostIP: net.IPv6zero, HostPort: 8080, HostPortEnd: 8080}, }, }, { - name: "ipv4 mapped host address with specific host port", - epAddrV4: ctrIP4, - epAddrV6: ctrIP6, - cfg: []portmapperapi.PortBindingReq{{PortBinding: types.PortBinding{Proto: types.TCP, Port: 80, HostIP: newIPNet(t, "::ffff:127.0.0.1/128").IP, HostPort: 8080}}}, - proxyPath: "/dummy/path/to/proxy", + name: "ipv4 mapped host address with specific host port", + epAddrV4: ctrIP4, + epAddrV6: ctrIP6, + cfg: []portmapperapi.PortBindingReq{{PortBinding: types.PortBinding{Proto: types.TCP, Port: 80, HostIP: newIPNet(t, "::ffff:127.0.0.1/128").IP, HostPort: 8080}}}, + enableProxy: true, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: newIPNet(t, "127.0.0.1/32").IP, HostPort: 8080, HostPortEnd: 8080}, }, @@ -334,7 +327,7 @@ func TestAddPortMappings(t *testing.T) { epAddrV4: ctrIP4, epAddrV6: ctrIP6, cfg: []portmapperapi.PortBindingReq{{PortBinding: types.PortBinding{Proto: types.TCP, Port: 80, HostPort: 8080, HostPortEnd: 8081}}}, - proxyPath: "/dummy/path/to/proxy", + enableProxy: true, busyPortIPv4: 8080, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: net.IPv4zero, HostPort: 8081, HostPortEnd: 8081}, @@ -349,7 +342,7 @@ func TestAddPortMappings(t *testing.T) { {PortBinding: types.PortBinding{Proto: types.TCP, Port: 80, HostIP: net.IPv4zero, HostPort: 8080, HostPortEnd: 8081}}, {PortBinding: types.PortBinding{Proto: types.TCP, Port: 80, HostIP: net.IPv6zero, HostPort: 8080, HostPortEnd: 8081}}, }, - proxyPath: "/dummy/path/to/proxy", + enableProxy: true, busyPortIPv4: 8080, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: net.IPv4zero, HostPort: 8081}, @@ -368,7 +361,7 @@ func TestAddPortMappings(t *testing.T) { {PortBinding: types.PortBinding{Proto: types.UDP, Port: 81, HostPort: 8080, HostPortEnd: 8083}}, {PortBinding: types.PortBinding{Proto: types.UDP, Port: 82, HostPort: 8080, HostPortEnd: 8083}}, }, - proxyPath: "/dummy/path/to/proxy", + enableProxy: true, busyPortIPv4: 8082, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: net.IPv4zero, HostPort: 8080, HostPortEnd: 8080}, @@ -394,7 +387,7 @@ func TestAddPortMappings(t *testing.T) { {PortBinding: types.PortBinding{Proto: types.TCP, Port: 81, HostPort: 8080, HostPortEnd: 8082}}, {PortBinding: types.PortBinding{Proto: types.TCP, Port: 82, HostPort: 8080, HostPortEnd: 8082}}, }, - proxyPath: "/dummy/path/to/proxy", + enableProxy: true, busyPortIPv4: 8081, expErr: "failed to bind host port 0.0.0.0:8081", }, @@ -405,7 +398,7 @@ func TestAddPortMappings(t *testing.T) { {PortBinding: types.PortBinding{Proto: types.TCP, HostIP: net.IPv4zero, Port: 80}}, {PortBinding: types.PortBinding{Proto: types.TCP, HostIP: net.IPv6zero, Port: 80}}, }, - proxyPath: "/dummy/path/to/proxy", + enableProxy: true, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: net.IPv4zero, HostPort: firstEphemPort}, {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: net.IPv6zero, HostPort: firstEphemPort}, @@ -417,7 +410,7 @@ func TestAddPortMappings(t *testing.T) { cfg: []portmapperapi.PortBindingReq{ {PortBinding: types.PortBinding{Proto: types.TCP, Port: 80}}, }, - proxyPath: "/dummy/path/to/proxy", + enableProxy: true, noProxy6To4: true, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: net.IPv4zero, HostPort: firstEphemPort}, @@ -436,45 +429,45 @@ func TestAddPortMappings(t *testing.T) { }, }, { - name: "default host ip is nonzero v4", - epAddrV4: ctrIP4, - epAddrV6: ctrIP6, - cfg: []portmapperapi.PortBindingReq{{PortBinding: types.PortBinding{Proto: types.TCP, Port: 80}}}, - proxyPath: "/dummy/path/to/proxy", - defHostIP: newIPNet(t, "127.0.0.1/8").IP, + name: "default host ip is nonzero v4", + epAddrV4: ctrIP4, + epAddrV6: ctrIP6, + cfg: []portmapperapi.PortBindingReq{{PortBinding: types.PortBinding{Proto: types.TCP, Port: 80}}}, + enableProxy: true, + defHostIP: newIPNet(t, "127.0.0.1/8").IP, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: newIPNet(t, "127.0.0.1/8").IP, HostPort: firstEphemPort}, }, }, { - name: "default host ip is nonzero IPv4-mapped IPv6", - epAddrV4: ctrIP4, - epAddrV6: ctrIP6, - cfg: []portmapperapi.PortBindingReq{{PortBinding: types.PortBinding{Proto: types.TCP, Port: 80}}}, - proxyPath: "/dummy/path/to/proxy", - defHostIP: newIPNet(t, "::ffff:127.0.0.1/72").IP, + name: "default host ip is nonzero IPv4-mapped IPv6", + epAddrV4: ctrIP4, + epAddrV6: ctrIP6, + cfg: []portmapperapi.PortBindingReq{{PortBinding: types.PortBinding{Proto: types.TCP, Port: 80}}}, + enableProxy: true, + defHostIP: newIPNet(t, "::ffff:127.0.0.1/72").IP, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: newIPNet(t, "127.0.0.1/8").IP, HostPort: firstEphemPort}, }, }, { - name: "default host ip is v6", - epAddrV4: ctrIP4, - epAddrV6: ctrIP6, - cfg: []portmapperapi.PortBindingReq{{PortBinding: types.PortBinding{Proto: types.TCP, Port: 80}}}, - proxyPath: "/dummy/path/to/proxy", - defHostIP: net.IPv6zero, + name: "default host ip is v6", + epAddrV4: ctrIP4, + epAddrV6: ctrIP6, + cfg: []portmapperapi.PortBindingReq{{PortBinding: types.PortBinding{Proto: types.TCP, Port: 80}}}, + enableProxy: true, + defHostIP: net.IPv6zero, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP6.IP, Port: 80, HostIP: net.IPv6zero, HostPort: firstEphemPort}, }, }, { - name: "default host ip is nonzero v6", - epAddrV4: ctrIP4, - epAddrV6: ctrIP6, - cfg: []portmapperapi.PortBindingReq{{PortBinding: types.PortBinding{Proto: types.TCP, Port: 80}}}, - proxyPath: "/dummy/path/to/proxy", - defHostIP: newIPNet(t, "::1/128").IP, + name: "default host ip is nonzero v6", + epAddrV4: ctrIP4, + epAddrV6: ctrIP6, + cfg: []portmapperapi.PortBindingReq{{PortBinding: types.PortBinding{Proto: types.TCP, Port: 80}}}, + enableProxy: true, + defHostIP: newIPNet(t, "::1/128").IP, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP6.IP, Port: 80, HostIP: newIPNet(t, "::1/128").IP, HostPort: firstEphemPort}, }, @@ -487,17 +480,17 @@ func TestAddPortMappings(t *testing.T) { {PortBinding: types.PortBinding{Proto: types.TCP, Port: 80, HostPort: 8080}}, {PortBinding: types.PortBinding{Proto: types.TCP, Port: 22, HostPort: 2222}}, }, - proxyPath: "/dummy/path/to/proxy", + enableProxy: true, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 22, HostIP: net.IPv4zero, HostPort: 2222}, {Proto: types.TCP, IP: ctrIP6.IP, Port: 22, HostIP: net.IPv6zero, HostPort: 2222}, {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: net.IPv4zero, HostPort: 8080}, {Proto: types.TCP, IP: ctrIP6.IP, Port: 80, HostIP: net.IPv6zero, HostPort: 8080}, }, - expReleaseErr: "failed to stop userland proxy for port mapping 0.0.0.0:2222:172.19.0.2:22/tcp: can't stop now\n" + - "failed to stop userland proxy for port mapping [::]:2222:[fdf8:b88e:bb5c:3483::2]:22/tcp: can't stop now\n" + - "failed to stop userland proxy for port mapping 0.0.0.0:8080:172.19.0.2:80/tcp: can't stop now\n" + - "failed to stop userland proxy for port mapping [::]:8080:[fdf8:b88e:bb5c:3483::2]:80/tcp: can't stop now", + expReleaseErr: "unmapping port binding 0.0.0.0:2222:172.19.0.2:22/tcp: failed to stop userland proxy: can't stop now\n" + + "unmapping port binding [::]:2222:[fdf8:b88e:bb5c:3483::2]:22/tcp: failed to stop userland proxy: can't stop now\n" + + "unmapping port binding 0.0.0.0:8080:172.19.0.2:80/tcp: failed to stop userland proxy: can't stop now\n" + + "unmapping port binding [::]:8080:[fdf8:b88e:bb5c:3483::2]:80/tcp: failed to stop userland proxy: can't stop now", }, { name: "disable nat6", @@ -507,8 +500,8 @@ func TestAddPortMappings(t *testing.T) { {PortBinding: types.PortBinding{Proto: types.TCP, Port: 22}}, {PortBinding: types.PortBinding{Proto: types.TCP, Port: 80}}, }, - proxyPath: "/dummy/path/to/proxy", - gwMode6: gwModeRouted, + enableProxy: true, + gwMode6: gwModeRouted, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 22, HostIP: net.IPv4zero, HostPort: firstEphemPort}, {Proto: types.TCP, IP: ctrIP6.IP, Port: 22, HostIP: net.IPv6zero}, @@ -524,9 +517,9 @@ func TestAddPortMappings(t *testing.T) { {PortBinding: types.PortBinding{Proto: types.TCP, Port: 22}}, {PortBinding: types.PortBinding{Proto: types.TCP, Port: 80}}, }, - proxyPath: "/dummy/path/to/proxy", - gwMode6: gwModeRouted, - defHostIP: net.IPv6loopback, + enableProxy: true, + gwMode6: gwModeRouted, + defHostIP: net.IPv6loopback, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP6.IP, Port: 22, HostIP: net.IPv6zero}, {Proto: types.TCP, IP: ctrIP6.IP, Port: 80, HostIP: net.IPv6zero}, @@ -540,8 +533,8 @@ func TestAddPortMappings(t *testing.T) { {PortBinding: types.PortBinding{Proto: types.TCP, Port: 22}}, {PortBinding: types.PortBinding{Proto: types.TCP, Port: 80}}, }, - proxyPath: "/dummy/path/to/proxy", - gwMode4: gwModeRouted, + enableProxy: true, + gwMode4: gwModeRouted, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 22, HostIP: net.IPv4zero}, {Proto: types.TCP, IP: ctrIP6.IP, Port: 22, HostIP: net.IPv6zero, HostPort: firstEphemPort}, @@ -557,9 +550,9 @@ func TestAddPortMappings(t *testing.T) { {PortBinding: types.PortBinding{Proto: types.TCP, Port: 22}}, {PortBinding: types.PortBinding{Proto: types.TCP, Port: 80}}, }, - proxyPath: "/dummy/path/to/proxy", - gwMode4: gwModeRouted, - gwMode6: gwModeRouted, + enableProxy: true, + gwMode4: gwModeRouted, + gwMode6: gwModeRouted, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 22, HostIP: net.IPv4zero}, {Proto: types.TCP, IP: ctrIP6.IP, Port: 22, HostIP: net.IPv6zero}, @@ -587,12 +580,12 @@ func TestAddPortMappings(t *testing.T) { expLogs: []string{"Cannot map from default host binding address to an IPv4-only container because the userland proxy is disabled"}, }, { - name: "routed mode specific address", - epAddrV4: ctrIP4, - epAddrV6: ctrIP6, - gwMode4: gwModeRouted, - gwMode6: gwModeRouted, - proxyPath: "/dummy/path/to/proxy", + name: "routed mode specific address", + epAddrV4: ctrIP4, + epAddrV6: ctrIP6, + gwMode4: gwModeRouted, + gwMode6: gwModeRouted, + enableProxy: true, cfg: []portmapperapi.PortBindingReq{ {PortBinding: types.PortBinding{Proto: types.TCP, Port: 22, HostIP: newIPNet(t, "127.0.0.1/8").IP}}, {PortBinding: types.PortBinding{Proto: types.TCP, Port: 22, HostIP: net.IPv6loopback}}, @@ -607,12 +600,12 @@ func TestAddPortMappings(t *testing.T) { }, }, { - name: "routed4 nat6 with ipv4 default binding", - epAddrV4: ctrIP4, - epAddrV6: ctrIP6, - gwMode4: gwModeRouted, - defHostIP: newIPNet(t, "127.0.0.1/8").IP, - proxyPath: "/dummy/path/to/proxy", + name: "routed4 nat6 with ipv4 default binding", + epAddrV4: ctrIP4, + epAddrV6: ctrIP6, + gwMode4: gwModeRouted, + defHostIP: newIPNet(t, "127.0.0.1/8").IP, + enableProxy: true, cfg: []portmapperapi.PortBindingReq{ {PortBinding: types.PortBinding{Proto: types.TCP, Port: 22}}, }, @@ -621,12 +614,12 @@ func TestAddPortMappings(t *testing.T) { }, }, { - name: "routed4 nat6 with ipv6 default binding", - epAddrV4: ctrIP4, - epAddrV6: ctrIP6, - gwMode4: gwModeRouted, - defHostIP: net.IPv6loopback, - proxyPath: "/dummy/path/to/proxy", + name: "routed4 nat6 with ipv6 default binding", + epAddrV4: ctrIP4, + epAddrV6: ctrIP6, + gwMode4: gwModeRouted, + defHostIP: net.IPv6loopback, + enableProxy: true, cfg: []portmapperapi.PortBindingReq{ {PortBinding: types.PortBinding{Proto: types.TCP, Port: 22}}, }, @@ -672,7 +665,7 @@ func TestAddPortMappings(t *testing.T) { {PortBinding: types.PortBinding{Proto: types.TCP, Port: 12345, HostPort: 12345, HostPortEnd: 12346}}, {PortBinding: types.PortBinding{Proto: types.TCP, Port: 12345, HostPort: 12345}}, }, - proxyPath: "/dummy/path/to/proxy", + enableProxy: true, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 12345, HostIP: net.IPv4zero, HostPort: 12345}, {Proto: types.TCP, IP: ctrIP6.IP, Port: 12345, HostIP: net.IPv6zero, HostPort: 12345}, @@ -693,8 +686,8 @@ func TestAddPortMappings(t *testing.T) { {PortBinding: types.PortBinding{Proto: types.TCP, Port: 22}}, {PortBinding: types.PortBinding{Proto: types.TCP, Port: 80}}, }, - proxyPath: "/dummy/path/to/proxy", - rootless: true, + enableProxy: true, + rootless: true, expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 22, HostIP: net.IPv4zero, HostPort: firstEphemPort}, {Proto: types.TCP, IP: ctrIP6.IP, Port: 22, HostIP: net.IPv6zero, HostPort: firstEphemPort}, @@ -727,17 +720,12 @@ func TestAddPortMappings(t *testing.T) { useStubFirewaller(t) // Mock the startProxy function used by the code under test. - origStartProxy := startProxy - defer func() { startProxy = origStartProxy }() proxies := map[proxyCall]bool{} // proxy -> is not stopped - startProxy = func(pb types.PortBinding, - proxyPath string, - listenSock *os.File, - ) (stop func() error, retErr error) { + startProxy := func(pb types.PortBinding, listenSock *os.File) (stop func() error, retErr error) { if tc.busyPortIPv4 > 0 && tc.busyPortIPv4 == int(pb.HostPort) && pb.HostIP.To4() != nil { return nil, errors.New("busy port") } - c := newProxyCall(pb.Proto.String(), pb.HostIP, int(pb.HostPort), pb.IP, int(pb.Port), proxyPath) + c := newProxyCall(pb.Proto.String(), pb.HostIP, int(pb.HostPort), pb.IP, int(pb.Port)) if _, ok := proxies[c]; ok { return nil, fmt.Errorf("duplicate proxy: %#v", c) } @@ -754,13 +742,6 @@ func TestAddPortMappings(t *testing.T) { }, nil } - // Mock the RootlessKit port driver. - origNewPortDriverClient := newPortDriverClient - defer func() { newPortDriverClient = origNewPortDriverClient }() - newPortDriverClient = func(ctx context.Context) (portDriverClient, error) { - return newMockPortDriverClient(ctx) - } - if len(tc.hostAddrs) > 0 { dummyLink := &netlink.Bridge{LinkAttrs: netlink.LinkAttrs{Name: "br-dummy"}} err := netlink.LinkAdd(dummyLink) @@ -783,6 +764,21 @@ func TestAddPortMappings(t *testing.T) { defer ul.Close() } + var pdc nat.PortDriverClient + if tc.rootless { + pdc = newMockPortDriverClient() + } + + pms := &drvregistry.PortMappers{} + err := nat.Register(pms, nat.Config{ + RlkClient: pdc, + EnableProxy: tc.enableProxy, + StartProxy: startProxy, + }) + assert.NilError(t, err) + err = routed.Register(pms) + assert.NilError(t, err) + n := &bridgeNetwork{ config: &networkConfiguration{ BridgeName: "dummybridge", @@ -792,26 +788,22 @@ func TestAddPortMappings(t *testing.T) { GwModeIPv6: tc.gwMode6, }, bridge: &bridgeInterface{}, - driver: newDriver(storeutils.NewTempStore(t), &drvregistry.PortMappers{}), + driver: newDriver(storeutils.NewTempStore(t), pms), } genericOption := map[string]interface{}{ netlabel.GenericData: &configuration{ - EnableIPTables: true, - EnableIP6Tables: true, - EnableUserlandProxy: tc.proxyPath != "", - UserlandProxyPath: tc.proxyPath, - Hairpin: tc.hairpin, - Rootless: tc.rootless, + EnableIPTables: true, + EnableIP6Tables: true, + Hairpin: tc.hairpin, }, } - err := n.driver.configure(genericOption) + err = n.driver.configure(genericOption) assert.NilError(t, err) fwn, err := n.newFirewallerNetwork(context.Background()) assert.NilError(t, err) assert.Check(t, fwn != nil, "no firewaller network") n.firewallerNetwork = fwn - assert.Check(t, is.Equal(n.driver.portDriverClient == nil, !tc.rootless)) expChildIP := func(hostIP net.IP) net.IP { if !tc.rootless { return hostIP @@ -860,7 +852,7 @@ func TestAddPortMappings(t *testing.T) { assert.Assert(t, is.Len(pbs, len(tc.expPBs))) fw := n.driver.firewaller.(*firewaller.StubFirewaller) - assert.Check(t, is.Equal(fw.Hairpin, tc.proxyPath == "")) + assert.Check(t, is.Equal(fw.Hairpin, !tc.enableProxy)) assert.Check(t, fw.IPv4) assert.Check(t, fw.IPv6) @@ -903,7 +895,7 @@ func TestAddPortMappings(t *testing.T) { } // Check a docker-proxy was started and stopped for each expected port binding. - if tc.proxyPath != "" { + if tc.enableProxy { expProxies := map[proxyCall]bool{} for _, expPB := range tc.expPBs { hip := expChildIP(expPB.HostIP) @@ -913,7 +905,7 @@ func TestAddPortMappings(t *testing.T) { } p := newProxyCall(expPB.Proto.String(), hip, int(expPB.HostPort), - expPB.IP, int(expPB.Port), tc.proxyPath) + expPB.IP, int(expPB.Port)) expProxies[p] = tc.expReleaseErr != "" } assert.Check(t, is.DeepEqual(expProxies, proxies)) @@ -921,8 +913,8 @@ func TestAddPortMappings(t *testing.T) { // Check the port driver has seen the expected port mappings and no others, // and that they have all been closed. - if n.driver.portDriverClient != nil { - pdc := n.driver.portDriverClient.(*mockPortDriverClient) + if pdc != nil { + pdc := pdc.(*mockPortDriverClient) expPorts := map[mockPortDriverPort]bool{} for _, expPB := range tc.expPBs { if expPB.HostPort == 0 { @@ -943,18 +935,16 @@ func TestAddPortMappings(t *testing.T) { } // Type for tracking calls to StartProxy. -type proxyCall struct{ proto, host, container, proxyPath string } +type proxyCall struct{ proto, host, container string } func newProxyCall(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int, - proxyPath string, ) proxyCall { return proxyCall{ proto: proto, host: fmt.Sprintf("%v:%v", hostIP, hostPort), container: fmt.Sprintf("%v:%v", containerIP, containerPort), - proxyPath: proxyPath, } } @@ -975,10 +965,10 @@ type mockPortDriverClient struct { openPorts map[mockPortDriverPort]bool } -func newMockPortDriverClient(_ context.Context) (*mockPortDriverClient, error) { +func newMockPortDriverClient() *mockPortDriverClient { return &mockPortDriverClient{ openPorts: map[mockPortDriverPort]bool{}, - }, nil + } } func (c *mockPortDriverClient) ChildHostIP(hostIP netip.Addr) netip.Addr { @@ -1002,3 +992,33 @@ func (c *mockPortDriverClient) AddPort(_ context.Context, proto string, hostIP, return nil }, nil } + +type stubPortMapper struct { + reqs [][]portmapperapi.PortBindingReq + mapped []portmapperapi.PortBinding +} + +func (pm *stubPortMapper) MapPorts(_ context.Context, reqs []portmapperapi.PortBindingReq, _ portmapperapi.Firewaller) ([]portmapperapi.PortBinding, error) { + if len(reqs) == 0 { + return []portmapperapi.PortBinding{}, nil + } + pm.reqs = append(pm.reqs, reqs) + pbs := sliceutil.Map(reqs, func(req portmapperapi.PortBindingReq) portmapperapi.PortBinding { + return portmapperapi.PortBinding{PortBinding: req.PortBinding} + }) + pm.mapped = append(pm.mapped, pbs...) + return pbs, nil +} + +func (pm *stubPortMapper) UnmapPorts(_ context.Context, reqs []portmapperapi.PortBinding, _ portmapperapi.Firewaller) error { + for _, req := range reqs { + idx := slices.IndexFunc(pm.mapped, func(pb portmapperapi.PortBinding) bool { + return pb.Equal(&req.PortBinding) + }) + if idx == -1 { + return fmt.Errorf("stubPortMapper.UnmapPorts: pb doesn't exist %v", req) + } + pm.mapped = slices.Delete(pm.mapped, idx, idx) + } + return nil +} diff --git a/daemon/libnetwork/drivers_linux.go b/daemon/libnetwork/drivers_linux.go index 14af028864..b8f3df0a6b 100644 --- a/daemon/libnetwork/drivers_linux.go +++ b/daemon/libnetwork/drivers_linux.go @@ -3,6 +3,7 @@ package libnetwork import ( "context" "fmt" + "os" "github.com/docker/docker/daemon/libnetwork/config" "github.com/docker/docker/daemon/libnetwork/datastore" @@ -14,6 +15,11 @@ import ( "github.com/docker/docker/daemon/libnetwork/drivers/null" "github.com/docker/docker/daemon/libnetwork/drivers/overlay" "github.com/docker/docker/daemon/libnetwork/drvregistry" + "github.com/docker/docker/daemon/libnetwork/internal/rlkclient" + "github.com/docker/docker/daemon/libnetwork/portmapper" + "github.com/docker/docker/daemon/libnetwork/portmappers/nat" + "github.com/docker/docker/daemon/libnetwork/portmappers/routed" + "github.com/docker/docker/daemon/libnetwork/types" ) func registerNetworkDrivers(r driverapi.Registerer, store *datastore.Store, pms *drvregistry.PortMappers, driverConfig func(string) map[string]interface{}) error { @@ -45,5 +51,28 @@ func registerNetworkDrivers(r driverapi.Registerer, store *datastore.Store, pms } func registerPortMappers(ctx context.Context, r *drvregistry.PortMappers, cfg *config.Config) error { + var pdc *rlkclient.PortDriverClient + if cfg.Rootless { + var err error + pdc, err = rlkclient.NewPortDriverClient(ctx) + if err != nil { + return fmt.Errorf("failed to create port driver client: %w", err) + } + } + + if err := nat.Register(r, nat.Config{ + RlkClient: pdc, + StartProxy: func(pb types.PortBinding, file *os.File) (func() error, error) { + return portmapper.StartProxy(pb, cfg.UserlandProxyPath, file) + }, + EnableProxy: cfg.EnableUserlandProxy && cfg.UserlandProxyPath != "", + }); err != nil { + return fmt.Errorf("registering nat portmapper: %w", err) + } + + if err := routed.Register(r); err != nil { + return fmt.Errorf("registering routed portmapper: %w", err) + } + return nil } diff --git a/daemon/libnetwork/portmapperapi/api.go b/daemon/libnetwork/portmapperapi/api.go index 8de53d5649..85e2fb67a1 100644 --- a/daemon/libnetwork/portmapperapi/api.go +++ b/daemon/libnetwork/portmapperapi/api.go @@ -5,6 +5,7 @@ import ( "net" "net/netip" "os" + "strings" "github.com/docker/docker/daemon/libnetwork/types" ) @@ -36,14 +37,12 @@ type PortMapper interface { type PortBindingReq struct { types.PortBinding + // Mapper is the name of the port mapper used to process this PortBindingReq. + Mapper string // ChildHostIP is a temporary field used to pass the host IP address as // seen from the daemon. (It'll be removed once the portmapper API is // implemented). ChildHostIP net.IP `json:"-"` - // DisableNAT is a temporary field used to indicate whether the port is - // mapped on the host or not. (It'll be removed once the portmapper API is - // implemented). - DisableNAT bool `json:"-"` } // Compare defines an ordering over PortBindingReq such that bindings that @@ -58,11 +57,8 @@ type PortBindingReq struct { // - same host ports or ranges are adjacent, then // - ordered by container IP (then host IP, if set). func (pbReq PortBindingReq) Compare(other PortBindingReq) int { - if pbReq.DisableNAT != other.DisableNAT { - if pbReq.DisableNAT { - return 1 // NAT disabled bindings come last - } - return -1 + if pbReq.Mapper != other.Mapper { + return strings.Compare(pbReq.Mapper, other.Mapper) } // Exact host port < host port range. aIsRange := pbReq.HostPort == 0 || pbReq.HostPort != pbReq.HostPortEnd @@ -97,6 +93,8 @@ func (pbReq PortBindingReq) Compare(other PortBindingReq) int { type PortBinding struct { types.PortBinding + // Mapper is the name of the port mapper used to process this PortBinding. + Mapper string // BoundSocket is used to reserve a host port for the binding. If the // userland proxy is in-use, it's passed to the proxy when the proxy is // started, then it's closed and set to nil here. diff --git a/daemon/libnetwork/portmapperapi/api_test.go b/daemon/libnetwork/portmapperapi/api_test.go index 21894b8845..1b124224ce 100644 --- a/daemon/libnetwork/portmapperapi/api_test.go +++ b/daemon/libnetwork/portmapperapi/api_test.go @@ -24,7 +24,7 @@ func TestPortBindingReqsCompare(t *testing.T) { assert.Check(t, pb.Compare(pb) == 0) //nolint:gocritic // ignore "dupArg: suspicious method call with the same argument and receiver (gocritic)" pbA, pbB = pb, pb - pbB.DisableNAT = true + pbB.Mapper = "routed" assert.Check(t, pbA.Compare(pbB) < 0) assert.Check(t, pbB.Compare(pbA) > 0) diff --git a/daemon/libnetwork/portmappers/nat/mapper_linux.go b/daemon/libnetwork/portmappers/nat/mapper_linux.go new file mode 100644 index 0000000000..c17d84eeea --- /dev/null +++ b/daemon/libnetwork/portmappers/nat/mapper_linux.go @@ -0,0 +1,325 @@ +package nat + +import ( + "context" + "errors" + "fmt" + "net" + "net/netip" + "os" + "strconv" + "syscall" + + "github.com/containerd/log" + "github.com/docker/docker/daemon/libnetwork/internal/rlkclient" + "github.com/docker/docker/daemon/libnetwork/portallocator" + "github.com/docker/docker/daemon/libnetwork/portmapperapi" + "github.com/docker/docker/daemon/libnetwork/types" +) + +const ( + driverName = "nat" + maxAllocatePortAttempts = 10 +) + +type PortDriverClient interface { + ChildHostIP(hostIP netip.Addr) netip.Addr + AddPort(ctx context.Context, proto string, hostIP, childIP netip.Addr, hostPort int) (func() error, error) +} + +type proxyStarter func(types.PortBinding, *os.File) (func() error, error) + +// Register the "nat" port-mapper with libnetwork. +func Register(r portmapperapi.Registerer, cfg Config) error { + return r.Register(driverName, NewPortMapper(cfg)) +} + +type PortMapper struct { + // pdc is used to interact with rootlesskit port driver. + pdc PortDriverClient + startProxy proxyStarter + enableProxy bool +} + +type Config struct { + // RlkClient is called by MapPorts to determine the ChildHostIP and ask + // rootlesskit to map ports in its netns. + RlkClient PortDriverClient + StartProxy proxyStarter + EnableProxy bool +} + +func NewPortMapper(cfg Config) PortMapper { + return PortMapper{ + pdc: cfg.RlkClient, + startProxy: cfg.StartProxy, + enableProxy: cfg.EnableProxy, + } +} + +// MapPorts allocates and binds host ports for the given cfg. The caller is +// responsible for ensuring that all entries in cfg map the same proto, +// container port, and host port range (their host addresses must differ). +func (pm PortMapper) MapPorts(ctx context.Context, cfg []portmapperapi.PortBindingReq, fwn portmapperapi.Firewaller) ([]portmapperapi.PortBinding, error) { + if len(cfg) == 0 { + return nil, nil + } + // Ensure that all of cfg's entries have the same proto and ports. + proto, port, hostPort, hostPortEnd := cfg[0].Proto, cfg[0].Port, cfg[0].HostPort, cfg[0].HostPortEnd + for _, c := range cfg[1:] { + if c.Proto != proto || c.Port != port || c.HostPort != hostPort || c.HostPortEnd != hostPortEnd { + return nil, types.InternalErrorf("port binding mismatch %d/%s:%d-%d, %d/%s:%d-%d", + port, proto, hostPort, hostPortEnd, + port, c.Proto, c.HostPort, c.HostPortEnd) + } + } + + // Try up to maxAllocatePortAttempts times to get a port that's not already allocated. + var bindings []portmapperapi.PortBinding + var err error + for i := 0; i < maxAllocatePortAttempts; i++ { + bindings, err = pm.attemptBindHostPorts(ctx, cfg, proto, hostPort, hostPortEnd, fwn) + if err == nil { + break + } + // There is no point in immediately retrying to map an explicitly chosen port. + if hostPort != 0 && hostPort == hostPortEnd { + log.G(ctx).WithError(err).Warnf("Failed to allocate and map port") + return nil, err + } + log.G(ctx).WithFields(log.Fields{ + "error": err, + "attempt": i + 1, + }).Warn("Failed to allocate and map port") + } + + if err != nil { + // If the retry budget is exhausted and no free port could be found, return + // the latest error. + return nil, err + } + + // Start userland proxy processes. + if pm.enableProxy { + for i := range bindings { + if bindings[i].BoundSocket == nil || bindings[i].RootlesskitUnsupported || bindings[i].StopProxy != nil { + continue + } + var err error + bindings[i].StopProxy, err = pm.startProxy( + bindings[i].ChildPortBinding(), bindings[i].BoundSocket, + ) + if err != nil { + return nil, fmt.Errorf("failed to start userland proxy for port mapping %s: %w", + bindings[i].PortBinding, err) + } + if err := bindings[i].BoundSocket.Close(); err != nil { + log.G(ctx).WithFields(log.Fields{ + "error": err, + "mapping": bindings[i].PortBinding, + }).Warnf("failed to close proxy socket") + } + bindings[i].BoundSocket = nil + } + } + + return bindings, nil +} + +func (pm PortMapper) UnmapPorts(ctx context.Context, pbs []portmapperapi.PortBinding, fwn portmapperapi.Firewaller) error { + var errs []error + for _, pb := range pbs { + if pb.BoundSocket != nil { + if err := pb.BoundSocket.Close(); err != nil { + errs = append(errs, fmt.Errorf("failed to close socket for port mapping %s: %w", pb, err)) + } + } + if pb.PortDriverRemove != nil { + if err := pb.PortDriverRemove(); err != nil { + errs = append(errs, err) + } + } + if pb.StopProxy != nil { + if err := pb.StopProxy(); err != nil && !errors.Is(err, os.ErrProcessDone) { + errs = append(errs, fmt.Errorf("failed to stop userland proxy: %w", err)) + } + } + } + if err := fwn.DelPorts(ctx, mergeChildHostIPs(pbs)); err != nil { + errs = append(errs, err) + } + for _, pb := range pbs { + portallocator.Get().ReleasePort(pb.ChildHostIP, pb.Proto.String(), int(pb.HostPort)) + } + return errors.Join(errs...) +} + +// attemptBindHostPorts allocates host ports for each NAT port mapping, and +// reserves those ports by binding them. +// +// If the allocator doesn't have an available port in the required range, or the +// port can't be bound (perhaps because another process has already bound it), +// all resources are released and an error is returned. When ports are +// successfully reserved, a PortBinding is returned for each mapping. +func (pm PortMapper) attemptBindHostPorts( + ctx context.Context, + cfg []portmapperapi.PortBindingReq, + proto types.Protocol, + hostPortStart, hostPortEnd uint16, + fwn portmapperapi.Firewaller, +) (_ []portmapperapi.PortBinding, retErr error) { + var err error + var port int + + addrs := make([]net.IP, 0, len(cfg)) + for i := range cfg { + cfg[i] = setChildHostIP(pm.pdc, cfg[i]) + addrs = append(addrs, cfg[i].ChildHostIP) + } + + pa := portallocator.NewOSAllocator() + port, socks, err := pa.RequestPortsInRange(addrs, proto, int(hostPortStart), int(hostPortEnd)) + if err != nil { + return nil, err + } + defer func() { + if retErr != nil { + pa.ReleasePorts(addrs, proto, port) + } + }() + + if len(socks) != len(cfg) { + for _, sock := range socks { + if err := sock.Close(); err != nil { + log.G(ctx).WithError(err).Warn("Failed to close socket") + } + } + return nil, types.InternalErrorf("port allocator returned %d sockets for %d port bindings", len(socks), len(cfg)) + } + + res := make([]portmapperapi.PortBinding, 0, len(cfg)) + defer func() { + if retErr != nil { + if err := pm.UnmapPorts(ctx, res, fwn); err != nil { + log.G(ctx).WithFields(log.Fields{ + "pbs": res, + "error": err, + }).Warn("Failed to release port bindings") + } + } + }() + + for i := range cfg { + pb := portmapperapi.PortBinding{ + PortBinding: cfg[i].PortBinding.GetCopy(), + BoundSocket: socks[i], + ChildHostIP: cfg[i].ChildHostIP, + } + pb.PortBinding.HostPort = uint16(port) + pb.PortBinding.HostPortEnd = pb.HostPort + res = append(res, pb) + } + + if err := configPortDriver(ctx, res, pm.pdc); err != nil { + return nil, err + } + if err := fwn.AddPorts(ctx, mergeChildHostIPs(res)); err != nil { + return nil, err + } + // Now the firewall rules are set up, it's safe to listen on the socket. (Listening + // earlier could result in dropped connections if the proxy becomes unreachable due + // to NAT rules sending packets directly to the container.) + // + // If not starting the proxy, nothing will ever accept a connection on the + // socket. Listen here anyway because SO_REUSEADDR is set, so bind() won't notice + // the problem if a port's bound to both INADDR_ANY and a specific address. (Also + // so the binding shows up in "netstat -at".) + if err := listenBoundPorts(res, pm.enableProxy); err != nil { + return nil, err + } + return res, nil +} + +func setChildHostIP(pdc PortDriverClient, req portmapperapi.PortBindingReq) portmapperapi.PortBindingReq { + if pdc == nil { + req.ChildHostIP = req.HostIP + return req + } + hip, _ := netip.AddrFromSlice(req.HostIP) + req.ChildHostIP = pdc.ChildHostIP(hip).AsSlice() + return req +} + +// mergeChildHostIPs take a slice of PortBinding and returns a slice of +// types.PortBinding, where the HostIP in each of the results has the +// value of ChildHostIP from the input (if present). +func mergeChildHostIPs(pbs []portmapperapi.PortBinding) []types.PortBinding { + res := make([]types.PortBinding, 0, len(pbs)) + for _, b := range pbs { + pb := b.PortBinding + if b.ChildHostIP != nil { + pb.HostIP = b.ChildHostIP + } + res = append(res, pb) + } + return res +} + +// configPortDriver passes the port binding's details to rootlesskit, and updates the +// port binding with callbacks to remove the rootlesskit config (or marks the binding as +// unsupported by rootlesskit). +func configPortDriver(ctx context.Context, pbs []portmapperapi.PortBinding, pdc PortDriverClient) error { + for i := range pbs { + b := pbs[i] + if pdc != nil && b.HostPort != 0 { + var err error + hip, ok := netip.AddrFromSlice(b.HostIP) + if !ok { + return fmt.Errorf("invalid host IP address in %s", b) + } + chip, ok := netip.AddrFromSlice(b.ChildHostIP) + if !ok { + return fmt.Errorf("invalid child host IP address %s in %s", b.ChildHostIP, b) + } + pbs[i].PortDriverRemove, err = pdc.AddPort(ctx, b.Proto.String(), hip, chip, int(b.HostPort)) + if err != nil { + var pErr *rlkclient.ProtocolUnsupportedError + if errors.As(err, &pErr) { + log.G(ctx).WithFields(log.Fields{ + "error": pErr, + }).Warnf("discarding request for %q", net.JoinHostPort(hip.String(), strconv.Itoa(int(b.HostPort)))) + pbs[i].RootlesskitUnsupported = true + continue + } + return err + } + } + } + return nil +} + +func listenBoundPorts(pbs []portmapperapi.PortBinding, proxyEnabled bool) error { + for i := range pbs { + if pbs[i].BoundSocket == nil || pbs[i].RootlesskitUnsupported || pbs[i].Proto == types.UDP { + continue + } + rc, err := pbs[i].BoundSocket.SyscallConn() + if err != nil { + return fmt.Errorf("raw conn not available on %d socket: %w", pbs[i].Proto, err) + } + if errC := rc.Control(func(fd uintptr) { + somaxconn := 0 + // SCTP sockets do not support somaxconn=0 + if proxyEnabled || pbs[i].Proto == types.SCTP { + somaxconn = -1 // silently capped to "/proc/sys/net/core/somaxconn" + } + err = syscall.Listen(int(fd), somaxconn) + }); errC != nil { + return fmt.Errorf("failed to Control %s socket: %w", pbs[i].Proto, err) + } + if err != nil { + return fmt.Errorf("failed to listen on %s socket: %w", pbs[i].Proto, err) + } + } + return nil +} diff --git a/daemon/libnetwork/portmappers/nat/mapper_linux_test.go b/daemon/libnetwork/portmappers/nat/mapper_linux_test.go new file mode 100644 index 0000000000..fed6c69def --- /dev/null +++ b/daemon/libnetwork/portmappers/nat/mapper_linux_test.go @@ -0,0 +1,36 @@ +package nat + +import ( + "context" + "testing" + + "github.com/docker/docker/daemon/libnetwork/portmapperapi" + "github.com/docker/docker/daemon/libnetwork/types" + "gotest.tools/v3/assert" + is "gotest.tools/v3/assert/cmp" +) + +func TestBindHostPortsError(t *testing.T) { + cfg := []portmapperapi.PortBindingReq{ + { + PortBinding: types.PortBinding{ + Proto: types.TCP, + Port: 80, + HostPort: 8080, + HostPortEnd: 8080, + }, + }, + { + PortBinding: types.PortBinding{ + Proto: types.TCP, + Port: 80, + HostPort: 8080, + HostPortEnd: 8081, + }, + }, + } + pm := &PortMapper{} + pbs, err := pm.MapPorts(context.Background(), cfg, nil) + assert.Check(t, is.Error(err, "port binding mismatch 80/tcp:8080-8080, 80/tcp:8080-8081")) + assert.Check(t, is.Nil(pbs)) +} diff --git a/daemon/libnetwork/portmappers/routed/mapper_linux.go b/daemon/libnetwork/portmappers/routed/mapper_linux.go new file mode 100644 index 0000000000..cbf691744e --- /dev/null +++ b/daemon/libnetwork/portmappers/routed/mapper_linux.go @@ -0,0 +1,60 @@ +// FIXME(thaJeztah): remove once we are a module; the go:build directive prevents go from downgrading language version to go1.16: +//go:build go1.23 + +package routed + +import ( + "context" + + "github.com/containerd/log" + "github.com/docker/docker/daemon/libnetwork/portmapperapi" + "github.com/docker/docker/daemon/libnetwork/types" + "github.com/docker/docker/internal/sliceutil" +) + +const driverName = "routed" + +// Register the "routed" port-mapper with libnetwork. +func Register(r portmapperapi.Registerer) error { + return r.Register(driverName, NewPortMapper()) +} + +type PortMapper struct{} + +func NewPortMapper() PortMapper { + return PortMapper{} +} + +// MapPorts sets up firewall rules to allow direct remote access to pbs. +func (pm PortMapper) MapPorts(ctx context.Context, reqs []portmapperapi.PortBindingReq, fwn portmapperapi.Firewaller) ([]portmapperapi.PortBinding, error) { + if len(reqs) == 0 { + return nil, nil + } + + res := make([]portmapperapi.PortBinding, 0, len(reqs)) + bindings := make([]types.PortBinding, 0, len(reqs)) + for _, c := range reqs { + pb := portmapperapi.PortBinding{PortBinding: c.GetCopy()} + if pb.HostPort != 0 || pb.HostPortEnd != 0 { + log.G(ctx).WithFields(log.Fields{"mapping": pb}).Infof( + "Host port ignored, because NAT is disabled") + pb.HostPort = 0 + pb.HostPortEnd = 0 + } + res = append(res, pb) + bindings = append(bindings, pb.PortBinding) + } + + if err := fwn.AddPorts(ctx, bindings); err != nil { + return nil, err + } + + return res, nil +} + +// UnmapPorts removes firewall rules allowing direct remote access to the pbs. +func (pm PortMapper) UnmapPorts(ctx context.Context, pbs []portmapperapi.PortBinding, fwn portmapperapi.Firewaller) error { + return fwn.DelPorts(ctx, sliceutil.Map(pbs, func(pb portmapperapi.PortBinding) types.PortBinding { + return pb.PortBinding + })) +}