Merge pull request #50362 from akerouanton/linux-osallocator

libnet/portallocator: introduce OSAllocator
This commit is contained in:
Albin Kerouanton
2025-07-15 13:56:16 +02:00
committed by GitHub
6 changed files with 430 additions and 137 deletions

View File

@@ -21,7 +21,6 @@ import (
"github.com/docker/docker/daemon/libnetwork/portallocator"
"github.com/docker/docker/daemon/libnetwork/portmapper"
"github.com/docker/docker/daemon/libnetwork/types"
"github.com/ishidawataru/sctp"
)
type portBinding struct {
@@ -503,7 +502,7 @@ func bindHostPorts(
var err error
for i := 0; i < maxAllocatePortAttempts; i++ {
var b []portBinding
b, err = attemptBindHostPorts(ctx, cfg, proto.String(), hostPort, hostPortEnd, proxyPath, pdc, fwn)
b, err = attemptBindHostPorts(ctx, cfg, proto, hostPort, hostPortEnd, proxyPath, pdc, fwn)
if err == nil {
return b, nil
}
@@ -530,7 +529,7 @@ func bindHostPorts(
func attemptBindHostPorts(
ctx context.Context,
cfg []portBindingReq,
proto string,
proto types.Protocol,
hostPortStart, hostPortEnd uint16,
proxyPath string,
pdc portDriverClient,
@@ -544,19 +543,26 @@ func attemptBindHostPorts(
addrs = append(addrs, c.childHostIP)
}
pa := portallocator.Get()
port, err = pa.RequestPortsInRange(addrs, proto, int(hostPortStart), int(hostPortEnd))
pa := portallocator.NewOSAllocator()
port, socks, err := pa.RequestPortsInRange(addrs, proto, int(hostPortStart), int(hostPortEnd))
if err != nil {
return nil, err
}
defer func() {
if retErr != nil {
for _, a := range addrs {
pa.ReleasePort(a, proto, port)
}
pa.ReleasePorts(addrs, proto, port)
}
}()
if len(socks) != len(cfg) {
for _, sock := range socks {
if err := sock.Close(); err != nil {
log.G(ctx).WithError(err).Warn("Failed to close socket")
}
}
return nil, types.InternalErrorf("port allocator returned %d sockets for %d port bindings", len(socks), len(cfg))
}
res := make([]portBinding, 0, len(cfg))
defer func() {
if retErr != nil {
@@ -566,21 +572,14 @@ func attemptBindHostPorts(
}
}()
for _, c := range cfg {
var pb portBinding
switch proto {
case "tcp":
pb, err = bindTCPOrUDP(c, port, syscall.SOCK_STREAM, syscall.IPPROTO_TCP)
case "udp":
pb, err = bindTCPOrUDP(c, port, syscall.SOCK_DGRAM, syscall.IPPROTO_UDP)
case "sctp":
pb, err = bindSCTP(c, port)
default:
return nil, fmt.Errorf("Unknown addr type: %s", proto)
}
if err != nil {
return nil, err
for i := range cfg {
pb := portBinding{
PortBinding: cfg[i].PortBinding.GetCopy(),
boundSocket: socks[i],
childHostIP: cfg[i].childHostIP,
}
pb.PortBinding.HostPort = uint16(port)
pb.PortBinding.HostPortEnd = pb.HostPort
res = append(res, pb)
}
@@ -604,117 +603,6 @@ func attemptBindHostPorts(
return res, nil
}
func bindTCPOrUDP(cfg portBindingReq, port, typ, proto int) (_ portBinding, retErr error) {
pb := portBinding{PortBinding: cfg.PortBinding.GetCopy()}
pb.HostPort = uint16(port)
pb.HostPortEnd = pb.HostPort
pb.childHostIP = cfg.childHostIP
var domain int
var sa syscall.Sockaddr
if hip := cfg.childHostIP.To4(); hip != nil {
domain = syscall.AF_INET
sa4 := syscall.SockaddrInet4{Port: port}
copy(sa4.Addr[:], hip)
sa = &sa4
} else {
domain = syscall.AF_INET6
sa6 := syscall.SockaddrInet6{Port: port}
copy(sa6.Addr[:], cfg.childHostIP)
sa = &sa6
}
sd, err := syscall.Socket(domain, typ|syscall.SOCK_CLOEXEC, proto)
if err != nil {
return portBinding{}, fmt.Errorf("failed to create socket for userland proxy for %s: %w", cfg, err)
}
defer func() {
if retErr != nil {
syscall.Close(sd)
}
}()
if err := syscall.SetsockoptInt(sd, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1); err != nil {
return portBinding{}, fmt.Errorf("failed to setsockopt(SO_REUSEADDR) for %s: %w", cfg, err)
}
if domain == syscall.AF_INET6 {
syscall.SetsockoptInt(sd, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, 1)
}
if typ == syscall.SOCK_DGRAM {
// Enable IP_PKTINFO for UDP sockets to get the destination address.
// The destination address will be used as the source address when
// sending back replies coming from the container.
lvl := syscall.IPPROTO_IP
opt := syscall.IP_PKTINFO
optName := "IP_PKTINFO"
if domain == syscall.AF_INET6 {
lvl = syscall.IPPROTO_IPV6
opt = syscall.IPV6_RECVPKTINFO
optName = "IPV6_RECVPKTINFO"
}
if err := syscall.SetsockoptInt(sd, lvl, opt, 1); err != nil {
return portBinding{}, fmt.Errorf("failed to setsockopt(%s) for %s: %w", optName, cfg, err)
}
}
if err := syscall.Bind(sd, sa); err != nil {
if cfg.HostPort == cfg.HostPortEnd {
return portBinding{}, fmt.Errorf("failed to bind host port for %s: %w", cfg, err)
}
return portBinding{}, fmt.Errorf("failed to bind host port %d for %s: %w", port, cfg, err)
}
pb.boundSocket = os.NewFile(uintptr(sd), "listener")
if pb.boundSocket == nil {
return portBinding{}, fmt.Errorf("failed to convert socket for userland proxy for %s", cfg)
}
return pb, nil
}
// bindSCTP is based on sctp.ListenSCTP. The socket is created and bound, but
// does not start listening.
func bindSCTP(cfg portBindingReq, port int) (_ portBinding, retErr error) {
pb := portBinding{PortBinding: cfg.GetCopy()}
pb.HostPort = uint16(port)
pb.HostPortEnd = pb.HostPort
pb.childHostIP = cfg.childHostIP
domain := syscall.AF_INET
if cfg.childHostIP.To4() == nil {
domain = syscall.AF_INET6
}
sd, err := syscall.Socket(domain, syscall.SOCK_STREAM|syscall.SOCK_CLOEXEC, syscall.IPPROTO_SCTP)
if err != nil {
return portBinding{}, fmt.Errorf("failed to create socket for userland proxy for %s: %w", cfg, err)
}
defer func() {
if retErr != nil {
syscall.Close(sd)
}
}()
if domain == syscall.AF_INET6 {
syscall.SetsockoptInt(sd, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, 1)
}
if errno := setSCTPInitMsg(sd, sctp.InitMsg{NumOstreams: sctp.SCTP_MAX_STREAM}); errno != 0 {
return portBinding{}, errno
}
if err := sctp.SCTPBind(sd,
&sctp.SCTPAddr{IPAddrs: []net.IPAddr{{IP: cfg.childHostIP}}, Port: int(cfg.HostPort)},
sctp.SCTP_BINDX_ADD_ADDR); err != nil {
return portBinding{}, fmt.Errorf("failed to bind socket for userland proxy for %s: %w", cfg, err)
}
pb.boundSocket = os.NewFile(uintptr(sd), "listener")
if pb.boundSocket == nil {
return portBinding{}, fmt.Errorf("failed to convert socket for userland proxy for %s", cfg)
}
return pb, nil
}
// configPortDriver passes the port binding's details to rootlesskit, and updates the
// port binding with callbacks to remove the rootlesskit config (or marks the binding as
// unsupported by rootlesskit).

View File

@@ -360,7 +360,7 @@ func TestAddPortMappings(t *testing.T) {
cfg: []types.PortBinding{{Proto: types.TCP, Port: 80, HostPort: 8080}},
proxyPath: "/dummy/path/to/proxy",
busyPortIPv4: 8080,
expErr: "failed to bind host port for 0.0.0.0:8080:172.19.0.2:80/tcp: address already in use",
expErr: "failed to bind host port 0.0.0.0:8080/tcp: address already in use",
},
{
name: "ipv4 mapped container address with specific host port",
@@ -450,7 +450,7 @@ func TestAddPortMappings(t *testing.T) {
},
proxyPath: "/dummy/path/to/proxy",
busyPortIPv4: 8081,
expErr: "failed to bind host port 8081 for 0.0.0.0:8080-8082:172.19.0.2:82/tcp",
expErr: "failed to bind host port 0.0.0.0:8081",
},
{
name: "map host ipv6 to ipv4 container with proxy",

View File

@@ -0,0 +1,196 @@
package portallocator
import (
"context"
"fmt"
"net"
"net/netip"
"os"
"syscall"
"github.com/containerd/log"
"github.com/docker/docker/daemon/libnetwork/types"
"github.com/ishidawataru/sctp"
)
type OSAllocator struct {
// allocator is used to logically reserve ports, to avoid those we know
// are already in use. This is useful to ensure callers don't burn their
// retry budget unnecessarily.
allocator *PortAllocator
}
func NewOSAllocator() OSAllocator {
return OSAllocator{
allocator: Get(),
}
}
// RequestPortsInRange reserves a port available in the range [portStart, portEnd]
// for all the specified addrs, and then try to bind those addresses to allocate
// the port from the OS. It returns the allocated port, and all the sockets
// bound, or an error if the reserved port isn't available. Callers must take
// care of closing the returned sockets.
//
// Due to the semantic of SO_REUSEADDR, the OSAllocator can't fully determine
// if a port is free when binding 0.0.0.0 or ::. If another socket is binding
// the same port, but it's not listening to it yet, the bind will succeed but a
// subsequent listen might fail. For this reason, RequestPortsInRange doesn't
// retry on failure — it's caller's responsibility.
//
// It's safe for concurrent use.
func (pa OSAllocator) RequestPortsInRange(addrs []net.IP, proto types.Protocol, portStart, portEnd int) (_ int, _ []*os.File, retErr error) {
port, err := pa.allocator.RequestPortsInRange(addrs, proto.String(), portStart, portEnd)
if err != nil {
return 0, nil, err
}
defer func() {
if retErr != nil {
for _, addr := range addrs {
pa.allocator.ReleasePort(addr, proto.String(), port)
}
}
}()
var boundSocks []*os.File
defer func() {
if retErr != nil {
for i, sock := range boundSocks {
if err := sock.Close(); err != nil {
log.G(context.TODO()).WithFields(log.Fields{
"addr": addrs[i],
"port": port,
}).WithError(err).Warnf("failed to close socket during port allocation")
}
}
}
}()
for _, addr := range addrs {
addr, _ := netip.AddrFromSlice(addr)
addrPort := netip.AddrPortFrom(addr.Unmap(), uint16(port))
var sock *os.File
switch proto {
case types.TCP:
sock, err = bindTCPOrUDP(addrPort, syscall.SOCK_STREAM, syscall.IPPROTO_TCP)
case types.UDP:
sock, err = bindTCPOrUDP(addrPort, syscall.SOCK_DGRAM, syscall.IPPROTO_UDP)
case types.SCTP:
sock, err = bindSCTP(addrPort)
default:
return 0, nil, fmt.Errorf("protocol %s not supported", proto)
}
if err != nil {
return 0, nil, err
}
boundSocks = append(boundSocks, sock)
}
return port, boundSocks, nil
}
// ReleasePorts releases a common port reserved for a list of addrs. It doesn't
// close the sockets bound by [RequestPortsInRange]. This must be taken care of
// independently by the caller.
func (pa OSAllocator) ReleasePorts(addrs []net.IP, proto types.Protocol, port int) {
for _, addr := range addrs {
pa.allocator.ReleasePort(addr, proto.String(), port)
}
}
func bindTCPOrUDP(addr netip.AddrPort, typ int, proto types.Protocol) (_ *os.File, retErr error) {
var domain int
var sa syscall.Sockaddr
if addr.Addr().Unmap().Is4() {
domain = syscall.AF_INET
sa = &syscall.SockaddrInet4{Addr: addr.Addr().As4(), Port: int(addr.Port())}
} else {
domain = syscall.AF_INET6
sa = &syscall.SockaddrInet6{Addr: addr.Addr().Unmap().As16(), Port: int(addr.Port())}
}
sd, err := syscall.Socket(domain, typ|syscall.SOCK_CLOEXEC, int(proto))
if err != nil {
return nil, fmt.Errorf("failed to create socket for %s/%s: %w", addr, proto, err)
}
defer func() {
if retErr != nil {
syscall.Close(sd)
}
}()
if err := syscall.SetsockoptInt(sd, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1); err != nil {
return nil, fmt.Errorf("failed to setsockopt(SO_REUSEADDR) for %s/%s: %w", addr, proto, err)
}
if domain == syscall.AF_INET6 {
syscall.SetsockoptInt(sd, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, 1)
}
if typ == syscall.SOCK_DGRAM {
// Enable IP_PKTINFO for UDP sockets to get the destination address.
// The destination address will be used as the source address when
// sending back replies coming from the container.
lvl := syscall.IPPROTO_IP
opt := syscall.IP_PKTINFO
optName := "IP_PKTINFO"
if domain == syscall.AF_INET6 {
lvl = syscall.IPPROTO_IPV6
opt = syscall.IPV6_RECVPKTINFO
optName = "IPV6_RECVPKTINFO"
}
if err := syscall.SetsockoptInt(sd, lvl, opt, 1); err != nil {
return nil, fmt.Errorf("failed to setsockopt(%s) for %s/%s: %w", optName, addr, proto, err)
}
}
if err := syscall.Bind(sd, sa); err != nil {
return nil, fmt.Errorf("failed to bind host port %s/%s: %w", addr, proto, err)
}
boundSocket := os.NewFile(uintptr(sd), "listener")
if boundSocket == nil {
return nil, fmt.Errorf("failed to convert socket to file for %s/%s", addr, proto)
}
return boundSocket, nil
}
// bindSCTP is based on sctp.ListenSCTP. The socket is created and bound, but
// does not start listening.
func bindSCTP(addr netip.AddrPort) (_ *os.File, retErr error) {
domain := syscall.AF_INET
if addr.Addr().Unmap().Is6() {
domain = syscall.AF_INET6
}
sd, err := syscall.Socket(domain, syscall.SOCK_STREAM|syscall.SOCK_CLOEXEC, syscall.IPPROTO_SCTP)
if err != nil {
return nil, fmt.Errorf("failed to create socket for %s/sctp: %w", addr, err)
}
defer func() {
if retErr != nil {
syscall.Close(sd)
}
}()
if domain == syscall.AF_INET6 {
syscall.SetsockoptInt(sd, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, 1)
}
if errno := setSCTPInitMsg(sd, sctp.InitMsg{NumOstreams: sctp.SCTP_MAX_STREAM}); errno != 0 {
return nil, errno
}
if err := sctp.SCTPBind(sd,
&sctp.SCTPAddr{IPAddrs: []net.IPAddr{{IP: addr.Addr().Unmap().AsSlice()}}, Port: int(addr.Port())},
sctp.SCTP_BINDX_ADD_ADDR); err != nil {
return nil, fmt.Errorf("failed to bind host port %s/sctp: %w", addr, err)
}
boundSocket := os.NewFile(uintptr(sd), "listener")
if boundSocket == nil {
return nil, fmt.Errorf("failed to convert socket %s/sctp", addr)
}
return boundSocket, nil
}

View File

@@ -0,0 +1,209 @@
package portallocator
import (
"io"
"net"
"os"
"testing"
"github.com/docker/docker/daemon/libnetwork/netutils"
"github.com/docker/docker/daemon/libnetwork/types"
"github.com/ishidawataru/sctp"
"golang.org/x/sys/unix"
"gotest.tools/v3/assert"
is "gotest.tools/v3/assert/cmp"
)
func listen(t *testing.T, proto types.Protocol, addr net.IP, port int) io.Closer {
var l io.Closer
var err error
switch proto {
case types.TCP:
l, err = net.ListenTCP("tcp", &net.TCPAddr{IP: addr, Port: port})
case types.UDP:
l, err = net.ListenUDP("udp", &net.UDPAddr{IP: addr, Port: port})
case types.SCTP:
l, err = sctp.ListenSCTP("sctp", &sctp.SCTPAddr{IPAddrs: []net.IPAddr{{IP: addr}}, Port: port})
default:
t.Fatalf("protocol %s not supported", proto)
}
assert.NilError(t, err)
return l
}
func closeSocks(t *testing.T, files []*os.File) {
for _, f := range files {
if f != nil {
err := f.Close()
assert.NilError(t, err)
}
}
}
func TestAllocateExactPort(t *testing.T) {
alloc := NewOSAllocator()
addrs := []net.IP{net.IPv4zero}
for _, expProto := range []types.Protocol{types.TCP, types.UDP, types.SCTP} {
t.Run(expProto.String(), func(t *testing.T) {
port, socks, err := alloc.RequestPortsInRange(addrs, expProto, 31234, 31234)
defer alloc.ReleasePorts(addrs, expProto, port)
defer closeSocks(t, socks)
assert.NilError(t, err)
assert.Equal(t, port, 31234)
assert.Check(t, is.Len(socks, 1))
})
}
}
func TestAllocateExactPortForMultipleAddrs(t *testing.T) {
alloc := NewOSAllocator()
addrs := []net.IP{
net.ParseIP("127.0.0.1"),
net.ParseIP("127.0.0.2"),
net.ParseIP("127.0.0.3"),
}
if netutils.IsV6Listenable() {
addrs = append(addrs, net.IPv6loopback)
}
for _, expProto := range []types.Protocol{types.TCP, types.UDP, types.SCTP} {
t.Run(expProto.String(), func(t *testing.T) {
port, socks, err := alloc.RequestPortsInRange(addrs, expProto, 31234, 31234)
defer alloc.ReleasePorts(addrs, expProto, port)
defer closeSocks(t, socks)
assert.NilError(t, err)
assert.Equal(t, port, 31234)
assert.Check(t, is.Len(socks, len(addrs)))
for i, sock := range socks {
sa, err := unix.Getsockname(int(sock.Fd()))
assert.NilError(t, err)
expAddr := addrs[i]
if expAddr.To4() != nil {
assert.Check(t, is.Equal(sa.(*unix.SockaddrInet4).Port, port))
addr := net.IP(sa.(*unix.SockaddrInet4).Addr[:])
assert.Check(t, addr.Equal(expAddr))
} else {
assert.Check(t, is.Equal(sa.(*unix.SockaddrInet6).Port, port))
addr := net.IP(sa.(*unix.SockaddrInet6).Addr[:])
assert.Check(t, addr.Equal(expAddr))
}
proto, err := unix.GetsockoptInt(int(sock.Fd()), unix.SOL_SOCKET, unix.SO_PROTOCOL)
assert.NilError(t, err)
assert.Check(t, is.Equal(proto, int(expProto)))
}
})
}
}
func TestAllocateExactPortInUse(t *testing.T) {
alloc := NewOSAllocator()
addrs := []net.IP{net.ParseIP("127.0.0.1")}
for _, tc := range []struct {
proto types.Protocol
expErr string
}{
{proto: types.TCP, expErr: "failed to bind host port 127.0.0.1:12345/tcp: address already in use"},
{proto: types.UDP, expErr: "failed to bind host port 127.0.0.1:12345/udp: address already in use"},
{proto: types.SCTP, expErr: "failed to bind host port 127.0.0.1:12345/sctp: address already in use"},
} {
t.Run(tc.proto.String(), func(t *testing.T) {
l := listen(t, tc.proto, net.IPv4zero, 12345)
defer l.Close()
// Port 12345 is in use, so the first allocation attempt should fail
_, _, err := alloc.RequestPortsInRange(addrs, tc.proto, 12345, 12345)
assert.ErrorContains(t, err, tc.expErr)
// Close port 12345, and retry the allocation — it should succeed this time
err = l.Close()
assert.NilError(t, err)
port, socks, err := alloc.RequestPortsInRange(addrs, tc.proto, 12345, 12345)
defer alloc.ReleasePorts(addrs, tc.proto, port)
defer closeSocks(t, socks)
assert.NilError(t, err)
assert.Equal(t, port, 12345)
})
}
}
func TestAllocateRangePortInUse(t *testing.T) {
alloc := NewOSAllocator()
addrs := []net.IP{net.ParseIP("127.0.0.1")}
for _, tc := range []struct {
proto types.Protocol
expErr string
}{
{proto: types.TCP, expErr: "failed to bind host port 127.0.0.1:8080/tcp: address already in use"},
{proto: types.UDP, expErr: "failed to bind host port 127.0.0.1:8080/udp: address already in use"},
{proto: types.SCTP, expErr: "failed to bind host port 127.0.0.1:8080/sctp: address already in use"},
} {
t.Run(tc.proto.String(), func(t *testing.T) {
l := listen(t, tc.proto, net.IPv4zero, 8080)
defer l.Close()
// Port 8080 is in use, so the first allocation attempt should fail
_, _, err := alloc.RequestPortsInRange(addrs, tc.proto, 8080, 8081)
assert.ErrorContains(t, err, tc.expErr)
// Retry allocation with same range — this time, it should pick 8081 successfully
port, socks81, err := alloc.RequestPortsInRange(addrs, tc.proto, 8080, 8081)
defer alloc.ReleasePorts(addrs, tc.proto, port)
defer closeSocks(t, socks81)
assert.NilError(t, err)
assert.Equal(t, port, 8081)
// Close port 8080, and try to allocate the same port again — it should succeed this time
err = l.Close()
assert.NilError(t, err)
port, socks80, err := alloc.RequestPortsInRange(addrs, tc.proto, 8080, 8081)
defer alloc.ReleasePorts(addrs, tc.proto, port)
defer closeSocks(t, socks80)
assert.NilError(t, err)
assert.Equal(t, port, 8080)
})
}
}
func TestRepeatAllocation(t *testing.T) {
alloc := NewOSAllocator()
addrs := []net.IP{net.ParseIP("127.0.0.1")}
for _, proto := range []types.Protocol{types.TCP, types.UDP, types.SCTP} {
t.Run(proto.String(), func(t *testing.T) {
// First allocation
port, socks, err := alloc.RequestPortsInRange(addrs, proto, 8080, 8080)
defer alloc.ReleasePorts(addrs, proto, port)
defer func() {
closeSocks(t, socks)
}()
assert.NilError(t, err)
assert.Equal(t, port, 8080)
// Release the port
alloc.ReleasePorts(addrs, proto, port)
closeSocks(t, socks)
// Repeat the same allocation
port, socks, err = alloc.RequestPortsInRange(addrs, proto, 8080, 8080)
assert.NilError(t, err)
assert.Equal(t, port, 8080)
})
}
}

View File

@@ -1,4 +1,4 @@
package bridge
package portallocator
import (
"syscall"

View File

@@ -1,6 +1,6 @@
//go:build linux && !386
package bridge
package portallocator
import (
"syscall"