diff --git a/libnetwork/drivers/bridge/port_mapping_linux.go b/libnetwork/drivers/bridge/port_mapping_linux.go index 00647bfa60..969a277caf 100644 --- a/libnetwork/drivers/bridge/port_mapping_linux.go +++ b/libnetwork/drivers/bridge/port_mapping_linux.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" "net" + "net/netip" + "slices" "strconv" "github.com/containerd/log" @@ -64,10 +66,23 @@ func (n *bridgeNetwork) addPortMappings( } }() + sortedCfg := slices.Clone(cfg) + sortAndNormPBs(sortedCfg) + proxyPath := n.userlandProxyPath() disableNAT4, disableNAT6 := n.getNATDisabled() - for _, c := range cfg { - toBind := make([]portBindingReq, 0, 2) + + // toBind accumulates port bindings that should be allocated the same host port + // (if required by NAT config). If the host address is unspecified, and defHostIP + // is 0.0.0.0, one iteration of the loop may generate bindings for v4 and v6. If + // a host address is specified, it'll either be IPv4 or IPv6, and only one + // binding will be added per iteration. Config for bindings that only differ in + // host IP are sorted next to each other, the loop continues until toBind has + // collected them all, for both v4 and v6. The addresses may be 0.0.0.0 and [::], + // or multiple addresses of both address families. Once there are no more + // bindings to collect, they're applied and toBind is reset. + var toBind []portBindingReq + for i, c := range sortedCfg { if bindingIPv4, ok := configurePortBindingIPv4(disableNAT4, c, containerIPv4, defHostIP); ok { toBind = append(toBind, bindingIPv4) } @@ -88,11 +103,22 @@ func (n *bridgeNetwork) addPortMappings( toBind = append(toBind, bindingIPv6) } + if i < len(sortedCfg)-1 && needSamePort(c, sortedCfg[i+1]) { + // This port binding matches the next, apart from host IP. So, continue + // collecting bindings, then allocate the same host port for all addresses. + continue + } + + // Allocate a host port, and reserve it by starting docker-proxy for each host + // address in toBind. newB, err := bindHostPorts(toBind, proxyPath) if err != nil { return nil, err } bindings = append(bindings, newB...) + + // Reset the collection of bindings now they're bound. + toBind = toBind[:0] } for _, b := range bindings { @@ -104,6 +130,70 @@ func (n *bridgeNetwork) addPortMappings( return bindings, nil } +// sortAndNormPBs normalises cfg by making HostPortEnd=HostPort (rather than 0) if the +// host port isn't a range - and sorts it into the ordering defined by cmpPortBinding. +func sortAndNormPBs(cfg []types.PortBinding) { + for i := range cfg { + if cfg[i].HostPortEnd == 0 { + cfg[i].HostPortEnd = cfg[i].HostPort + } + } + slices.SortFunc(cfg, cmpPortBinding) +} + +// cmpPortBinding defines an ordering over PortBinding such that bindings that differ +// only in host IP are adjacent (those bindings should be allocated the same port). +// +// Exact host ports are placed before ranges (in case exact ports fall within ranges, +// giving a better chance of allocating the exact ports), then PortBindings with the: +// - same container port are adjacent (lowest ports first), then +// - same protocols are adjacent (tcp < udp < sctp), then +// - same host ports or ranges are adjacent, then +// - ordered by container IP (then host IP, if set). +func cmpPortBinding(a, b types.PortBinding) int { + // Exact host port < host port range. + aIsRange := a.HostPort == 0 || a.HostPort != a.HostPortEnd + bIsRange := b.HostPort == 0 || b.HostPort != b.HostPortEnd + if aIsRange != bIsRange { + if aIsRange { + return 1 + } + return -1 + } + if a.Port != b.Port { + return int(a.Port) - int(b.Port) + } + if a.Proto != b.Proto { + return int(a.Proto) - int(b.Proto) + } + if a.HostPort != b.HostPort { + return int(a.HostPort) - int(b.HostPort) + } + if a.HostPortEnd != b.HostPortEnd { + return int(a.HostPortEnd) - int(b.HostPortEnd) + } + aHostIP, _ := netip.AddrFromSlice(a.HostIP) + bHostIP, _ := netip.AddrFromSlice(b.HostIP) + if c := aHostIP.Unmap().Compare(bHostIP.Unmap()); c != 0 { + return c + } + aIP, _ := netip.AddrFromSlice(a.IP) + bIP, _ := netip.AddrFromSlice(b.IP) + return aIP.Unmap().Compare(bIP.Unmap()) +} + +// needSamePort returns true iff a and b only differ in the host IP address, +// meaning they should be allocated the same host port (so that, if v4/v6 +// addresses are returned in a DNS response or similar, clients can bind without +// needing to adjust the port number depending on which address is used). +func needSamePort(a, b types.PortBinding) bool { + return a.Port == b.Port && + a.Proto == b.Proto && + a.HostPort == b.HostPort && + a.HostPortEnd == b.HostPortEnd && + a.IP.Equal(b.IP) +} + // configurePortBindingIPv4 returns a new port binding with the HostIP field populated // if a binding is required, else nil. func configurePortBindingIPv4(disableNAT bool, bnd types.PortBinding, containerIPv4, defHostIP net.IP) (portBindingReq, bool) { @@ -125,10 +215,6 @@ func configurePortBindingIPv4(disableNAT bool, bnd types.PortBinding, containerI // Unmap the addresses if they're IPv4-mapped IPv6. bnd.HostIP = bnd.HostIP.To4() bnd.IP = containerIPv4.To4() - // Adjust HostPortEnd if this is not a range. - if bnd.HostPortEnd == 0 { - bnd.HostPortEnd = bnd.HostPort - } return portBindingReq{ PortBinding: bnd, disableNAT: disableNAT, @@ -164,10 +250,6 @@ func configurePortBindingIPv6(disableNAT bool, bnd types.PortBinding, containerI } } bnd.IP = containerIP - // Adjust HostPortEnd if this is not a range. - if bnd.HostPortEnd == 0 { - bnd.HostPortEnd = bnd.HostPort - } return portBindingReq{ PortBinding: bnd, disableNAT: disableNAT, diff --git a/libnetwork/drivers/bridge/port_mapping_linux_test.go b/libnetwork/drivers/bridge/port_mapping_linux_test.go index 21ee5fa388..d779ba61e7 100644 --- a/libnetwork/drivers/bridge/port_mapping_linux_test.go +++ b/libnetwork/drivers/bridge/port_mapping_linux_test.go @@ -31,9 +31,9 @@ func TestPortMappingConfig(t *testing.T) { t.Fatalf("Failed to setup driver config: %v", err) } - binding1 := types.PortBinding{Proto: types.UDP, Port: uint16(400), HostPort: uint16(54000)} - binding2 := types.PortBinding{Proto: types.TCP, Port: uint16(500), HostPort: uint16(65000)} - binding3 := types.PortBinding{Proto: types.SCTP, Port: uint16(300), HostPort: uint16(65000)} + binding1 := types.PortBinding{Proto: types.SCTP, Port: uint16(300), HostPort: uint16(65000)} + binding2 := types.PortBinding{Proto: types.UDP, Port: uint16(400), HostPort: uint16(54000)} + binding3 := types.PortBinding{Proto: types.TCP, Port: uint16(500), HostPort: uint16(65000)} portBindings := []types.PortBinding{binding1, binding2, binding3} sbOptions := make(map[string]interface{}) @@ -180,6 +180,57 @@ func loopbackUp() error { return nlHandle.LinkSetUp(iface) } +func TestCmpPortBindings(t *testing.T) { + pb := types.PortBinding{ + Proto: types.TCP, + IP: net.ParseIP("172.17.0.2"), + Port: 80, + HostIP: net.ParseIP("192.168.1.2"), + HostPort: 8080, + HostPortEnd: 8080, + } + var pbA, pbB types.PortBinding + + assert.Check(t, cmpPortBinding(pb, pb) == 0) + + pbA, pbB = pb, pb + pbA.Port = 22 + assert.Check(t, cmpPortBinding(pbA, pbB) < 0) + assert.Check(t, cmpPortBinding(pbB, pbA) > 0) + + pbA, pbB = pb, pb + pbB.Proto = types.UDP + assert.Check(t, cmpPortBinding(pbA, pbB) < 0) + assert.Check(t, cmpPortBinding(pbB, pbA) > 0) + + pbA, pbB = pb, pb + pbA.Port = 22 + pbA.Proto = types.UDP + assert.Check(t, cmpPortBinding(pbA, pbB) < 0) + assert.Check(t, cmpPortBinding(pbB, pbA) > 0) + + pbA, pbB = pb, pb + pbB.HostPort = 8081 + assert.Check(t, cmpPortBinding(pbA, pbB) < 0) + assert.Check(t, cmpPortBinding(pbB, pbA) > 0) + + pbA, pbB = pb, pb + pbB.HostPort, pbB.HostPortEnd = 0, 0 + assert.Check(t, cmpPortBinding(pbA, pbB) < 0) + assert.Check(t, cmpPortBinding(pbB, pbA) > 0) + + pbA, pbB = pb, pb + pbB.HostPortEnd = 8081 + assert.Check(t, cmpPortBinding(pbA, pbB) < 0) + assert.Check(t, cmpPortBinding(pbB, pbA) > 0) + + pbA, pbB = pb, pb + pbA.HostPortEnd = 8080 + pbB.HostPortEnd = 8081 + assert.Check(t, cmpPortBinding(pbA, pbB) < 0) + assert.Check(t, cmpPortBinding(pbB, pbA) > 0) +} + func TestBindHostPortsError(t *testing.T) { cfg := []portBindingReq{ { @@ -323,9 +374,8 @@ func TestAddPortMappings(t *testing.T) { }, busyPortIPv4: 8080, expPBs: []types.PortBinding{ - {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: net.IPv4zero, HostPort: 8081, HostPortEnd: 8081}, - // Note that, unlike the previous test, IPv4/IPv6 get different host ports. - {Proto: types.TCP, IP: ctrIP6.IP, Port: 80, HostIP: net.IPv6zero, HostPort: 8080, HostPortEnd: 8080}, + {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: net.IPv4zero, HostPort: 8081}, + {Proto: types.TCP, IP: ctrIP6.IP, Port: 80, HostIP: net.IPv6zero, HostPort: 8081}, }, }, { @@ -344,14 +394,14 @@ func TestAddPortMappings(t *testing.T) { expPBs: []types.PortBinding{ {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: net.IPv4zero, HostPort: 8080, HostPortEnd: 8080}, {Proto: types.TCP, IP: ctrIP6.IP, Port: 80, HostIP: net.IPv6zero, HostPort: 8080, HostPortEnd: 8080}, - {Proto: types.TCP, IP: ctrIP4.IP, Port: 81, HostIP: net.IPv4zero, HostPort: 8081, HostPortEnd: 8081}, - {Proto: types.TCP, IP: ctrIP6.IP, Port: 81, HostIP: net.IPv6zero, HostPort: 8081, HostPortEnd: 8081}, - {Proto: types.TCP, IP: ctrIP4.IP, Port: 82, HostIP: net.IPv4zero, HostPort: 8083, HostPortEnd: 8083}, - {Proto: types.TCP, IP: ctrIP6.IP, Port: 82, HostIP: net.IPv6zero, HostPort: 8083, HostPortEnd: 8083}, {Proto: types.UDP, IP: ctrIP4.IP, Port: 80, HostIP: net.IPv4zero, HostPort: 8080, HostPortEnd: 8080}, {Proto: types.UDP, IP: ctrIP6.IP, Port: 80, HostIP: net.IPv6zero, HostPort: 8080, HostPortEnd: 8080}, + {Proto: types.TCP, IP: ctrIP4.IP, Port: 81, HostIP: net.IPv4zero, HostPort: 8081, HostPortEnd: 8081}, + {Proto: types.TCP, IP: ctrIP6.IP, Port: 81, HostIP: net.IPv6zero, HostPort: 8081, HostPortEnd: 8081}, {Proto: types.UDP, IP: ctrIP4.IP, Port: 81, HostIP: net.IPv4zero, HostPort: 8081, HostPortEnd: 8081}, {Proto: types.UDP, IP: ctrIP6.IP, Port: 81, HostIP: net.IPv6zero, HostPort: 8081, HostPortEnd: 8081}, + {Proto: types.TCP, IP: ctrIP4.IP, Port: 82, HostIP: net.IPv4zero, HostPort: 8083, HostPortEnd: 8083}, + {Proto: types.TCP, IP: ctrIP6.IP, Port: 82, HostIP: net.IPv6zero, HostPort: 8083, HostPortEnd: 8083}, {Proto: types.UDP, IP: ctrIP4.IP, Port: 82, HostIP: net.IPv4zero, HostPort: 8083, HostPortEnd: 8083}, {Proto: types.UDP, IP: ctrIP6.IP, Port: 82, HostIP: net.IPv6zero, HostPort: 8083, HostPortEnd: 8083}, }, @@ -436,17 +486,20 @@ func TestAddPortMappings(t *testing.T) { name: "error releasing bindings", epAddrV4: ctrIP4, epAddrV6: ctrIP6, - cfg: []types.PortBinding{{Proto: types.TCP, Port: 80, HostPort: 8080}, {Proto: types.TCP, Port: 22, HostPort: 2222}}, + cfg: []types.PortBinding{ + {Proto: types.TCP, Port: 80, HostPort: 8080}, + {Proto: types.TCP, Port: 22, HostPort: 2222}, + }, expPBs: []types.PortBinding{ - {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: net.IPv4zero, HostPort: 8080}, - {Proto: types.TCP, IP: ctrIP6.IP, Port: 80, HostIP: net.IPv6zero, HostPort: 8080}, {Proto: types.TCP, IP: ctrIP4.IP, Port: 22, HostIP: net.IPv4zero, HostPort: 2222}, {Proto: types.TCP, IP: ctrIP6.IP, Port: 22, HostIP: net.IPv6zero, HostPort: 2222}, + {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: net.IPv4zero, HostPort: 8080}, + {Proto: types.TCP, IP: ctrIP6.IP, Port: 80, HostIP: net.IPv6zero, HostPort: 8080}, }, - expReleaseErr: "failed to stop docker-proxy for port mapping tcp/172.19.0.2:80/0.0.0.0:8080: can't stop now\n" + - "failed to stop docker-proxy for port mapping tcp/fdf8:b88e:bb5c:3483::2:80/:::8080: can't stop now\n" + - "failed to stop docker-proxy for port mapping tcp/172.19.0.2:22/0.0.0.0:2222: can't stop now\n" + - "failed to stop docker-proxy for port mapping tcp/fdf8:b88e:bb5c:3483::2:22/:::2222: can't stop now", + expReleaseErr: "failed to stop docker-proxy for port mapping tcp/172.19.0.2:22/0.0.0.0:2222: can't stop now\n" + + "failed to stop docker-proxy for port mapping tcp/fdf8:b88e:bb5c:3483::2:22/:::2222: can't stop now\n" + + "failed to stop docker-proxy for port mapping tcp/172.19.0.2:80/0.0.0.0:8080: can't stop now\n" + + "failed to stop docker-proxy for port mapping tcp/fdf8:b88e:bb5c:3483::2:80/:::8080: can't stop now", }, { name: "disable nat6", @@ -497,6 +550,35 @@ func TestAddPortMappings(t *testing.T) { {Proto: types.TCP, IP: ctrIP6.IP, Port: 80, HostIP: net.IPv6zero}, }, }, + { + name: "same ports for matching mappings with different host addresses", + epAddrV4: ctrIP4, + epAddrV6: ctrIP6, + cfg: []types.PortBinding{ + // These two should both get the same host port. + {Proto: types.TCP, Port: 80, HostIP: newIPNet(t, "fd0c:9167:5b11::2/64").IP}, + {Proto: types.TCP, Port: 80, HostIP: newIPNet(t, "192.168.1.2/24").IP}, + // These three should all get the same host port. + {Proto: types.TCP, Port: 22, HostIP: newIPNet(t, "fd0c:9167:5b11::2/64").IP}, + {Proto: types.TCP, Port: 22, HostIP: newIPNet(t, "fd0c:9167:5b11::3/64").IP}, + {Proto: types.TCP, Port: 22, HostIP: newIPNet(t, "192.168.1.2/24").IP}, + // These two should get different host ports, and the exact-port should be allocated + // before the range. + {Proto: types.TCP, Port: 12345, HostPort: 12345, HostPortEnd: 12346}, + {Proto: types.TCP, Port: 12345, HostPort: 12345}, + }, + expPBs: []types.PortBinding{ + {Proto: types.TCP, IP: ctrIP4.IP, Port: 12345, HostIP: net.IPv4zero, HostPort: 12345}, + {Proto: types.TCP, IP: ctrIP6.IP, Port: 12345, HostIP: net.IPv6zero, HostPort: 12345}, + {Proto: types.TCP, IP: ctrIP4.IP, Port: 22, HostIP: newIPNet(t, "192.168.1.2/24").IP, HostPort: firstEphemPort}, + {Proto: types.TCP, IP: ctrIP6.IP, Port: 22, HostIP: newIPNet(t, "fd0c:9167:5b11::2/64").IP, HostPort: firstEphemPort}, + {Proto: types.TCP, IP: ctrIP6.IP, Port: 22, HostIP: newIPNet(t, "fd0c:9167:5b11::3/64").IP, HostPort: firstEphemPort}, + {Proto: types.TCP, IP: ctrIP4.IP, Port: 80, HostIP: newIPNet(t, "192.168.1.2/24").IP, HostPort: firstEphemPort + 1}, + {Proto: types.TCP, IP: ctrIP6.IP, Port: 80, HostIP: newIPNet(t, "fd0c:9167:5b11::2/64").IP, HostPort: firstEphemPort + 1}, + {Proto: types.TCP, IP: ctrIP4.IP, Port: 12345, HostIP: net.IPv4zero, HostPort: 12346}, + {Proto: types.TCP, IP: ctrIP6.IP, Port: 12345, HostIP: net.IPv6zero, HostPort: 12346}, + }, + }, } for _, tc := range testcases {