From 75c60598b77a27f2f8e032667b941dd54b0bbec9 Mon Sep 17 00:00:00 2001 From: Rob Murray Date: Wed, 12 Feb 2025 10:49:02 +0000 Subject: [PATCH 1/5] Move clearConntrackEntries to bridge_linux.go Signed-off-by: Rob Murray --- libnetwork/drivers/bridge/bridge_linux.go | 33 +++++++++++++++++ .../drivers/bridge/setup_ip_tables_linux.go | 35 ------------------- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/libnetwork/drivers/bridge/bridge_linux.go b/libnetwork/drivers/bridge/bridge_linux.go index a38b8390a7..1b3c7405d5 100644 --- a/libnetwork/drivers/bridge/bridge_linux.go +++ b/libnetwork/drivers/bridge/bridge_linux.go @@ -1607,6 +1607,39 @@ func (d *driver) RevokeExternalConnectivity(nid, eid string) error { return nil } +// clearConntrackEntries flushes conntrack entries matching endpoint IP address +// or matching one of the exposed UDP port. +// In the first case, this could happen if packets were received by the host +// between userland proxy startup and iptables setup. +// In the latter case, this could happen if packets were received whereas there +// were nowhere to route them, as netfilter creates entries in such case. +// This is required because iptables NAT rules are evaluated by netfilter only +// when creating a new conntrack entry. When Docker latter adds NAT rules, +// netfilter ignore them for any packet matching a pre-existing conntrack entry. +// As such, we need to flush all those conntrack entries to make sure NAT rules +// are correctly applied to all packets. +// See: #8795, #44688 & #44742. +func clearConntrackEntries(nlh nlwrap.Handle, ep *bridgeEndpoint) { + var ipv4List []net.IP + var ipv6List []net.IP + var udpPorts []uint16 + + if ep.addr != nil { + ipv4List = append(ipv4List, ep.addr.IP) + } + if ep.addrv6 != nil { + ipv6List = append(ipv6List, ep.addrv6.IP) + } + for _, pb := range ep.portMapping { + if pb.Proto == types.UDP { + udpPorts = append(udpPorts, pb.HostPort) + } + } + + iptables.DeleteConntrackEntries(nlh, ipv4List, ipv6List) + iptables.DeleteConntrackEntriesByPort(nlh, types.UDP, udpPorts) +} + func (d *driver) handleFirewalldReload() { if !d.config.EnableIPTables && !d.config.EnableIP6Tables { return diff --git a/libnetwork/drivers/bridge/setup_ip_tables_linux.go b/libnetwork/drivers/bridge/setup_ip_tables_linux.go index a7e89f2eef..b4aa9f195b 100644 --- a/libnetwork/drivers/bridge/setup_ip_tables_linux.go +++ b/libnetwork/drivers/bridge/setup_ip_tables_linux.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "net" "net/netip" "os" @@ -12,7 +11,6 @@ import ( "github.com/docker/docker/errdefs" "github.com/docker/docker/internal/nlwrap" "github.com/docker/docker/libnetwork/iptables" - "github.com/docker/docker/libnetwork/types" "github.com/vishvananda/netlink" ) @@ -787,39 +785,6 @@ func setupInternalNetworkRules(bridgeIface string, prefix netip.Prefix, icc, ins return setIcc(version, bridgeIface, icc, true, insert) } -// clearConntrackEntries flushes conntrack entries matching endpoint IP address -// or matching one of the exposed UDP port. -// In the first case, this could happen if packets were received by the host -// between userland proxy startup and iptables setup. -// In the latter case, this could happen if packets were received whereas there -// were nowhere to route them, as netfilter creates entries in such case. -// This is required because iptables NAT rules are evaluated by netfilter only -// when creating a new conntrack entry. When Docker latter adds NAT rules, -// netfilter ignore them for any packet matching a pre-existing conntrack entry. -// As such, we need to flush all those conntrack entries to make sure NAT rules -// are correctly applied to all packets. -// See: #8795, #44688 & #44742. -func clearConntrackEntries(nlh nlwrap.Handle, ep *bridgeEndpoint) { - var ipv4List []net.IP - var ipv6List []net.IP - var udpPorts []uint16 - - if ep.addr != nil { - ipv4List = append(ipv4List, ep.addr.IP) - } - if ep.addrv6 != nil { - ipv6List = append(ipv6List, ep.addrv6.IP) - } - for _, pb := range ep.portMapping { - if pb.Proto == types.UDP { - udpPorts = append(udpPorts, pb.HostPort) - } - } - - iptables.DeleteConntrackEntries(nlh, ipv4List, ipv6List) - iptables.DeleteConntrackEntriesByPort(nlh, types.UDP, udpPorts) -} - // mirroredWSL2Workaround adds or removes an IPv4 NAT rule, depending on whether // docker's host Linux appears to be a guest running under WSL2 in with mirrored // mode networking. From aa4abaf82095ba21e20f141a29dcd15a32e7984f Mon Sep 17 00:00:00 2001 From: Rob Murray Date: Thu, 10 Apr 2025 20:06:53 +0100 Subject: [PATCH 2/5] Use firewaller (iptabler) structs in iptables unit tests Signed-off-by: Rob Murray --- .../bridge/setup_ip_tables_linux_test.go | 283 ++++++------------ 1 file changed, 93 insertions(+), 190 deletions(-) diff --git a/libnetwork/drivers/bridge/setup_ip_tables_linux_test.go b/libnetwork/drivers/bridge/setup_ip_tables_linux_test.go index aa11b85a05..dcb8c9ffdf 100644 --- a/libnetwork/drivers/bridge/setup_ip_tables_linux_test.go +++ b/libnetwork/drivers/bridge/setup_ip_tables_linux_test.go @@ -3,18 +3,15 @@ package bridge import ( "context" "net" + "net/netip" "os" "os/exec" "path/filepath" "strings" "testing" - "github.com/docker/docker/internal/nlwrap" "github.com/docker/docker/internal/testutils/netnsutils" - "github.com/docker/docker/internal/testutils/storeutils" - "github.com/docker/docker/libnetwork/driverapi" "github.com/docker/docker/libnetwork/iptables" - "github.com/docker/docker/libnetwork/netlabel" "github.com/docker/docker/libnetwork/types" "github.com/vishvananda/netlink" "gotest.tools/v3/assert" @@ -22,41 +19,18 @@ import ( ) const ( - iptablesTestBridgeIP = "192.168.42.1" + defaultBridgeName = "testbridge" ) -// A testRegisterer implements the driverapi.Registerer interface. -type testRegisterer struct { - t *testing.T - d *driver -} - -func (r *testRegisterer) RegisterDriver(name string, di driverapi.Driver, _ driverapi.Capability) error { - if got, want := name, "bridge"; got != want { - r.t.Fatalf("got driver name %s, want %s", got, want) - } - d, ok := di.(*driver) - if !ok { - r.t.Fatalf("got driver type %T, want %T", di, &driver{}) - } - r.d = d - return nil -} - func TestProgramIPTable(t *testing.T) { // Create a test bridge with a basic bridge configuration (name + IPv4). defer netnsutils.SetupTestOSContext(t)() - nh, err := nlwrap.NewHandle() - if err != nil { - t.Fatal(err) - } - - createTestBridge(getBasicTestConfig(), &bridgeInterface{nlh: nh}, t) - _, err = iptables.GetIptable(iptables.IPv4).NewChain(DockerForwardChain, iptables.Filter) + _, err := iptables.GetIptable(iptables.IPv4).NewChain(DockerForwardChain, iptables.Filter) assert.NilError(t, err) // Store various iptables chain rules we care for. + const iptablesTestBridgeIP = "192.168.42.1" rules := []struct { rule iptables.Rule descr string @@ -71,7 +45,22 @@ func TestProgramIPTable(t *testing.T) { // Assert the chain rules' insertion and removal. for _, c := range rules { - assertIPTableChainProgramming(c.rule, c.descr, t) + // Add + if err := programChainRule(c.rule, c.descr, true); err != nil { + t.Fatalf("Failed to program iptable rule %s: %s", c.descr, err.Error()) + } + + if !c.rule.Exists() { + t.Fatalf("Failed to effectively program iptable rule: %s", c.descr) + } + + // Remove + if err := programChainRule(c.rule, c.descr, false); err != nil { + t.Fatalf("Failed to remove iptable rule %s: %s", c.descr, err.Error()) + } + if c.rule.Exists() { + t.Fatalf("Failed to effectively remove iptable rule: %s", c.descr) + } } } @@ -79,143 +68,68 @@ func TestSetupIPChains(t *testing.T) { // Create a test bridge with a basic bridge configuration (name + IPv4). defer netnsutils.SetupTestOSContext(t)() - nh, err := nlwrap.NewHandle() - if err != nil { - t.Fatal(err) - } - - driverconfig := configuration{ - EnableIPTables: true, - } - d := &driver{ - config: driverconfig, - } - assertChainConfig(d, t) - - config := getBasicTestConfig() - br := &bridgeInterface{nlh: nh} - createTestBridge(config, br, t) - - assertBridgeConfig(config, br, d, t) - - config.EnableIPMasquerade = true - assertBridgeConfig(config, br, d, t) - - config.EnableICC = true - assertBridgeConfig(config, br, d, t) - - config.EnableIPMasquerade = false - assertBridgeConfig(config, br, d, t) -} - -func getBasicTestConfig() *networkConfiguration { - config := &networkConfiguration{ - BridgeName: DefaultBridgeName, - EnableIPv4: true, - AddressIPv4: &net.IPNet{IP: net.ParseIP(iptablesTestBridgeIP), Mask: net.CIDRMask(16, 32)}, - } - return config -} - -func createTestBridge(config *networkConfiguration, br *bridgeInterface, t *testing.T) { - if err := setupDevice(config, br); err != nil { - t.Fatalf("Failed to create the testing Bridge: %s", err.Error()) - } - if err := setupBridgeIPv4(config, br); err != nil { - t.Fatalf("Failed to bring up the testing Bridge: %s", err.Error()) - } - if config.EnableIPv6 { - if err := setupBridgeIPv6(config, br); err != nil { - t.Fatalf("Failed to bring up the testing Bridge: %s", err.Error()) - } - } -} - -// Assert base function which pushes iptables chain rules on insertion and removal. -func assertIPTableChainProgramming(rule iptables.Rule, descr string, t *testing.T) { - t.Helper() - - // Add - if err := programChainRule(rule, descr, true); err != nil { - t.Fatalf("Failed to program iptable rule %s: %s", descr, err.Error()) - } - - if !rule.Exists() { - t.Fatalf("Failed to effectively program iptable rule: %s", descr) - } - - // Remove - if err := programChainRule(rule, descr, false); err != nil { - t.Fatalf("Failed to remove iptable rule %s: %s", descr, err.Error()) - } - if rule.Exists() { - t.Fatalf("Failed to effectively remove iptable rule: %s", descr) - } -} - -// Assert function which create chains. -func assertChainConfig(d *driver, t *testing.T) { - var err error - - err = setupIPChains(iptables.IPv4, !d.config.EnableUserlandProxy) + ipt := firewaller{IPv4: true} + err := ipt.init() assert.NilError(t, err) - if d.config.EnableIP6Tables { - err = setupIPChains(iptables.IPv6, !d.config.EnableUserlandProxy) - assert.NilError(t, err) - } -} - -// Assert function which pushes chains based on bridge config parameters. -func assertBridgeConfig(config *networkConfiguration, br *bridgeInterface, d *driver, t *testing.T) { - nw := bridgeNetwork{ - config: config, - driver: d, - bridge: br, + nc := networkConfig{ + IfName: defaultBridgeName, + Config4: networkConfigFam{ + Prefix: netip.MustParsePrefix("192.168.42.0/24"), + }, } - fwn, err := nw.newIptablesNetwork() - assert.NilError(t, err) - assert.Check(t, fwn != nil, "no firewaller network") + assertBridgeConfig(t, ipt, nc) + + nc.Masquerade = true + assertBridgeConfig(t, ipt, nc) + + nc.ICC = true + assertBridgeConfig(t, ipt, nc) + + nc.Masquerade = false + assertBridgeConfig(t, ipt, nc) } // Regression test for https://github.com/moby/moby/issues/46445 func TestSetupIP6TablesWithHostIPv4(t *testing.T) { defer netnsutils.SetupTestOSContext(t)() - d := newDriver(storeutils.NewTempStore(t)) - dc := &configuration{ - EnableIPTables: true, - EnableIP6Tables: true, + ipt := firewaller{ + IPv4: true, + IPv6: true, } - if err := d.configure(map[string]interface{}{netlabel.GenericData: dc}); err != nil { - t.Fatal(err) + err := ipt.init() + assert.NilError(t, err) + + nc := networkConfig{ + IfName: defaultBridgeName, + Masquerade: true, + Config4: networkConfigFam{ + HostIP: netip.MustParseAddr("192.0.2.2"), + Prefix: netip.MustParsePrefix("192.168.42.0/24"), + }, + Config6: networkConfigFam{ + Prefix: netip.MustParsePrefix("2001:db8::/64"), + }, } - nc := &networkConfiguration{ - BridgeName: DefaultBridgeName, - AddressIPv4: &net.IPNet{IP: net.ParseIP(iptablesTestBridgeIP), Mask: net.CIDRMask(16, 32)}, - EnableIPMasquerade: true, - EnableIPv4: true, - EnableIPv6: true, - AddressIPv6: &net.IPNet{IP: net.ParseIP("2001:db8::1"), Mask: net.CIDRMask(64, 128)}, - HostIPv4: net.ParseIP("192.0.2.2"), - } - nh, err := nlwrap.NewHandle() - if err != nil { - t.Fatal(err) - } - br := &bridgeInterface{nlh: nh} - createTestBridge(nc, br, t) - assertBridgeConfig(nc, br, d, t) + assertBridgeConfig(t, ipt, nc) +} + +// Assert function which pushes chains based on bridge config parameters. +func assertBridgeConfig(t *testing.T, ipt firewaller, nc networkConfig) { + t.Helper() + n, err := ipt.NewNetwork(nc) + assert.NilError(t, err) + err = n.delNetworkLevelRules() + assert.NilError(t, err) } func TestOutgoingNATRules(t *testing.T) { br := "br-nattest" - brIPv4 := &net.IPNet{IP: net.ParseIP(iptablesTestBridgeIP), Mask: net.CIDRMask(16, 32)} - brIPv6 := &net.IPNet{IP: net.ParseIP("2001:db8::1"), Mask: net.CIDRMask(64, 128)} - maskedBrIPv4 := &net.IPNet{IP: brIPv4.IP.Mask(brIPv4.Mask), Mask: brIPv4.Mask} - maskedBrIPv6 := &net.IPNet{IP: brIPv6.IP.Mask(brIPv6.Mask), Mask: brIPv6.Mask} - hostIPv4 := net.ParseIP("192.0.2.2") - hostIPv6 := net.ParseIP("2001:db8:1::1") + maskedBrIPv4 := netip.MustParsePrefix("192.168.42.1/16").Masked() + maskedBrIPv6 := netip.MustParsePrefix("2001:db8::1/64").Masked() + hostIPv4 := netip.MustParseAddr("192.0.2.2") + hostIPv6 := netip.MustParseAddr("2001:db8:1::1") for _, tc := range []struct { desc string enableIPTables bool @@ -223,8 +137,8 @@ func TestOutgoingNATRules(t *testing.T) { enableIPv4 bool enableIPv6 bool enableIPMasquerade bool - hostIPv4 net.IP - hostIPv6 net.IP + hostIPv4 netip.Addr + hostIPv6 netip.Addr // Hairpin NAT rules are not tested here because they are orthogonal to outgoing NAT. They // exist to support the port forwarding DNAT rules: without any port forwarding there would be // no need for any hairpin NAT rules, and when there is port forwarding then hairpin NAT rules @@ -350,45 +264,34 @@ func TestOutgoingNATRules(t *testing.T) { } { t.Run(tc.desc, func(t *testing.T) { defer netnsutils.SetupTestOSContext(t)() - dc := &configuration{ - EnableIPTables: tc.enableIPTables, - EnableIP6Tables: tc.enableIP6Tables, + + ipt := firewaller{ + IPv4: tc.enableIPTables, + IPv6: tc.enableIP6Tables, } - r := &testRegisterer{t: t} - if err := Register(r, storeutils.NewTempStore(t), map[string]interface{}{netlabel.GenericData: dc}); err != nil { - t.Fatal(err) - } - if r.d == nil { - t.Fatal("testRegisterer.RegisterDriver never called") - } - nc := &networkConfiguration{ - BridgeName: br, - AddressIPv4: brIPv4, - AddressIPv6: brIPv6, - EnableIPv4: tc.enableIPv4, - EnableIPv6: tc.enableIPv6, - EnableIPMasquerade: tc.enableIPMasquerade, - HostIPv4: tc.hostIPv4, - HostIPv6: tc.hostIPv6, - } - ipv4Data := []driverapi.IPAMData{{Pool: maskedBrIPv4, Gateway: brIPv4}} - ipv6Data := []driverapi.IPAMData{{Pool: maskedBrIPv6, Gateway: brIPv6}} - if !nc.EnableIPv4 { - nc.AddressIPv4 = nil - ipv4Data = nil - } - if !nc.EnableIPv6 { - nc.AddressIPv6 = nil - ipv6Data = nil - } - if err := r.d.CreateNetwork(context.Background(), "nattest", map[string]interface{}{netlabel.GenericData: nc}, nil, ipv4Data, ipv6Data); err != nil { - t.Fatal(err) + err := ipt.init() + assert.NilError(t, err) + + nc := networkConfig{ + IfName: br, + Masquerade: tc.enableIPMasquerade, + Config4: networkConfigFam{ + HostIP: tc.hostIPv4, + Prefix: maskedBrIPv4, + }, + Config6: networkConfigFam{ + HostIP: tc.hostIPv6, + Prefix: maskedBrIPv6, + }, } + n, err := ipt.NewNetwork(nc) + assert.NilError(t, err) + defer func() { - if err := r.d.DeleteNetwork("nattest"); err != nil { - t.Fatal(err) - } + err = n.delNetworkLevelRules() + assert.NilError(t, err) }() + // Log the contents of all chains to aid troubleshooting. for _, ipv := range []iptables.IPVersion{iptables.IPv4, iptables.IPv6} { ipt := iptables.GetIptable(ipv) From 8c36a22e79889cde6486e0e6e288aabda159f386 Mon Sep 17 00:00:00 2001 From: Rob Murray Date: Tue, 15 Apr 2025 11:13:56 +0100 Subject: [PATCH 3/5] Rename function insertMirroredWSL2Rule It's now shouldInsertMirroredWSL2Rule, because it's a test and doesn't do the insertion. Signed-off-by: Rob Murray --- libnetwork/drivers/bridge/setup_ip_tables_linux.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libnetwork/drivers/bridge/setup_ip_tables_linux.go b/libnetwork/drivers/bridge/setup_ip_tables_linux.go index b4aa9f195b..3061b7f793 100644 --- a/libnetwork/drivers/bridge/setup_ip_tables_linux.go +++ b/libnetwork/drivers/bridge/setup_ip_tables_linux.go @@ -822,10 +822,10 @@ func mirroredWSL2Workaround(ipv iptables.IPVersion, hairpin bool) error { if ipv != iptables.IPv4 { return nil } - return programChainRule(mirroredWSL2Rule(), "WSL2 loopback", insertMirroredWSL2Rule(hairpin)) + return programChainRule(mirroredWSL2Rule(), "WSL2 loopback", shouldInsertMirroredWSL2Rule(hairpin)) } -// insertMirroredWSL2Rule returns true if the NAT rule for mirrored WSL2 workaround +// shouldInsertMirroredWSL2Rule returns true if the NAT rule for mirrored WSL2 workaround // is required. It is required if: // - the userland proxy is running. If not, there's nothing on the host to catch // the packet, so the loopback0 rule as wouldn't be useful. However, without @@ -833,7 +833,7 @@ func mirroredWSL2Workaround(ipv iptables.IPVersion, hairpin bool) error { // running - no workaround is needed, the normal DNAT/masquerading works. // - and, the host Linux appears to be running under Windows WSL2 with mirrored // mode networking. -func insertMirroredWSL2Rule(hairpin bool) bool { +func shouldInsertMirroredWSL2Rule(hairpin bool) bool { if hairpin { return false } From 282b3f7b979999bcbfdece2668a6662c908fae9c Mon Sep 17 00:00:00 2001 From: Rob Murray Date: Tue, 15 Apr 2025 10:50:22 +0100 Subject: [PATCH 4/5] Move bridge driver iptables code into its own package Signed-off-by: Rob Murray --- libnetwork/drivers/bridge/bridge_linux.go | 93 ++----- .../drivers/bridge/bridge_linux_test.go | 120 ++++----- .../iptabler/iptabler.go} | 242 ++++++++++++------ .../iptabler/iptabler_test.go} | 137 +++++++--- .../bridge/{ => internal/iptabler}/link.go | 10 +- .../drivers/bridge/internal/iptabler/port.go | 240 +++++++++++++++++ .../drivers/bridge/port_mapping_linux.go | 227 ---------------- 7 files changed, 580 insertions(+), 489 deletions(-) rename libnetwork/drivers/bridge/{setup_ip_tables_linux.go => internal/iptabler/iptabler.go} (81%) rename libnetwork/drivers/bridge/{setup_ip_tables_linux_test.go => internal/iptabler/iptabler_test.go} (77%) rename libnetwork/drivers/bridge/{ => internal/iptabler}/link.go (74%) create mode 100644 libnetwork/drivers/bridge/internal/iptabler/port.go diff --git a/libnetwork/drivers/bridge/bridge_linux.go b/libnetwork/drivers/bridge/bridge_linux.go index 1b3c7405d5..3f7a9e0579 100644 --- a/libnetwork/drivers/bridge/bridge_linux.go +++ b/libnetwork/drivers/bridge/bridge_linux.go @@ -12,11 +12,11 @@ import ( "github.com/containerd/log" "github.com/docker/docker/errdefs" - "github.com/docker/docker/internal/modprobe" "github.com/docker/docker/internal/nlwrap" "github.com/docker/docker/internal/otelutil" "github.com/docker/docker/libnetwork/datastore" "github.com/docker/docker/libnetwork/driverapi" + "github.com/docker/docker/libnetwork/drivers/bridge/internal/iptabler" "github.com/docker/docker/libnetwork/drivers/bridge/internal/rlkclient" "github.com/docker/docker/libnetwork/internal/netiputil" "github.com/docker/docker/libnetwork/iptables" @@ -51,10 +51,10 @@ const ( const spanPrefix = "libnetwork.drivers.bridge" -type ( - iptableCleanFunc func() error - iptablesCleanFuncs []iptableCleanFunc -) +// DockerForwardChain is where libnetwork.programIngress puts Swarm's jump to DOCKER-INGRESS. +// +// FIXME(robmry) - it doesn't belong here. +const DockerForwardChain = iptabler.DockerForwardChain // configuration info for the "bridge" driver. type configuration struct { @@ -67,12 +67,6 @@ type configuration struct { Rootless bool } -type firewaller struct { - IPv4 bool - IPv6 bool - Hairpin bool -} - // networkConfiguration for network specific configuration type networkConfiguration struct { ID string @@ -144,7 +138,7 @@ type bridgeNetwork struct { config *networkConfiguration endpoints map[string]*bridgeEndpoint // key: endpoint id driver *driver // The network's driver - iptablesNetwork *iptablesNetwork + iptablesNetwork *iptabler.Network sync.Mutex } @@ -165,7 +159,7 @@ type driver struct { nlh nlwrap.Handle portDriverClient portDriverClient configNetwork sync.Mutex - firewaller firewaller + firewaller *iptabler.Iptabler sync.Mutex } @@ -401,7 +395,7 @@ func parseErr(label, value, errString string) error { return types.InvalidParameterErrorf("failed to parse %s value: %v (%s)", label, value, errString) } -func (n *bridgeNetwork) newIptablesNetwork() (*iptablesNetwork, error) { +func (n *bridgeNetwork) newIptablesNetwork() (*iptabler.Network, error) { config4, err := makeNetworkConfigFam(n.config.HostIPv4, n.bridge.bridgeIPv4, n.gwMode(iptables.IPv4)) if err != nil { return nil, err @@ -410,7 +404,7 @@ func (n *bridgeNetwork) newIptablesNetwork() (*iptablesNetwork, error) { if err != nil { return nil, err } - return n.driver.firewaller.NewNetwork(networkConfig{ + return n.driver.firewaller.NewNetwork(iptabler.NetworkConfig{ IfName: n.config.BridgeName, Internal: n.config.Internal, ICC: n.config.EnableICC, @@ -420,8 +414,8 @@ func (n *bridgeNetwork) newIptablesNetwork() (*iptablesNetwork, error) { }) } -func makeNetworkConfigFam(hostIP net.IP, bridgePrefix *net.IPNet, gwm gwMode) (networkConfigFam, error) { - c := networkConfigFam{ +func makeNetworkConfigFam(hostIP net.IP, bridgePrefix *net.IPNet, gwm gwMode) (iptabler.NetworkConfigFam, error) { + c := iptabler.NetworkConfigFam{ Routed: gwm.routed(), Unprotected: gwm.unprotected(), } @@ -429,14 +423,14 @@ func makeNetworkConfigFam(hostIP net.IP, bridgePrefix *net.IPNet, gwm gwMode) (n var ok bool c.HostIP, ok = netip.AddrFromSlice(hostIP) if !ok { - return networkConfigFam{}, fmt.Errorf("invalid host address %q", hostIP) + return iptabler.NetworkConfigFam{}, fmt.Errorf("invalid host address %q", hostIP) } c.HostIP = c.HostIP.Unmap() } if bridgePrefix != nil { p, ok := netiputil.ToPrefix(bridgePrefix) if !ok { - return networkConfigFam{}, fmt.Errorf("invalid bridge prefix %s", bridgePrefix) + return iptabler.NetworkConfigFam{}, fmt.Errorf("invalid bridge prefix %s", bridgePrefix) } c.Prefix = p.Masked() } @@ -507,12 +501,13 @@ func (d *driver) configure(option map[string]interface{}) error { return errdefs.InvalidParameter(fmt.Errorf("invalid configuration type (%T) passed", opt)) } - d.firewaller = firewaller{ + var err error + d.firewaller, err = iptabler.NewIptabler(iptabler.FirewallConfig{ IPv4: config.EnableIPTables, IPv6: config.EnableIP6Tables, Hairpin: !config.EnableUserlandProxy || config.UserlandProxyPath == "", - } - if err := d.firewaller.init(); err != nil { + }) + if err != nil { return err } iptables.OnReloaded(d.handleFirewalldReload) @@ -534,56 +529,6 @@ func (d *driver) configure(option map[string]interface{}) error { return d.initStore() } -func (fw *firewaller) init() error { - if fw.IPv4 { - removeIPChains(iptables.IPv4) - - if err := setupIPChains(iptables.IPv4, fw.Hairpin); err != nil { - return err - } - - // Make sure on firewall reload, first thing being re-played is chains creation - iptables.OnReloaded(func() { - log.G(context.TODO()).Debugf("Recreating iptables chains on firewall reload") - if err := setupIPChains(iptables.IPv4, fw.Hairpin); err != nil { - log.G(context.TODO()).WithError(err).Error("Error reloading iptables chains") - } - }) - } - - if fw.IPv6 { - if err := modprobe.LoadModules(context.TODO(), func() error { - iptable := iptables.GetIptable(iptables.IPv6) - _, err := iptable.Raw("-t", "filter", "-n", "-L", "FORWARD") - return err - }, "ip6_tables"); err != nil { - log.G(context.TODO()).WithError(err).Debug("Loading ip6_tables") - } - - removeIPChains(iptables.IPv6) - - err := setupIPChains(iptables.IPv6, fw.Hairpin) - if err != nil { - // If the chains couldn't be set up, it's probably because the kernel has no IPv6 - // support, or it doesn't have module ip6_tables loaded. It won't be possible to - // create IPv6 networks without enabling ip6_tables in the kernel, or disabling - // ip6tables in the daemon config. But, allow the daemon to start because IPv4 - // will work. So, log the problem, and continue. - log.G(context.TODO()).WithError(err).Warn("ip6tables is enabled, but cannot set up ip6tables chains") - } else { - // Make sure on firewall reload, first thing being re-played is chains creation - iptables.OnReloaded(func() { - log.G(context.TODO()).Debugf("Recreating ip6tables chains on firewall reload") - if err := setupIPChains(iptables.IPv6, fw.Hairpin); err != nil { - log.G(context.TODO()).WithError(err).Error("Error reloading ip6tables chains") - } - }) - } - } - - return nil -} - func (d *driver) getNetwork(id string) (*bridgeNetwork, error) { d.Lock() defer d.Unlock() @@ -1051,7 +996,7 @@ func (d *driver) deleteNetwork(nid string) error { // Don't delete the bridge interface if it was not created by libnetwork. } - if err := n.iptablesNetwork.delNetworkLevelRules(); err != nil { + if err := n.iptablesNetwork.DelNetworkLevelRules(); err != nil { log.G(context.TODO()).WithError(err).Warnf("Failed to clean iptables rules for bridge network") } @@ -1676,7 +1621,7 @@ func (d *driver) handleFirewalldReloadNw(nid string) { return } - if err := nw.iptablesNetwork.reapplyNetworkLevelRules(); err != nil { + if err := nw.iptablesNetwork.ReapplyNetworkLevelRules(); err != nil { log.G(context.Background()).WithFields(log.Fields{ "nid": nw.id, "error": err, diff --git a/libnetwork/drivers/bridge/bridge_linux_test.go b/libnetwork/drivers/bridge/bridge_linux_test.go index b354903d3b..4d13b9e501 100644 --- a/libnetwork/drivers/bridge/bridge_linux_test.go +++ b/libnetwork/drivers/bridge/bridge_linux_test.go @@ -32,6 +32,13 @@ import ( "gotest.tools/v3/icmd" ) +const ( + // FIXME(robmry) - remove these and make the tests work for non-iptables firewalls. + dockerChain = "DOCKER" + isolationChain1 = "DOCKER-ISOLATION-STAGE-1" + isolationChain2 = "DOCKER-ISOLATION-STAGE-2" +) + func TestEndpointMarshalling(t *testing.T) { ip1, _ := types.ParseCIDR("172.22.0.9/16") ip2, _ := types.ParseCIDR("2001:db8::9") @@ -301,6 +308,8 @@ func TestCreateFullOptions(t *testing.T) { func TestCreateNoConfig(t *testing.T) { defer netnsutils.SetupTestOSContext(t)() d := newDriver(storeutils.NewTempStore(t)) + err := d.configure(nil) + assert.NilError(t, err) netconfig := &networkConfiguration{BridgeName: DefaultBridgeName, EnableIPv4: true} genericOption := make(map[string]interface{}) @@ -613,17 +622,17 @@ func TestCreateMultipleNetworks(t *testing.T) { // Verify the network isolation rules are installed for each network func verifyV4INCEntries(networks map[string]*bridgeNetwork, t *testing.T) { iptable := iptables.GetIptable(iptables.IPv4) - out1, err := iptable.Raw("-S", IsolationChain1) + out1, err := iptable.Raw("-S", isolationChain1) if err != nil { t.Fatal(err) } - out2, err := iptable.Raw("-S", IsolationChain2) + out2, err := iptable.Raw("-S", isolationChain2) if err != nil { t.Fatal(err) } for _, n := range networks { - re := regexp.MustCompile(fmt.Sprintf("-i %s ! -o %s -j %s", n.config.BridgeName, n.config.BridgeName, IsolationChain2)) + re := regexp.MustCompile(fmt.Sprintf("-i %s ! -o %s -j %s", n.config.BridgeName, n.config.BridgeName, isolationChain2)) matches := re.FindAllString(string(out1[:]), -1) if len(matches) != 1 { t.Fatalf("Cannot find expected inter-network isolation rules in IP Tables for network %s:\n%s.", n.id, string(out1[:])) @@ -964,7 +973,7 @@ func TestLinkContainers(t *testing.T) { t.Fatalf("Failed to program external connectivity: %v", err) } - out, _ := iptable.Raw("-L", DockerChain) + out, _ := iptable.Raw("-L", dockerChain) for _, pm := range exposedPorts { regex := fmt.Sprintf("%s dpt:%d", pm.Proto.String(), pm.Port) re := regexp.MustCompile(regex) @@ -990,7 +999,7 @@ func TestLinkContainers(t *testing.T) { t.Fatal("Failed to unlink ep1 and ep2") } - out, _ = iptable.Raw("-L", DockerChain) + out, _ = iptable.Raw("-L", dockerChain) for _, pm := range exposedPorts { regex := fmt.Sprintf("%s dpt:%d", pm.Proto.String(), pm.Port) re := regexp.MustCompile(regex) @@ -1018,7 +1027,7 @@ func TestLinkContainers(t *testing.T) { } err = d.ProgramExternalConnectivity(context.Background(), "net1", "ep2", sbOptions) if err != nil { - out, _ = iptable.Raw("-L", DockerChain) + out, _ = iptable.Raw("-L", dockerChain) for _, pm := range exposedPorts { regex := fmt.Sprintf("%s dpt:%d", pm.Proto.String(), pm.Port) re := regexp.MustCompile(regex) @@ -1243,57 +1252,6 @@ func TestSetDefaultGw(t *testing.T) { } } -func TestCleanupIptableRules(t *testing.T) { - defer netnsutils.SetupTestOSContext(t)() - bridgeChains := []struct { - name string - table iptables.Table - expRemoved bool - }{ - {name: DockerChain, table: iptables.Nat, expRemoved: true}, - // The filter-FORWARD chain has references to DockerChain and IsolationChain1, - // so the chains won't be removed - but they should be flushed. (This has - // long/always been the case for the daemon, its filter-FORWARD rules aren't - // removed.) - {name: DockerChain, table: iptables.Filter}, - {name: IsolationChain1, table: iptables.Filter}, - } - - ipVersions := []iptables.IPVersion{iptables.IPv4, iptables.IPv6} - - for _, version := range ipVersions { - err := setupIPChains(version, true) - assert.NilError(t, err, "version:%s", version) - - iptable := iptables.GetIptable(version) - for _, chainInfo := range bridgeChains { - exists := iptable.ExistChain(chainInfo.name, chainInfo.table) - assert.Check(t, exists, "version:%s chain:%s table:%v", - version, chainInfo.name, chainInfo.table) - } - - // Insert RETURN rules so that there's something to flush. - for _, chainInfo := range bridgeChains { - out, err := iptable.Raw("-t", string(chainInfo.table), "-A", chainInfo.name, "-j", "RETURN") - assert.NilError(t, err, "version:%s chain:%s table:%v out:%s", - version, chainInfo.name, chainInfo.table, out) - } - - removeIPChains(version) - - for _, chainInfo := range bridgeChains { - exists := iptable.Exists(chainInfo.table, chainInfo.name, "-A", chainInfo.name, "-j", "RETURN") - assert.Check(t, !exists, "version:%s chain:%s table:%v", - version, chainInfo.name, chainInfo.table) - if chainInfo.expRemoved { - exists := iptable.ExistChain(chainInfo.name, chainInfo.table) - assert.Check(t, !exists, "version:%s chain:%s table:%v", - version, chainInfo.name, chainInfo.table) - } - } - } -} - func TestCreateWithExistingBridge(t *testing.T) { defer netnsutils.SetupTestOSContext(t)() d := newDriver(storeutils.NewTempStore(t)) @@ -1407,3 +1365,51 @@ func TestCreateParallel(t *testing.T) { t.Fatalf("Success should be 1 instead: %d", success) } } + +// Regression test for https://github.com/moby/moby/issues/46445 +func TestSetupIP6TablesWithHostIPv4(t *testing.T) { + defer netnsutils.SetupTestOSContext(t)() + d := newDriver(storeutils.NewTempStore(t)) + dc := &configuration{ + EnableIPTables: true, + EnableIP6Tables: true, + } + if err := d.configure(map[string]interface{}{netlabel.GenericData: dc}); err != nil { + t.Fatal(err) + } + nc := &networkConfiguration{ + BridgeName: DefaultBridgeName, + AddressIPv4: &net.IPNet{IP: net.ParseIP("192.168.42.1"), Mask: net.CIDRMask(16, 32)}, + EnableIPMasquerade: true, + EnableIPv4: true, + EnableIPv6: true, + AddressIPv6: &net.IPNet{IP: net.ParseIP("2001:db8::1"), Mask: net.CIDRMask(64, 128)}, + HostIPv4: net.ParseIP("192.0.2.2"), + } + + // Create test bridge. + nh, err := nlwrap.NewHandle() + if err != nil { + t.Fatal(err) + } + br := &bridgeInterface{nlh: nh} + if err := setupDevice(nc, br); err != nil { + t.Fatalf("Failed to create the testing Bridge: %s", err.Error()) + } + if err := setupBridgeIPv4(nc, br); err != nil { + t.Fatalf("Failed to bring up the testing Bridge: %s", err.Error()) + } + if err := setupBridgeIPv6(nc, br); err != nil { + t.Fatalf("Failed to bring up the testing Bridge: %s", err.Error()) + } + + // Check firewall configuration succeeds. + nw := bridgeNetwork{ + config: nc, + driver: d, + bridge: br, + } + fwn, err := nw.newIptablesNetwork() + assert.NilError(t, err) + assert.Check(t, fwn != nil, "no firewaller network") +} diff --git a/libnetwork/drivers/bridge/setup_ip_tables_linux.go b/libnetwork/drivers/bridge/internal/iptabler/iptabler.go similarity index 81% rename from libnetwork/drivers/bridge/setup_ip_tables_linux.go rename to libnetwork/drivers/bridge/internal/iptabler/iptabler.go index 3061b7f793..bd424fd2ce 100644 --- a/libnetwork/drivers/bridge/setup_ip_tables_linux.go +++ b/libnetwork/drivers/bridge/internal/iptabler/iptabler.go @@ -1,4 +1,6 @@ -package bridge +//go:build linux + +package iptabler import ( "context" @@ -9,17 +11,21 @@ import ( "github.com/containerd/log" "github.com/docker/docker/errdefs" + "github.com/docker/docker/internal/modprobe" "github.com/docker/docker/internal/nlwrap" "github.com/docker/docker/libnetwork/iptables" "github.com/vishvananda/netlink" ) -// DockerChain: DOCKER iptable chain name const ( - DockerChain = "DOCKER" + // dockerChain: DOCKER iptable chain name + dockerChain = "DOCKER" + // DockerForwardChain contains Docker's filter-FORWARD rules. + // + // FIXME(robmry) - only exported because it's used to set up the jump to swarm's DOCKER-INGRESS chain. DockerForwardChain = "DOCKER-FORWARD" - DockerBridgeChain = "DOCKER-BRIDGE" - DockerCTChain = "DOCKER-CT" + dockerBridgeChain = "DOCKER-BRIDGE" + dockerCTChain = "DOCKER-CT" // Isolation between bridge networks is achieved in two stages by means // of the following two chains in the filter table. The first chain matches @@ -30,39 +36,100 @@ const ( // bridge. A positive match identifies a packet originated from one bridge // network's bridge destined to another bridge network's bridge and will // result in the packet being dropped. No match returns to the parent chain. - - IsolationChain1 = "DOCKER-ISOLATION-STAGE-1" - IsolationChain2 = "DOCKER-ISOLATION-STAGE-2" + isolationChain1 = "DOCKER-ISOLATION-STAGE-1" + isolationChain2 = "DOCKER-ISOLATION-STAGE-2" ) +type FirewallConfig struct { + IPv4 bool + IPv6 bool + Hairpin bool +} + +type Iptabler struct { + FirewallConfig +} + // Path to the executable installed in Linux under WSL2 that reports on // WSL config. https://github.com/microsoft/WSL/releases/tag/2.0.4 // Can be modified by tests. var wslinfoPath = "/usr/bin/wslinfo" +func NewIptabler(config FirewallConfig) (*Iptabler, error) { + ipt := &Iptabler{FirewallConfig: config} + + if ipt.IPv4 { + removeIPChains(iptables.IPv4) + + if err := setupIPChains(iptables.IPv4, ipt.Hairpin); err != nil { + return nil, err + } + + // Make sure on firewall reload, first thing being re-played is chains creation + iptables.OnReloaded(func() { + log.G(context.TODO()).Debugf("Recreating iptables chains on firewall reload") + if err := setupIPChains(iptables.IPv4, ipt.Hairpin); err != nil { + log.G(context.TODO()).WithError(err).Error("Error reloading iptables chains") + } + }) + } + + if ipt.IPv6 { + if err := modprobe.LoadModules(context.TODO(), func() error { + iptable := iptables.GetIptable(iptables.IPv6) + _, err := iptable.Raw("-t", "filter", "-n", "-L", "FORWARD") + return err + }, "ip6_tables"); err != nil { + log.G(context.TODO()).WithError(err).Debug("Loading ip6_tables") + } + + removeIPChains(iptables.IPv6) + + err := setupIPChains(iptables.IPv6, ipt.Hairpin) + if err != nil { + // If the chains couldn't be set up, it's probably because the kernel has no IPv6 + // support, or it doesn't have module ip6_tables loaded. It won't be possible to + // create IPv6 networks without enabling ip6_tables in the kernel, or disabling + // ip6tables in the daemon config. But, allow the daemon to start because IPv4 + // will work. So, log the problem, and continue. + log.G(context.TODO()).WithError(err).Warn("ip6tables is enabled, but cannot set up ip6tables chains") + } else { + // Make sure on firewall reload, first thing being re-played is chains creation + iptables.OnReloaded(func() { + log.G(context.TODO()).Debugf("Recreating ip6tables chains on firewall reload") + if err := setupIPChains(iptables.IPv6, ipt.Hairpin); err != nil { + log.G(context.TODO()).WithError(err).Error("Error reloading ip6tables chains") + } + }) + } + } + + return ipt, nil +} + func setupIPChains(version iptables.IPVersion, hairpin bool) (retErr error) { iptable := iptables.GetIptable(version) - _, err := iptable.NewChain(DockerChain, iptables.Nat) + _, err := iptable.NewChain(dockerChain, iptables.Nat) if err != nil { - return fmt.Errorf("failed to create NAT chain %s: %v", DockerChain, err) + return fmt.Errorf("failed to create NAT chain %s: %v", dockerChain, err) } defer func() { if retErr != nil { - if err := iptable.RemoveExistingChain(DockerChain, iptables.Nat); err != nil { - log.G(context.TODO()).Warnf("failed on removing iptables NAT chain %s on cleanup: %v", DockerChain, err) + if err := iptable.RemoveExistingChain(dockerChain, iptables.Nat); err != nil { + log.G(context.TODO()).Warnf("failed on removing iptables NAT chain %s on cleanup: %v", dockerChain, err) } } }() - _, err = iptable.NewChain(DockerChain, iptables.Filter) + _, err = iptable.NewChain(dockerChain, iptables.Filter) if err != nil { - return fmt.Errorf("failed to create FILTER chain %s: %v", DockerChain, err) + return fmt.Errorf("failed to create FILTER chain %s: %v", dockerChain, err) } defer func() { if retErr != nil { - if err := iptable.RemoveExistingChain(DockerChain, iptables.Filter); err != nil { - log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", DockerChain, err) + if err := iptable.RemoveExistingChain(dockerChain, iptables.Filter); err != nil { + log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", dockerChain, err) } } }() @@ -79,50 +146,50 @@ func setupIPChains(version iptables.IPVersion, hairpin bool) (retErr error) { } }() - _, err = iptable.NewChain(DockerBridgeChain, iptables.Filter) + _, err = iptable.NewChain(dockerBridgeChain, iptables.Filter) if err != nil { - return fmt.Errorf("failed to create FILTER chain %s: %v", DockerBridgeChain, err) + return fmt.Errorf("failed to create FILTER chain %s: %v", dockerBridgeChain, err) } defer func() { if retErr != nil { - if err := iptable.RemoveExistingChain(DockerBridgeChain, iptables.Filter); err != nil { - log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", DockerBridgeChain, err) + if err := iptable.RemoveExistingChain(dockerBridgeChain, iptables.Filter); err != nil { + log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", dockerBridgeChain, err) } } }() - _, err = iptable.NewChain(DockerCTChain, iptables.Filter) + _, err = iptable.NewChain(dockerCTChain, iptables.Filter) if err != nil { - return fmt.Errorf("failed to create FILTER chain %s: %v", DockerCTChain, err) + return fmt.Errorf("failed to create FILTER chain %s: %v", dockerCTChain, err) } defer func() { if retErr != nil { - if err := iptable.RemoveExistingChain(DockerCTChain, iptables.Filter); err != nil { - log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", DockerCTChain, err) + if err := iptable.RemoveExistingChain(dockerCTChain, iptables.Filter); err != nil { + log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", dockerCTChain, err) } } }() - _, err = iptable.NewChain(IsolationChain1, iptables.Filter) + _, err = iptable.NewChain(isolationChain1, iptables.Filter) if err != nil { return fmt.Errorf("failed to create FILTER isolation chain: %v", err) } defer func() { if retErr != nil { - if err := iptable.RemoveExistingChain(IsolationChain1, iptables.Filter); err != nil { - log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", IsolationChain1, err) + if err := iptable.RemoveExistingChain(isolationChain1, iptables.Filter); err != nil { + log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", isolationChain1, err) } } }() - _, err = iptable.NewChain(IsolationChain2, iptables.Filter) + _, err = iptable.NewChain(isolationChain2, iptables.Filter) if err != nil { return fmt.Errorf("failed to create FILTER isolation chain: %v", err) } defer func() { if retErr != nil { - if err := iptable.RemoveExistingChain(IsolationChain2, iptables.Filter); err != nil { - log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", IsolationChain2, err) + if err := iptable.RemoveExistingChain(isolationChain2, iptables.Filter); err != nil { + log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", isolationChain2, err) } } }() @@ -144,13 +211,13 @@ func setupIPChains(version iptables.IPVersion, hairpin bool) (retErr error) { if err := iptable.EnsureJumpRule("FORWARD", DockerForwardChain); err != nil { return err } - if err := iptable.EnsureJumpRule(DockerForwardChain, DockerBridgeChain); err != nil { + if err := iptable.EnsureJumpRule(DockerForwardChain, dockerBridgeChain); err != nil { return err } - if err := iptable.EnsureJumpRule(DockerForwardChain, IsolationChain1); err != nil { + if err := iptable.EnsureJumpRule(DockerForwardChain, isolationChain1); err != nil { return err } - if err := iptable.EnsureJumpRule(DockerForwardChain, DockerCTChain); err != nil { + if err := iptable.EnsureJumpRule(DockerForwardChain, dockerCTChain); err != nil { return err } @@ -163,12 +230,12 @@ func setupIPChains(version iptables.IPVersion, hairpin bool) (retErr error) { if version == iptables.IPv6 { ipsetName = "docker-ext-bridges-v6" } - if err := iptable.DeleteJumpRule("FORWARD", DockerChain, + if err := iptable.DeleteJumpRule("FORWARD", dockerChain, "-m", "set", "--match-set", ipsetName, "dst"); err != nil { log.G(context.TODO()).WithFields(log.Fields{"error": err, "set": ipsetName}).Debug( "deleting legacy ipset dest match rule") } - if err := iptable.DeleteJumpRule("FORWARD", IsolationChain1); err != nil { + if err := iptable.DeleteJumpRule("FORWARD", isolationChain1); err != nil { return err } if err := iptable.DeleteJumpRule("FORWARD", "ACCEPT", @@ -182,54 +249,59 @@ func setupIPChains(version iptables.IPVersion, hairpin bool) (retErr error) { return nil } -type networkConfigFam struct { +type ( + iptableCleanFunc func() error + iptablesCleanFuncs []iptableCleanFunc +) + +type NetworkConfigFam struct { HostIP netip.Addr Prefix netip.Prefix Routed bool Unprotected bool } -type networkConfig struct { +type NetworkConfig struct { IfName string Internal bool ICC bool Masquerade bool - Config4 networkConfigFam - Config6 networkConfigFam + Config4 NetworkConfigFam + Config6 NetworkConfigFam } -type iptablesNetwork struct { - networkConfig - fw *firewaller +type Network struct { + NetworkConfig + ipt *Iptabler cleanFuncs iptablesCleanFuncs } -func (fw *firewaller) NewNetwork(nc networkConfig) (_ *iptablesNetwork, retErr error) { - n := &iptablesNetwork{ - fw: fw, - networkConfig: nc, +func (ipt *Iptabler) NewNetwork(nc NetworkConfig) (_ *Network, retErr error) { + n := &Network{ + ipt: ipt, + NetworkConfig: nc, } defer func() { if retErr != nil { - if err := n.delNetworkLevelRules(); err != nil { + if err := n.DelNetworkLevelRules(); err != nil { log.G(context.TODO()).WithError(err).Warnf("Failed to delete network level rules following earlier error") } } }() - if err := n.reapplyNetworkLevelRules(); err != nil { + if err := n.ReapplyNetworkLevelRules(); err != nil { return nil, err } return n, nil } -func (n *iptablesNetwork) reapplyNetworkLevelRules() error { - if n.fw.IPv4 { +func (n *Network) ReapplyNetworkLevelRules() error { + if n.ipt.IPv4 { if err := n.configure(iptables.IPv4, n.Config4); err != nil { return err } } - if n.fw.IPv6 { + if n.ipt.IPv6 { if err := n.configure(iptables.IPv6, n.Config6); err != nil { return err } @@ -237,7 +309,7 @@ func (n *iptablesNetwork) reapplyNetworkLevelRules() error { return nil } -func (n *iptablesNetwork) delNetworkLevelRules() error { +func (n *Network) DelNetworkLevelRules() error { var errs []error for _, cleanFunc := range n.cleanFuncs { if err := cleanFunc(); err != nil { @@ -248,7 +320,7 @@ func (n *iptablesNetwork) delNetworkLevelRules() error { return errors.Join(errs...) } -func (n *iptablesNetwork) configure(ipv iptables.IPVersion, conf networkConfigFam) error { +func (n *Network) configure(ipv iptables.IPVersion, conf NetworkConfigFam) error { if !conf.Prefix.IsValid() { // Delete INC rules, in case they were created by a 28.0.0 daemon that didn't check // whether the network had iptables/ip6tables enabled. @@ -261,11 +333,11 @@ func (n *iptablesNetwork) configure(ipv iptables.IPVersion, conf networkConfigFa return nil } -func (n *iptablesNetwork) registerCleanFunc(clean iptableCleanFunc) { +func (n *Network) registerCleanFunc(clean iptableCleanFunc) { n.cleanFuncs = append(n.cleanFuncs, clean) } -func (n *iptablesNetwork) setupIPTables(ipVersion iptables.IPVersion, config networkConfigFam) error { +func (n *Network) setupIPTables(ipVersion iptables.IPVersion, config NetworkConfigFam) error { if n.Internal { if err := setupInternalNetworkRules(n.IfName, config.Prefix, n.ICC, true); err != nil { return fmt.Errorf("Failed to Setup IP tables: %w", err) @@ -303,7 +375,7 @@ func (n *iptablesNetwork) setupIPTables(ipVersion iptables.IPVersion, config net return setDefaultForwardRule(ipVersion, n.IfName, config.Unprotected, false) }) - ctRule := iptables.Rule{IPVer: ipVersion, Table: iptables.Filter, Chain: DockerCTChain, Args: []string{ + ctRule := iptables.Rule{IPVer: ipVersion, Table: iptables.Filter, Chain: dockerCTChain, Args: []string{ "-o", n.IfName, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT", @@ -314,9 +386,9 @@ func (n *iptablesNetwork) setupIPTables(ipVersion iptables.IPVersion, config net n.registerCleanFunc(func() error { return appendOrDelChainRule(ctRule, "bridge ct related", false) }) - jumpToDockerRule := iptables.Rule{IPVer: ipVersion, Table: iptables.Filter, Chain: DockerBridgeChain, Args: []string{ + jumpToDockerRule := iptables.Rule{IPVer: ipVersion, Table: iptables.Filter, Chain: dockerBridgeChain, Args: []string{ "-o", n.IfName, - "-j", DockerChain, + "-j", dockerChain, }} if err := appendOrDelChainRule(jumpToDockerRule, "jump to docker", true); err != nil { return err @@ -342,7 +414,7 @@ func setICMP(ipv iptables.IPVersion, bridgeName string, enable bool) error { if ipv == iptables.IPv6 { icmpProto = "icmpv6" } - icmpRule := iptables.Rule{IPVer: ipv, Table: iptables.Filter, Chain: DockerChain, Args: []string{ + icmpRule := iptables.Rule{IPVer: ipv, Table: iptables.Filter, Chain: dockerChain, Args: []string{ "-o", bridgeName, "-p", icmpProto, "-j", "ACCEPT", @@ -354,7 +426,7 @@ func addNATJumpRules(ipVer iptables.IPVersion, hairpinMode, enable bool) error { preroute := iptables.Rule{IPVer: ipVer, Table: iptables.Nat, Chain: "PREROUTING", Args: []string{ "-m", "addrtype", "--dst-type", "LOCAL", - "-j", DockerChain, + "-j", dockerChain, }} if enable { if err := preroute.Append(); err != nil { @@ -369,7 +441,7 @@ func addNATJumpRules(ipVer iptables.IPVersion, hairpinMode, enable bool) error { output := iptables.Rule{IPVer: ipVer, Table: iptables.Nat, Chain: "OUTPUT", Args: []string{ "-m", "addrtype", "--dst-type", "LOCAL", - "-j", DockerChain, + "-j", dockerChain, }} if !hairpinMode { output.Args = append(output.Args, "!", "--dst", loopbackAddress(ipVer)) @@ -396,14 +468,14 @@ func deleteLegacyFilterRules(ipVer iptables.IPVersion, bridgeName string) error // These rules have been replaced by an ipset-matching rule. link := []string{ "-o", bridgeName, - "-j", DockerChain, + "-j", dockerChain, } if iptable.Exists(iptables.Filter, "FORWARD", link...) { del := append([]string{string(iptables.Delete), "FORWARD"}, link...) if output, err := iptable.Raw(del...); err != nil { return err } else if len(output) != 0 { - return fmt.Errorf("could not delete linking rule from %s-%s: %s", iptables.Filter, DockerChain, output) + return fmt.Errorf("could not delete linking rule from %s-%s: %s", iptables.Filter, dockerChain, output) } } @@ -420,7 +492,7 @@ func deleteLegacyFilterRules(ipVer iptables.IPVersion, bridgeName string) error if output, err := iptable.Raw(del...); err != nil { return err } else if len(output) != 0 { - return fmt.Errorf("could not delete establish rule from %s-%s: %s", iptables.Filter, DockerChain, output) + return fmt.Errorf("could not delete establish rule from %s-%s: %s", iptables.Filter, dockerChain, output) } } @@ -453,7 +525,7 @@ func setDefaultForwardRule(ipVersion iptables.IPVersion, ifName string, unprotec action = "ACCEPT" } - rule := iptables.Rule{IPVer: ipVersion, Table: iptables.Filter, Chain: DockerChain, Args: []string{ + rule := iptables.Rule{IPVer: ipVersion, Table: iptables.Filter, Chain: dockerChain, Args: []string{ "!", "-i", ifName, "-o", ifName, "-j", action, @@ -467,7 +539,7 @@ func setDefaultForwardRule(ipVersion iptables.IPVersion, ifName string, unprotec return nil } -func (n *iptablesNetwork) setupNonInternalNetworkRules(ipVer iptables.IPVersion, config networkConfigFam, enable bool) error { +func (n *Network) setupNonInternalNetworkRules(ipVer iptables.IPVersion, config NetworkConfigFam, enable bool) error { var natArgs, hpNatArgs []string if config.HostIP.IsValid() { // The user wants IPv4/IPv6 SNAT with the given address. @@ -501,8 +573,8 @@ func (n *iptablesNetwork) setupNonInternalNetworkRules(ipVer iptables.IPVersion, // enable access to ports published by containers in the same network. But, the INC rules // will block access to that published port from containers in other networks. (However, // users may add a rule to DOCKER-USER to work around the INC rules if needed.) - if !n.fw.Hairpin { - skipDNAT := iptables.Rule{IPVer: ipVer, Table: iptables.Nat, Chain: DockerChain, Args: []string{ + if !n.ipt.Hairpin { + skipDNAT := iptables.Rule{IPVer: ipVer, Table: iptables.Nat, Chain: dockerChain, Args: []string{ "-i", n.IfName, "-j", "RETURN", }} @@ -514,7 +586,7 @@ func (n *iptablesNetwork) setupNonInternalNetworkRules(ipVer iptables.IPVersion, // In hairpin mode, masquerade traffic from localhost. If hairpin is disabled or if we're tearing down // that bridge, make sure the iptables rule isn't lying around. - if err := programChainRule(hpNatRule, "MASQ LOCAL HOST", enable && n.fw.Hairpin); err != nil { + if err := programChainRule(hpNatRule, "MASQ LOCAL HOST", enable && n.ipt.Hairpin); err != nil { return err } @@ -656,7 +728,7 @@ func setINC(version iptables.IPVersion, iface string, routed, enable bool) (retE // Anything is allowed into a routed network at this stage, so RETURN. Port // filtering rules in the DOCKER chain will drop anything that's not destined // for an open port. - if err := iptable.ProgramRule(iptables.Filter, IsolationChain1, actionI, []string{ + if err := iptable.ProgramRule(iptables.Filter, isolationChain1, actionI, []string{ "-o", iface, "-j", "RETURN", }); err != nil { @@ -667,7 +739,7 @@ func setINC(version iptables.IPVersion, iface string, routed, enable bool) (retE } // Allow responses from the routed network into whichever network made the request. - if err := iptable.ProgramRule(iptables.Filter, IsolationChain1, actionI, []string{ + if err := iptable.ProgramRule(iptables.Filter, isolationChain1, actionI, []string{ "-i", iface, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT", @@ -679,10 +751,10 @@ func setINC(version iptables.IPVersion, iface string, routed, enable bool) (retE } } - if err := iptable.ProgramRule(iptables.Filter, IsolationChain1, actionA, []string{ + if err := iptable.ProgramRule(iptables.Filter, isolationChain1, actionA, []string{ "-i", iface, "!", "-o", iface, - "-j", IsolationChain2, + "-j", isolationChain2, }); err != nil { log.G(context.TODO()).WithError(err).Warnf("Failed to %s inter-network communication rule", actionMsg) if enable { @@ -690,7 +762,7 @@ func setINC(version iptables.IPVersion, iface string, routed, enable bool) (retE } } - if err := iptable.ProgramRule(iptables.Filter, IsolationChain2, actionI, []string{ + if err := iptable.ProgramRule(iptables.Filter, isolationChain2, actionI, []string{ "-o", iface, "-j", "DROP", }); err != nil { @@ -714,13 +786,13 @@ func removeIPChains(version iptables.IPVersion) { // Remove chains for _, chainInfo := range []iptables.ChainInfo{ - {Name: DockerChain, Table: iptables.Nat, IPVersion: version}, - {Name: DockerChain, Table: iptables.Filter, IPVersion: version}, + {Name: dockerChain, Table: iptables.Nat, IPVersion: version}, + {Name: dockerChain, Table: iptables.Filter, IPVersion: version}, {Name: DockerForwardChain, Table: iptables.Filter, IPVersion: version}, - {Name: DockerBridgeChain, Table: iptables.Filter, IPVersion: version}, - {Name: DockerCTChain, Table: iptables.Filter, IPVersion: version}, - {Name: IsolationChain1, Table: iptables.Filter, IPVersion: version}, - {Name: IsolationChain2, Table: iptables.Filter, IPVersion: version}, + {Name: dockerBridgeChain, Table: iptables.Filter, IPVersion: version}, + {Name: dockerCTChain, Table: iptables.Filter, IPVersion: version}, + {Name: isolationChain1, Table: iptables.Filter, IPVersion: version}, + {Name: isolationChain2, Table: iptables.Filter, IPVersion: version}, {Name: oldIsolationChain, Table: iptables.Filter, IPVersion: version}, } { if err := chainInfo.Remove(); err != nil { @@ -749,13 +821,13 @@ func setupInternalNetworkRules(bridgeIface string, prefix netip.Prefix, icc, ins inDropRule = iptables.Rule{ IPVer: version, Table: iptables.Filter, - Chain: IsolationChain1, + Chain: isolationChain1, Args: []string{"-i", bridgeIface, "!", "-d", prefix.String(), "-j", "DROP"}, } outDropRule = iptables.Rule{ IPVer: version, Table: iptables.Filter, - Chain: IsolationChain1, + Chain: isolationChain1, Args: []string{"-o", bridgeIface, "!", "-s", prefix.String(), "-j", "DROP"}, } } else { @@ -763,13 +835,13 @@ func setupInternalNetworkRules(bridgeIface string, prefix netip.Prefix, icc, ins inDropRule = iptables.Rule{ IPVer: version, Table: iptables.Filter, - Chain: IsolationChain1, + Chain: isolationChain1, Args: []string{"-i", bridgeIface, "!", "-o", bridgeIface, "!", "-d", prefix.String(), "-j", "DROP"}, } outDropRule = iptables.Rule{ IPVer: version, Table: iptables.Filter, - Chain: IsolationChain1, + Chain: isolationChain1, Args: []string{"!", "-i", bridgeIface, "-o", bridgeIface, "!", "-s", prefix.String(), "-j", "DROP"}, } } @@ -865,7 +937,7 @@ func mirroredWSL2Rule() iptables.Rule { return iptables.Rule{ IPVer: iptables.IPv4, Table: iptables.Nat, - Chain: DockerChain, + Chain: dockerChain, Args: []string{"-i", "loopback0", "-d", "127.0.0.0/8", "-j", "RETURN"}, } } diff --git a/libnetwork/drivers/bridge/setup_ip_tables_linux_test.go b/libnetwork/drivers/bridge/internal/iptabler/iptabler_test.go similarity index 77% rename from libnetwork/drivers/bridge/setup_ip_tables_linux_test.go rename to libnetwork/drivers/bridge/internal/iptabler/iptabler_test.go index dcb8c9ffdf..70e9cdd4da 100644 --- a/libnetwork/drivers/bridge/setup_ip_tables_linux_test.go +++ b/libnetwork/drivers/bridge/internal/iptabler/iptabler_test.go @@ -1,4 +1,6 @@ -package bridge +//go:build linux + +package iptabler import ( "context" @@ -22,6 +24,64 @@ const ( defaultBridgeName = "testbridge" ) +func TestCleanupIptableRules(t *testing.T) { + // Check for existence of a dummy rule to make sure iptables is initialised - then can + // check whether firewalld is running. + _ = iptables.GetIptable(iptables.IPv4).Exists(iptables.Filter, "FORWARD", "-j", "DROP") + if fw, _ := iptables.UsingFirewalld(); fw { + t.Skip("firewalld is running in the host netns, it can't modify rules in the test's netns") + } + + defer netnsutils.SetupTestOSContext(t)() + bridgeChains := []struct { + name string + table iptables.Table + expRemoved bool + }{ + {name: dockerChain, table: iptables.Nat, expRemoved: true}, + // The filter-FORWARD chain has references to dockerChain and isolationChain1, + // so the chains won't be removed - but they should be flushed. (This has + // long/always been the case for the daemon, its filter-FORWARD rules aren't + // removed.) + {name: dockerChain, table: iptables.Filter}, + {name: isolationChain1, table: iptables.Filter}, + } + + ipVersions := []iptables.IPVersion{iptables.IPv4, iptables.IPv6} + + for _, version := range ipVersions { + err := setupIPChains(version, true) + assert.NilError(t, err, "version:%s", version) + + iptable := iptables.GetIptable(version) + for _, chainInfo := range bridgeChains { + exists := iptable.ExistChain(chainInfo.name, chainInfo.table) + assert.Check(t, exists, "version:%s chain:%s table:%v", + version, chainInfo.name, chainInfo.table) + } + + // Insert RETURN rules so that there's something to flush. + for _, chainInfo := range bridgeChains { + out, err := iptable.Raw("-t", string(chainInfo.table), "-A", chainInfo.name, "-j", "RETURN") + assert.NilError(t, err, "version:%s chain:%s table:%v out:%s", + version, chainInfo.name, chainInfo.table, out) + } + + removeIPChains(version) + + for _, chainInfo := range bridgeChains { + exists := iptable.Exists(chainInfo.table, chainInfo.name, "-A", chainInfo.name, "-j", "RETURN") + assert.Check(t, !exists, "version:%s chain:%s table:%v", + version, chainInfo.name, chainInfo.table) + if chainInfo.expRemoved { + exists := iptable.ExistChain(chainInfo.name, chainInfo.table) + assert.Check(t, !exists, "version:%s chain:%s table:%v", + version, chainInfo.name, chainInfo.table) + } + } + } +} + func TestProgramIPTable(t *testing.T) { // Create a test bridge with a basic bridge configuration (name + IPv4). defer netnsutils.SetupTestOSContext(t)() @@ -36,11 +96,11 @@ func TestProgramIPTable(t *testing.T) { descr string }{ {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-d", "127.1.2.3", "-i", "lo", "-o", "lo", "-j", "DROP"}}, "Test Loopback"}, - {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", iptablesTestBridgeIP, "!", "-o", DefaultBridgeName, "-j", "MASQUERADE"}}, "NAT Test"}, - {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-o", DefaultBridgeName, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}}, "Test ACCEPT INCOMING"}, - {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-i", DefaultBridgeName, "!", "-o", DefaultBridgeName, "-j", "ACCEPT"}}, "Test ACCEPT NON_ICC OUTGOING"}, - {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-i", DefaultBridgeName, "-o", DefaultBridgeName, "-j", "ACCEPT"}}, "Test enable ICC"}, - {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-i", DefaultBridgeName, "-o", DefaultBridgeName, "-j", "DROP"}}, "Test disable ICC"}, + {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", iptablesTestBridgeIP, "!", "-o", defaultBridgeName, "-j", "MASQUERADE"}}, "NAT Test"}, + {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-o", defaultBridgeName, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}}, "Test ACCEPT INCOMING"}, + {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-i", defaultBridgeName, "!", "-o", defaultBridgeName, "-j", "ACCEPT"}}, "Test ACCEPT NON_ICC OUTGOING"}, + {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-i", defaultBridgeName, "-o", defaultBridgeName, "-j", "ACCEPT"}}, "Test enable ICC"}, + {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-i", defaultBridgeName, "-o", defaultBridgeName, "-j", "DROP"}}, "Test disable ICC"}, } // Assert the chain rules' insertion and removal. @@ -68,13 +128,12 @@ func TestSetupIPChains(t *testing.T) { // Create a test bridge with a basic bridge configuration (name + IPv4). defer netnsutils.SetupTestOSContext(t)() - ipt := firewaller{IPv4: true} - err := ipt.init() + ipt, err := NewIptabler(FirewallConfig{IPv4: true}) assert.NilError(t, err) - nc := networkConfig{ + nc := NetworkConfig{ IfName: defaultBridgeName, - Config4: networkConfigFam{ + Config4: NetworkConfigFam{ Prefix: netip.MustParsePrefix("192.168.42.0/24"), }, } @@ -94,21 +153,21 @@ func TestSetupIPChains(t *testing.T) { // Regression test for https://github.com/moby/moby/issues/46445 func TestSetupIP6TablesWithHostIPv4(t *testing.T) { defer netnsutils.SetupTestOSContext(t)() - ipt := firewaller{ + + ipt, err := NewIptabler(FirewallConfig{ IPv4: true, IPv6: true, - } - err := ipt.init() + }) assert.NilError(t, err) - nc := networkConfig{ + nc := NetworkConfig{ IfName: defaultBridgeName, Masquerade: true, - Config4: networkConfigFam{ + Config4: NetworkConfigFam{ HostIP: netip.MustParseAddr("192.0.2.2"), Prefix: netip.MustParsePrefix("192.168.42.0/24"), }, - Config6: networkConfigFam{ + Config6: NetworkConfigFam{ Prefix: netip.MustParsePrefix("2001:db8::/64"), }, } @@ -116,16 +175,16 @@ func TestSetupIP6TablesWithHostIPv4(t *testing.T) { } // Assert function which pushes chains based on bridge config parameters. -func assertBridgeConfig(t *testing.T, ipt firewaller, nc networkConfig) { +func assertBridgeConfig(t *testing.T, ipt *Iptabler, nc NetworkConfig) { t.Helper() n, err := ipt.NewNetwork(nc) assert.NilError(t, err) - err = n.delNetworkLevelRules() + err = n.DelNetworkLevelRules() assert.NilError(t, err) } func TestOutgoingNATRules(t *testing.T) { - br := "br-nattest" + const br = "br-nattest" maskedBrIPv4 := netip.MustParsePrefix("192.168.42.1/16").Masked() maskedBrIPv6 := netip.MustParsePrefix("2001:db8::1/64").Masked() hostIPv4 := netip.MustParseAddr("192.0.2.2") @@ -158,13 +217,13 @@ func TestOutgoingNATRules(t *testing.T) { enableIPv6: true, }, { - desc: "iptables/ip6tables disabled", + desc: "iptables and ip6tables disabled", enableIPv4: true, enableIPv6: true, enableIPMasquerade: true, }, { - desc: "host IP with iptables/ip6tables disabled", + desc: "host IP with iptables and ip6tables disabled", enableIPv4: true, enableIPv6: true, enableIPMasquerade: true, @@ -264,22 +323,20 @@ func TestOutgoingNATRules(t *testing.T) { } { t.Run(tc.desc, func(t *testing.T) { defer netnsutils.SetupTestOSContext(t)() - - ipt := firewaller{ + ipt, err := NewIptabler(FirewallConfig{ IPv4: tc.enableIPTables, IPv6: tc.enableIP6Tables, - } - err := ipt.init() + }) assert.NilError(t, err) - nc := networkConfig{ + nc := NetworkConfig{ IfName: br, Masquerade: tc.enableIPMasquerade, - Config4: networkConfigFam{ + Config4: NetworkConfigFam{ HostIP: tc.hostIPv4, Prefix: maskedBrIPv4, }, - Config6: networkConfigFam{ + Config6: NetworkConfigFam{ HostIP: tc.hostIPv6, Prefix: maskedBrIPv6, }, @@ -288,7 +345,7 @@ func TestOutgoingNATRules(t *testing.T) { assert.NilError(t, err) defer func() { - err = n.delNetworkLevelRules() + err = n.DelNetworkLevelRules() assert.NilError(t, err) }() @@ -303,7 +360,7 @@ func TestOutgoingNATRules(t *testing.T) { t.Logf("%s: %s %s table rules:\n%s", tc.desc, ipv, table, string(out)) } } - for _, rc := range []struct { + for i, rc := range []struct { want bool rule iptables.Rule }{ @@ -314,7 +371,7 @@ func TestOutgoingNATRules(t *testing.T) { {tc.wantIPv6Masq, iptables.Rule{IPVer: iptables.IPv6, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", maskedBrIPv6.String(), "!", "-o", br, "-j", "MASQUERADE"}}}, {tc.wantIPv6Snat, iptables.Rule{IPVer: iptables.IPv6, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", maskedBrIPv6.String(), "!", "-o", br, "-j", "SNAT", "--to-source", hostIPv6.String()}}}, } { - assert.Equal(t, rc.rule.Exists(), rc.want) + assert.Check(t, is.Equal(rc.rule.Exists(), rc.want), "rule:%d", i) } }) } @@ -360,12 +417,10 @@ func TestMirroredWSL2Workaround(t *testing.T) { restoreWslinfoPath := simulateWSL2MirroredMode(t, tc.loopback0, tc.wslinfoPerm) defer restoreWslinfoPath() - config := configuration{EnableIPTables: true} - if tc.userlandProxy { - config.UserlandProxyPath = "some-proxy" - config.EnableUserlandProxy = true - } - err := setupIPChains(iptables.IPv4, !tc.userlandProxy) + _, err := NewIptabler(FirewallConfig{ + IPv4: true, + Hairpin: !tc.userlandProxy, + }) assert.NilError(t, err) assert.Check(t, is.Equal(mirroredWSL2Rule().Exists(), tc.expLoopback0Rule)) }) @@ -435,21 +490,21 @@ func TestMirroredWSL2LoopbackFiltering(t *testing.T) { defer restoreWslinfoPath() hostIP := net.ParseIP("127.0.0.1") - err := filterPortMappedOnLoopback(context.TODO(), types.PortBinding{ + err := filterPortMappedOnLoopback(context.Background(), types.PortBinding{ Proto: types.TCP, IP: hostIP, HostPort: 8000, }, hostIP, true) assert.NilError(t, err) + out, err := exec.Command("iptables-save", "-t", "raw").CombinedOutput() + assert.NilError(t, err) + // Checking this after trying to create rules, to make sure the init code in iptables/firewalld.go has run. if fw, _ := iptables.UsingFirewalld(); fw { t.Skip("firewalld is running in the host netns, it can't modify rules in the test's netns") } - out, err := exec.Command("iptables-save", "-t", "raw").CombinedOutput() - assert.NilError(t, err) - if tc.expLoopback0Rule { assert.Check(t, is.Equal(strings.Count(string(out), "-A PREROUTING"), 2)) assert.Check(t, is.Contains(string(out), "-A PREROUTING -d 127.0.0.1/32 -i loopback0 -p tcp -m tcp --dport 8000 -j ACCEPT")) diff --git a/libnetwork/drivers/bridge/link.go b/libnetwork/drivers/bridge/internal/iptabler/link.go similarity index 74% rename from libnetwork/drivers/bridge/link.go rename to libnetwork/drivers/bridge/internal/iptabler/link.go index 63be801fb1..90e9d5bb08 100644 --- a/libnetwork/drivers/bridge/link.go +++ b/libnetwork/drivers/bridge/internal/iptabler/link.go @@ -1,6 +1,6 @@ //go:build linux -package bridge +package iptabler import ( "context" @@ -12,7 +12,7 @@ import ( "github.com/docker/docker/libnetwork/types" ) -func (n *iptablesNetwork) AddLink(ctx context.Context, parentIP, childIP netip.Addr, ports []types.TransportPort) error { +func (n *Network) AddLink(ctx context.Context, parentIP, childIP netip.Addr, ports []types.TransportPort) error { if !parentIP.IsValid() || parentIP.IsUnspecified() { return fmt.Errorf("cannot link to a container with an empty parent IP address") } @@ -20,7 +20,7 @@ func (n *iptablesNetwork) AddLink(ctx context.Context, parentIP, childIP netip.A return fmt.Errorf("cannot link to a container with an empty child IP address") } - chain := iptables.ChainInfo{Name: DockerChain} + chain := iptables.ChainInfo{Name: dockerChain} for _, port := range ports { if err := chain.Link(iptables.Append, parentIP, childIP, int(port.Port), port.Proto.String(), n.IfName); err != nil { return err @@ -29,8 +29,8 @@ func (n *iptablesNetwork) AddLink(ctx context.Context, parentIP, childIP netip.A return nil } -func (n *iptablesNetwork) DelLink(ctx context.Context, parentIP, childIP netip.Addr, ports []types.TransportPort) { - chain := iptables.ChainInfo{Name: DockerChain} +func (n *Network) DelLink(ctx context.Context, parentIP, childIP netip.Addr, ports []types.TransportPort) { + chain := iptables.ChainInfo{Name: dockerChain} for _, port := range ports { if err := chain.Link(iptables.Delete, parentIP, childIP, int(port.Port), port.Proto.String(), n.IfName); err != nil { log.G(ctx).WithFields(log.Fields{ diff --git a/libnetwork/drivers/bridge/internal/iptabler/port.go b/libnetwork/drivers/bridge/internal/iptabler/port.go new file mode 100644 index 0000000000..a109787cb1 --- /dev/null +++ b/libnetwork/drivers/bridge/internal/iptabler/port.go @@ -0,0 +1,240 @@ +//go:build linux + +package iptabler + +import ( + "context" + "net" + "os" + "strconv" + + "github.com/containerd/log" + "github.com/docker/docker/libnetwork/iptables" + "github.com/docker/docker/libnetwork/types" +) + +func (n *Network) AddPorts(ctx context.Context, pbs []types.PortBinding) error { + return n.modPorts(ctx, pbs, true) +} + +func (n *Network) DelPorts(ctx context.Context, pbs []types.PortBinding) error { + return n.modPorts(ctx, pbs, false) +} + +func (n *Network) modPorts(ctx context.Context, pbs []types.PortBinding, enable bool) error { + for _, pb := range pbs { + if err := n.setPerPortIptables(ctx, pb, enable); err != nil { + return err + } + } + return nil +} + +func (n *Network) setPerPortIptables(ctx context.Context, b types.PortBinding, enable bool) error { + v := iptables.IPv4 + enabled := n.ipt.IPv4 + config := n.Config4 + if b.IP.To4() == nil { + v = iptables.IPv6 + enabled = n.ipt.IPv6 + config = n.Config6 + } + + if !enabled || n.Internal { + // Nothing to do. + return nil + } + + if err := filterPortMappedOnLoopback(ctx, b, b.HostIP, enable); err != nil { + return err + } + + if err := n.filterDirectAccess(ctx, b, enable); err != nil { + return err + } + + if (b.IP.To4() != nil) != (b.HostIP.To4() != nil) { + // The binding is between containerV4 and hostV6 (not vice versa as that + // will have been rejected earlier). It's handled by docker-proxy. So, no + // further iptables rules are required. + return nil + } + + if err := n.setPerPortNAT(v, b, enable); err != nil { + return err + } + + if !config.Unprotected { + if err := setPerPortForwarding(b, v, n.IfName, enable); err != nil { + return err + } + } + return nil +} + +func (n *Network) setPerPortNAT(ipv iptables.IPVersion, b types.PortBinding, enable bool) error { + if b.HostPort == 0 { + // NAT is disabled. + return nil + } + // iptables interprets "0.0.0.0" as "0.0.0.0/32", whereas we + // want "0.0.0.0/0". "0/0" is correctly interpreted as "any + // value" by both iptables and ip6tables. + hostIP := "0/0" + if !b.HostIP.IsUnspecified() { + hostIP = b.HostIP.String() + } + args := []string{ + "-p", b.Proto.String(), + "-d", hostIP, + "--dport", strconv.Itoa(int(b.HostPort)), + "-j", "DNAT", + "--to-destination", net.JoinHostPort(b.IP.String(), strconv.Itoa(int(b.Port))), + } + if !n.ipt.Hairpin { + args = append(args, "!", "-i", n.IfName) + } + if ipv == iptables.IPv6 { + args = append(args, "!", "-s", "fe80::/10") + } + rule := iptables.Rule{IPVer: ipv, Table: iptables.Nat, Chain: dockerChain, Args: args} + if err := appendOrDelChainRule(rule, "DNAT", enable); err != nil { + return err + } + + rule = iptables.Rule{IPVer: ipv, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{ + "-p", b.Proto.String(), + "-s", b.IP.String(), + "-d", b.IP.String(), + "--dport", strconv.Itoa(int(b.Port)), + "-j", "MASQUERADE", + }} + if err := appendOrDelChainRule(rule, "MASQUERADE", n.ipt.Hairpin && enable); err != nil { + return err + } + + return nil +} + +func setPerPortForwarding(b types.PortBinding, ipv iptables.IPVersion, bridgeName string, enable bool) error { + // Insert rules for open ports at the top of the filter table's DOCKER + // chain (a per-network DROP rule, which must come after these per-port + // per-container ACCEPT rules, is appended to the chain when the network + // is created). + rule := iptables.Rule{IPVer: ipv, Table: iptables.Filter, Chain: dockerChain, Args: []string{ + "!", "-i", bridgeName, + "-o", bridgeName, + "-p", b.Proto.String(), + "-d", b.IP.String(), + "--dport", strconv.Itoa(int(b.Port)), + "-j", "ACCEPT", + }} + if err := programChainRule(rule, "OPEN PORT", enable); err != nil { + return err + } + + if b.Proto == types.SCTP && os.Getenv("DOCKER_IPTABLES_SCTP_CHECKSUM") == "1" { + // Linux kernel v4.9 and below enables NETIF_F_SCTP_CRC for veth by + // the following commit. + // This introduces a problem when combined with a physical NIC without + // NETIF_F_SCTP_CRC. As for a workaround, here we add an iptables entry + // to fill the checksum. + // + // https://github.com/torvalds/linux/commit/c80fafbbb59ef9924962f83aac85531039395b18 + rule := iptables.Rule{IPVer: ipv, Table: iptables.Mangle, Chain: "POSTROUTING", Args: []string{ + "-p", b.Proto.String(), + "--sport", strconv.Itoa(int(b.Port)), + "-j", "CHECKSUM", + "--checksum-fill", + }} + if err := appendOrDelChainRule(rule, "SCTP CHECKSUM", enable); err != nil { + return err + } + } + + return nil +} + +// filterPortMappedOnLoopback adds an iptables rule that drops remote +// connections to ports mapped on loopback addresses. +// +// This is a no-op if the portBinding is for IPv6 (IPv6 loopback address is +// non-routable), or over a network with gw_mode=routed (PBs in routed mode +// don't map ports on the host). +func filterPortMappedOnLoopback(ctx context.Context, b types.PortBinding, hostIP net.IP, enable bool) error { + if rawRulesDisabled(ctx) { + return nil + } + if b.HostPort == 0 || !hostIP.IsLoopback() || hostIP.To4() == nil { + return nil + } + + acceptMirrored := iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Raw, Chain: "PREROUTING", Args: []string{ + "-p", b.Proto.String(), + "-d", hostIP.String(), + "--dport", strconv.Itoa(int(b.HostPort)), + "-i", "loopback0", + "-j", "ACCEPT", + }} + enableMirrored := enable && isRunningUnderWSL2MirroredMode() + if err := appendOrDelChainRule(acceptMirrored, "LOOPBACK FILTERING - ACCEPT MIRRORED", enableMirrored); err != nil { + return err + } + + drop := iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Raw, Chain: "PREROUTING", Args: []string{ + "-p", b.Proto.String(), + "-d", hostIP.String(), + "--dport", strconv.Itoa(int(b.HostPort)), + "!", "-i", "lo", + "-j", "DROP", + }} + if err := appendOrDelChainRule(drop, "LOOPBACK FILTERING - DROP", enable); err != nil { + return err + } + + return nil +} + +// filterDirectAccess adds an iptables rule that drops 'direct' remote +// connections made to the container's IP address, when the network gateway +// mode is "nat". +// +// This is a no-op if the gw_mode is "nat-unprotected" or "routed". +func (n *Network) filterDirectAccess(ctx context.Context, b types.PortBinding, enable bool) error { + if rawRulesDisabled(ctx) { + return nil + } + ipv := iptables.IPv4 + config := n.Config4 + if b.IP.To4() == nil { + ipv = iptables.IPv6 + config = n.Config6 + } + + // gw_mode=nat-unprotected means there's minimal security for NATed ports, + // so don't filter direct access. + if config.Unprotected || config.Routed { + return nil + } + + drop := iptables.Rule{IPVer: ipv, Table: iptables.Raw, Chain: "PREROUTING", Args: []string{ + "-p", b.Proto.String(), + "-d", b.IP.String(), // Container IP address + "--dport", strconv.Itoa(int(b.Port)), // Container port + "!", "-i", n.IfName, + "-j", "DROP", + }} + if err := appendOrDelChainRule(drop, "DIRECT ACCESS FILTERING - DROP", enable); err != nil { + return err + } + + return nil +} + +func rawRulesDisabled(ctx context.Context) bool { + if os.Getenv("DOCKER_INSECURE_NO_IPTABLES_RAW") == "1" { + log.G(ctx).Debug("DOCKER_INSECURE_NO_IPTABLES_RAW=1 - skipping raw rules") + return true + } + return false +} diff --git a/libnetwork/drivers/bridge/port_mapping_linux.go b/libnetwork/drivers/bridge/port_mapping_linux.go index 667353466d..386aa5c6d9 100644 --- a/libnetwork/drivers/bridge/port_mapping_linux.go +++ b/libnetwork/drivers/bridge/port_mapping_linux.go @@ -16,7 +16,6 @@ import ( "github.com/containerd/log" "github.com/docker/docker/libnetwork/drivers/bridge/internal/rlkclient" - "github.com/docker/docker/libnetwork/iptables" "github.com/docker/docker/libnetwork/netutils" "github.com/docker/docker/libnetwork/portallocator" "github.com/docker/docker/libnetwork/portmapper" @@ -773,232 +772,6 @@ func (n *bridgeNetwork) releasePortBindings(pbs []portBinding) error { return errors.Join(errs...) } -func (n *iptablesNetwork) AddPorts(ctx context.Context, pbs []types.PortBinding) error { - return n.modPorts(ctx, pbs, true) -} - -func (n *iptablesNetwork) DelPorts(ctx context.Context, pbs []types.PortBinding) error { - return n.modPorts(ctx, pbs, false) -} - -func (n *iptablesNetwork) modPorts(ctx context.Context, pbs []types.PortBinding, enable bool) error { - for _, pb := range pbs { - if err := n.setPerPortIptables(ctx, pb, enable); err != nil { - return err - } - } - return nil -} - -func (n *iptablesNetwork) setPerPortIptables(ctx context.Context, b types.PortBinding, enable bool) error { - v := iptables.IPv4 - enabled := n.fw.IPv4 - config := n.Config4 - if b.IP.To4() == nil { - v = iptables.IPv6 - enabled = n.fw.IPv6 - config = n.Config6 - } - - if !enabled { - // Nothing to do, iptables/ip6tables is not enabled. - return nil - } - - if err := filterPortMappedOnLoopback(ctx, b, b.HostIP, enable); err != nil { - return err - } - - if err := n.filterDirectAccess(ctx, b, enable); err != nil { - return err - } - - if (b.IP.To4() != nil) != (b.HostIP.To4() != nil) { - // The binding is between containerV4 and hostV6 (not vice versa as that - // will have been rejected earlier). It's handled by docker-proxy. So, no - // further iptables rules are required. - return nil - } - - if err := n.setPerPortNAT(v, b, enable); err != nil { - return err - } - - if !config.Unprotected { - if err := setPerPortForwarding(b, v, n.IfName, enable); err != nil { - return err - } - } - return nil -} - -func (n *iptablesNetwork) setPerPortNAT(ipv iptables.IPVersion, b types.PortBinding, enable bool) error { - if b.HostPort == 0 { - // NAT is disabled. - return nil - } - // iptables interprets "0.0.0.0" as "0.0.0.0/32", whereas we - // want "0.0.0.0/0". "0/0" is correctly interpreted as "any - // value" by both iptables and ip6tables. - hostIP := "0/0" - if !b.HostIP.IsUnspecified() { - hostIP = b.HostIP.String() - } - args := []string{ - "-p", b.Proto.String(), - "-d", hostIP, - "--dport", strconv.Itoa(int(b.HostPort)), - "-j", "DNAT", - "--to-destination", net.JoinHostPort(b.IP.String(), strconv.Itoa(int(b.Port))), - } - if !n.fw.Hairpin { - args = append(args, "!", "-i", n.IfName) - } - if ipv == iptables.IPv6 { - args = append(args, "!", "-s", "fe80::/10") - } - rule := iptables.Rule{IPVer: ipv, Table: iptables.Nat, Chain: DockerChain, Args: args} - if err := appendOrDelChainRule(rule, "DNAT", enable); err != nil { - return err - } - - rule = iptables.Rule{IPVer: ipv, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{ - "-p", b.Proto.String(), - "-s", b.IP.String(), - "-d", b.IP.String(), - "--dport", strconv.Itoa(int(b.Port)), - "-j", "MASQUERADE", - }} - if err := appendOrDelChainRule(rule, "MASQUERADE", n.fw.Hairpin && enable); err != nil { - return err - } - - return nil -} - -func setPerPortForwarding(b types.PortBinding, ipv iptables.IPVersion, bridgeName string, enable bool) error { - // Insert rules for open ports at the top of the filter table's DOCKER - // chain (a per-network DROP rule, which must come after these per-port - // per-container ACCEPT rules, is appended to the chain when the network - // is created). - rule := iptables.Rule{IPVer: ipv, Table: iptables.Filter, Chain: DockerChain, Args: []string{ - "!", "-i", bridgeName, - "-o", bridgeName, - "-p", b.Proto.String(), - "-d", b.IP.String(), - "--dport", strconv.Itoa(int(b.Port)), - "-j", "ACCEPT", - }} - if err := programChainRule(rule, "OPEN PORT", enable); err != nil { - return err - } - - if b.Proto == types.SCTP && os.Getenv("DOCKER_IPTABLES_SCTP_CHECKSUM") == "1" { - // Linux kernel v4.9 and below enables NETIF_F_SCTP_CRC for veth by - // the following commit. - // This introduces a problem when combined with a physical NIC without - // NETIF_F_SCTP_CRC. As for a workaround, here we add an iptables entry - // to fill the checksum. - // - // https://github.com/torvalds/linux/commit/c80fafbbb59ef9924962f83aac85531039395b18 - rule := iptables.Rule{IPVer: ipv, Table: iptables.Mangle, Chain: "POSTROUTING", Args: []string{ - "-p", b.Proto.String(), - "--sport", strconv.Itoa(int(b.Port)), - "-j", "CHECKSUM", - "--checksum-fill", - }} - if err := appendOrDelChainRule(rule, "SCTP CHECKSUM", enable); err != nil { - return err - } - } - - return nil -} - -// filterPortMappedOnLoopback adds an iptables rule that drops remote -// connections to ports mapped on loopback addresses. -// -// This is a no-op if the portBinding is for IPv6 (IPv6 loopback address is -// non-routable), or over a network with gw_mode=routed (PBs in routed mode -// don't map ports on the host). -func filterPortMappedOnLoopback(ctx context.Context, b types.PortBinding, hostIP net.IP, enable bool) error { - if rawRulesDisabled(ctx) { - return nil - } - if b.HostPort == 0 || !hostIP.IsLoopback() || hostIP.To4() == nil { - return nil - } - - acceptMirrored := iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Raw, Chain: "PREROUTING", Args: []string{ - "-p", b.Proto.String(), - "-d", hostIP.String(), - "--dport", strconv.Itoa(int(b.HostPort)), - "-i", "loopback0", - "-j", "ACCEPT", - }} - enableMirrored := enable && isRunningUnderWSL2MirroredMode() - if err := appendOrDelChainRule(acceptMirrored, "LOOPBACK FILTERING - ACCEPT MIRRORED", enableMirrored); err != nil { - return err - } - - drop := iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Raw, Chain: "PREROUTING", Args: []string{ - "-p", b.Proto.String(), - "-d", hostIP.String(), - "--dport", strconv.Itoa(int(b.HostPort)), - "!", "-i", "lo", - "-j", "DROP", - }} - if err := appendOrDelChainRule(drop, "LOOPBACK FILTERING - DROP", enable); err != nil { - return err - } - - return nil -} - -// filterDirectAccess adds an iptables rule that drops 'direct' remote -// connections made to the container's IP address, when the network gateway -// mode is "nat". -// -// This is a no-op if the gw_mode is "nat-unprotected" or "routed". -func (n *iptablesNetwork) filterDirectAccess(ctx context.Context, b types.PortBinding, enable bool) error { - if rawRulesDisabled(ctx) { - return nil - } - ipv := iptables.IPv4 - config := n.Config4 - if b.IP.To4() == nil { - ipv = iptables.IPv6 - config = n.Config6 - } - - // gw_mode=nat-unprotected means there's minimal security for NATed ports, - // so don't filter direct access. - if config.Unprotected || config.Routed { - return nil - } - - drop := iptables.Rule{IPVer: ipv, Table: iptables.Raw, Chain: "PREROUTING", Args: []string{ - "-p", b.Proto.String(), - "-d", b.IP.String(), // Container IP address - "--dport", strconv.Itoa(int(b.Port)), // Container port - "!", "-i", n.IfName, - "-j", "DROP", - }} - if err := appendOrDelChainRule(drop, "DIRECT ACCESS FILTERING - DROP", enable); err != nil { - return err - } - - return nil -} - -func rawRulesDisabled(ctx context.Context) bool { - if os.Getenv("DOCKER_INSECURE_NO_IPTABLES_RAW") == "1" { - log.G(ctx).Debug("DOCKER_INSECURE_NO_IPTABLES_RAW=1 - skipping raw rules") - return true - } - return false -} - func (n *bridgeNetwork) reapplyPerPortIptables() { n.Lock() var allPBs []portBinding From dea236e0cee31407ca8328e94710f02fa93e3466 Mon Sep 17 00:00:00 2001 From: Rob Murray Date: Thu, 10 Apr 2025 19:20:12 +0100 Subject: [PATCH 5/5] Split iptabler into multiple files Signed-off-by: Rob Murray --- .../bridge/internal/iptabler/iptabler.go | 678 ------------------ .../bridge/internal/iptabler/iptabler_test.go | 448 ------------ .../bridge/internal/iptabler/network.go | 596 +++++++++++++++ .../bridge/internal/iptabler/network_test.go | 312 ++++++++ .../drivers/bridge/internal/iptabler/port.go | 8 + .../drivers/bridge/internal/iptabler/wsl2.go | 104 +++ .../bridge/internal/iptabler/wsl2_test.go | 160 +++++ 7 files changed, 1180 insertions(+), 1126 deletions(-) create mode 100644 libnetwork/drivers/bridge/internal/iptabler/network.go create mode 100644 libnetwork/drivers/bridge/internal/iptabler/network_test.go create mode 100644 libnetwork/drivers/bridge/internal/iptabler/wsl2.go create mode 100644 libnetwork/drivers/bridge/internal/iptabler/wsl2_test.go diff --git a/libnetwork/drivers/bridge/internal/iptabler/iptabler.go b/libnetwork/drivers/bridge/internal/iptabler/iptabler.go index bd424fd2ce..b88e2739a6 100644 --- a/libnetwork/drivers/bridge/internal/iptabler/iptabler.go +++ b/libnetwork/drivers/bridge/internal/iptabler/iptabler.go @@ -4,17 +4,11 @@ package iptabler import ( "context" - "errors" "fmt" - "net/netip" - "os" "github.com/containerd/log" - "github.com/docker/docker/errdefs" "github.com/docker/docker/internal/modprobe" - "github.com/docker/docker/internal/nlwrap" "github.com/docker/docker/libnetwork/iptables" - "github.com/vishvananda/netlink" ) const ( @@ -50,11 +44,6 @@ type Iptabler struct { FirewallConfig } -// Path to the executable installed in Linux under WSL2 that reports on -// WSL config. https://github.com/microsoft/WSL/releases/tag/2.0.4 -// Can be modified by tests. -var wslinfoPath = "/usr/bin/wslinfo" - func NewIptabler(config FirewallConfig) (*Iptabler, error) { ipt := &Iptabler{FirewallConfig: config} @@ -249,400 +238,6 @@ func setupIPChains(version iptables.IPVersion, hairpin bool) (retErr error) { return nil } -type ( - iptableCleanFunc func() error - iptablesCleanFuncs []iptableCleanFunc -) - -type NetworkConfigFam struct { - HostIP netip.Addr - Prefix netip.Prefix - Routed bool - Unprotected bool -} - -type NetworkConfig struct { - IfName string - Internal bool - ICC bool - Masquerade bool - Config4 NetworkConfigFam - Config6 NetworkConfigFam -} - -type Network struct { - NetworkConfig - ipt *Iptabler - cleanFuncs iptablesCleanFuncs -} - -func (ipt *Iptabler) NewNetwork(nc NetworkConfig) (_ *Network, retErr error) { - n := &Network{ - ipt: ipt, - NetworkConfig: nc, - } - defer func() { - if retErr != nil { - if err := n.DelNetworkLevelRules(); err != nil { - log.G(context.TODO()).WithError(err).Warnf("Failed to delete network level rules following earlier error") - } - } - }() - - if err := n.ReapplyNetworkLevelRules(); err != nil { - return nil, err - } - return n, nil -} - -func (n *Network) ReapplyNetworkLevelRules() error { - if n.ipt.IPv4 { - if err := n.configure(iptables.IPv4, n.Config4); err != nil { - return err - } - } - if n.ipt.IPv6 { - if err := n.configure(iptables.IPv6, n.Config6); err != nil { - return err - } - } - return nil -} - -func (n *Network) DelNetworkLevelRules() error { - var errs []error - for _, cleanFunc := range n.cleanFuncs { - if err := cleanFunc(); err != nil { - errs = append(errs, err) - } - } - n.cleanFuncs = nil - return errors.Join(errs...) -} - -func (n *Network) configure(ipv iptables.IPVersion, conf NetworkConfigFam) error { - if !conf.Prefix.IsValid() { - // Delete INC rules, in case they were created by a 28.0.0 daemon that didn't check - // whether the network had iptables/ip6tables enabled. - // This preserves https://github.com/moby/moby/commit/8cc4d1d4a2b6408232041f9ba4dff966eba80cc0 - return setINC(ipv, n.IfName, conf.Routed, false) - } - if err := n.setupIPTables(ipv, conf); err != nil { - return err - } - return nil -} - -func (n *Network) registerCleanFunc(clean iptableCleanFunc) { - n.cleanFuncs = append(n.cleanFuncs, clean) -} - -func (n *Network) setupIPTables(ipVersion iptables.IPVersion, config NetworkConfigFam) error { - if n.Internal { - if err := setupInternalNetworkRules(n.IfName, config.Prefix, n.ICC, true); err != nil { - return fmt.Errorf("Failed to Setup IP tables: %w", err) - } - n.registerCleanFunc(func() error { - return setupInternalNetworkRules(n.IfName, config.Prefix, n.ICC, false) - }) - } else { - if err := n.setupNonInternalNetworkRules(ipVersion, config, true); err != nil { - return fmt.Errorf("Failed to Setup IP tables: %w", err) - } - n.registerCleanFunc(func() error { - return n.setupNonInternalNetworkRules(ipVersion, config, false) - }) - - if err := iptables.AddInterfaceFirewalld(n.IfName); err != nil { - return err - } - n.registerCleanFunc(func() error { - if err := iptables.DelInterfaceFirewalld(n.IfName); err != nil && !errdefs.IsNotFound(err) { - return err - } - return nil - }) - - if err := deleteLegacyFilterRules(ipVersion, n.IfName); err != nil { - return fmt.Errorf("failed to delete legacy rules in filter-FORWARD: %w", err) - } - - err := setDefaultForwardRule(ipVersion, n.IfName, config.Unprotected, true) - if err != nil { - return err - } - n.registerCleanFunc(func() error { - return setDefaultForwardRule(ipVersion, n.IfName, config.Unprotected, false) - }) - - ctRule := iptables.Rule{IPVer: ipVersion, Table: iptables.Filter, Chain: dockerCTChain, Args: []string{ - "-o", n.IfName, - "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", - "-j", "ACCEPT", - }} - if err := appendOrDelChainRule(ctRule, "bridge ct related", true); err != nil { - return err - } - n.registerCleanFunc(func() error { - return appendOrDelChainRule(ctRule, "bridge ct related", false) - }) - jumpToDockerRule := iptables.Rule{IPVer: ipVersion, Table: iptables.Filter, Chain: dockerBridgeChain, Args: []string{ - "-o", n.IfName, - "-j", dockerChain, - }} - if err := appendOrDelChainRule(jumpToDockerRule, "jump to docker", true); err != nil { - return err - } - n.registerCleanFunc(func() error { - return appendOrDelChainRule(jumpToDockerRule, "jump to docker", false) - }) - - // Register the cleanup function first. Then, if setINC fails after creating - // some rules, they will be deleted. - n.registerCleanFunc(func() error { - return setINC(ipVersion, n.IfName, config.Routed, false) - }) - if err := setINC(ipVersion, n.IfName, config.Routed, true); err != nil { - return err - } - } - return nil -} - -func setICMP(ipv iptables.IPVersion, bridgeName string, enable bool) error { - icmpProto := "icmp" - if ipv == iptables.IPv6 { - icmpProto = "icmpv6" - } - icmpRule := iptables.Rule{IPVer: ipv, Table: iptables.Filter, Chain: dockerChain, Args: []string{ - "-o", bridgeName, - "-p", icmpProto, - "-j", "ACCEPT", - }} - return appendOrDelChainRule(icmpRule, "ICMP", enable) -} - -func addNATJumpRules(ipVer iptables.IPVersion, hairpinMode, enable bool) error { - preroute := iptables.Rule{IPVer: ipVer, Table: iptables.Nat, Chain: "PREROUTING", Args: []string{ - "-m", "addrtype", - "--dst-type", "LOCAL", - "-j", dockerChain, - }} - if enable { - if err := preroute.Append(); err != nil { - return fmt.Errorf("failed to append jump rules to nat-PREROUTING: %s", err) - } - } else { - if err := preroute.Delete(); err != nil { - return fmt.Errorf("failed to remove jump rules from nat-PREROUTING: %s", err) - } - } - - output := iptables.Rule{IPVer: ipVer, Table: iptables.Nat, Chain: "OUTPUT", Args: []string{ - "-m", "addrtype", - "--dst-type", "LOCAL", - "-j", dockerChain, - }} - if !hairpinMode { - output.Args = append(output.Args, "!", "--dst", loopbackAddress(ipVer)) - } - if enable { - if err := output.Append(); err != nil { - return fmt.Errorf("failed to append jump rules to nat-OUTPUT: %s", err) - } - } else { - if err := output.Delete(); err != nil { - return fmt.Errorf("failed to remove jump rules from nat-OUTPUT: %s", err) - } - } - - return nil -} - -// deleteLegacyFilterRules removes the legacy per-bridge rules from the filter-FORWARD -// chain. This is required for users upgrading the Engine to v28.0. -// TODO(aker): drop this function once Mirantis latest LTS is v28.0 (or higher). -func deleteLegacyFilterRules(ipVer iptables.IPVersion, bridgeName string) error { - iptable := iptables.GetIptable(ipVer) - // Delete legacy per-bridge jump to the DOCKER chain from the FORWARD chain, if it exists. - // These rules have been replaced by an ipset-matching rule. - link := []string{ - "-o", bridgeName, - "-j", dockerChain, - } - if iptable.Exists(iptables.Filter, "FORWARD", link...) { - del := append([]string{string(iptables.Delete), "FORWARD"}, link...) - if output, err := iptable.Raw(del...); err != nil { - return err - } else if len(output) != 0 { - return fmt.Errorf("could not delete linking rule from %s-%s: %s", iptables.Filter, dockerChain, output) - } - } - - // Delete legacy per-bridge related/established rule if it exists. These rules - // have been replaced by an ipset-matching rule. - establish := []string{ - "-o", bridgeName, - "-m", "conntrack", - "--ctstate", "RELATED,ESTABLISHED", - "-j", "ACCEPT", - } - if iptable.Exists(iptables.Filter, "FORWARD", establish...) { - del := append([]string{string(iptables.Delete), "FORWARD"}, establish...) - if output, err := iptable.Raw(del...); err != nil { - return err - } else if len(output) != 0 { - return fmt.Errorf("could not delete establish rule from %s-%s: %s", iptables.Filter, dockerChain, output) - } - } - - return nil -} - -// loopbackAddress returns the loopback address for the given IP version. -func loopbackAddress(version iptables.IPVersion) string { - switch version { - case iptables.IPv4, "": - // IPv4 (default for backward-compatibility) - return "127.0.0.0/8" - case iptables.IPv6: - return "::1/128" - default: - panic("unknown IP version: " + version) - } -} - -func setDefaultForwardRule(ipVersion iptables.IPVersion, ifName string, unprotected bool, enable bool) error { - // Normally, DROP anything that hasn't been ACCEPTed by a per-port/protocol - // rule. This prevents direct access to un-mapped ports from remote hosts - // that can route directly to the container's address (by setting up a - // route via the host's address). - action := "DROP" - if unprotected { - // If the user really wants to allow all access from the wider network, - // explicitly ACCEPT anything so that the filter-FORWARD chain's - // default policy can't interfere. - action = "ACCEPT" - } - - rule := iptables.Rule{IPVer: ipVersion, Table: iptables.Filter, Chain: dockerChain, Args: []string{ - "!", "-i", ifName, - "-o", ifName, - "-j", action, - }} - - // Append to the filter table's DOCKER chain (the default rule must follow - // per-port ACCEPT rules, which will be inserted at the top of the chain). - if err := appendOrDelChainRule(rule, "DEFAULT FWD", enable); err != nil { - return fmt.Errorf("failed to add default-drop rule: %w", err) - } - return nil -} - -func (n *Network) setupNonInternalNetworkRules(ipVer iptables.IPVersion, config NetworkConfigFam, enable bool) error { - var natArgs, hpNatArgs []string - if config.HostIP.IsValid() { - // The user wants IPv4/IPv6 SNAT with the given address. - hostAddr := config.HostIP.String() - natArgs = []string{"-s", config.Prefix.String(), "!", "-o", n.IfName, "-j", "SNAT", "--to-source", hostAddr} - hpNatArgs = []string{"-m", "addrtype", "--src-type", "LOCAL", "-o", n.IfName, "-j", "SNAT", "--to-source", hostAddr} - } else { - // Use MASQUERADE, which picks the src-ip based on next-hop from the route table - natArgs = []string{"-s", config.Prefix.String(), "!", "-o", n.IfName, "-j", "MASQUERADE"} - hpNatArgs = []string{"-m", "addrtype", "--src-type", "LOCAL", "-o", n.IfName, "-j", "MASQUERADE"} - } - natRule := iptables.Rule{IPVer: ipVer, Table: iptables.Nat, Chain: "POSTROUTING", Args: natArgs} - hpNatRule := iptables.Rule{IPVer: ipVer, Table: iptables.Nat, Chain: "POSTROUTING", Args: hpNatArgs} - - // Set NAT. - nat := !config.Routed - if n.Masquerade { - if nat { - if err := programChainRule(natRule, "NAT", enable); err != nil { - return err - } - } - // If the userland proxy is running (!hairpin), skip DNAT for packets originating from - // this new network. Then, the proxy can pick up the packet from the host address the dest - // port is published to. Otherwise, if the packet is DNAT'd, it's forwarded straight to the - // target network, and will be dropped by network isolation rules if it didn't originate in - // the same bridge network. (So, with the proxy enabled, this skip allows a container in one - // network to reach a port published by a container in another bridge network.) - // - // If the userland proxy is disabled, don't skip, so packets will be DNAT'd. That will - // enable access to ports published by containers in the same network. But, the INC rules - // will block access to that published port from containers in other networks. (However, - // users may add a rule to DOCKER-USER to work around the INC rules if needed.) - if !n.ipt.Hairpin { - skipDNAT := iptables.Rule{IPVer: ipVer, Table: iptables.Nat, Chain: dockerChain, Args: []string{ - "-i", n.IfName, - "-j", "RETURN", - }} - if err := programChainRule(skipDNAT, "SKIP DNAT", enable); err != nil { - return err - } - } - } - - // In hairpin mode, masquerade traffic from localhost. If hairpin is disabled or if we're tearing down - // that bridge, make sure the iptables rule isn't lying around. - if err := programChainRule(hpNatRule, "MASQ LOCAL HOST", enable && n.ipt.Hairpin); err != nil { - return err - } - - // Set Inter Container Communication. - if err := setIcc(ipVer, n.IfName, n.ICC, false, enable); err != nil { - return err - } - - // Allow ICMP in routed mode. - if !nat { - if err := setICMP(ipVer, n.IfName, enable); err != nil { - return err - } - } - - // Handle outgoing packets. This rule was previously added unconditionally - // to ACCEPT packets that weren't ICC - an extra rule was needed to enable - // ICC if needed. Those rules are now combined. So, outRuleNoICC is only - // needed for ICC=false, along with the DROP rule for ICC added by setIcc. - outRuleNoICC := iptables.Rule{IPVer: ipVer, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{ - "-i", n.IfName, - "!", "-o", n.IfName, - "-j", "ACCEPT", - }} - // If there's a version of outRuleNoICC in the FORWARD chain, created by moby 28.0.0 or older, delete it. - if enable { - if err := outRuleNoICC.WithChain("FORWARD").Delete(); err != nil { - return fmt.Errorf("deleting FORWARD chain outRuleNoICC: %w", err) - } - } - if n.ICC { - // Accept outgoing traffic to anywhere, including other containers on this bridge. - outRuleICC := iptables.Rule{IPVer: ipVer, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{ - "-i", n.IfName, - "-j", "ACCEPT", - }} - if err := appendOrDelChainRule(outRuleICC, "ACCEPT OUTGOING", enable); err != nil { - return err - } - // If there's a version of outRuleICC in the FORWARD chain, created by moby 28.0.0 or older, delete it. - if enable { - if err := outRuleICC.WithChain("FORWARD").Delete(); err != nil { - return fmt.Errorf("deleting FORWARD chain outRuleICC: %w", err) - } - } - } else { - // Accept outgoing traffic to anywhere, apart from other containers on this bridge. - // setIcc added a DROP rule for ICC traffic. - if err := appendOrDelChainRule(outRuleNoICC, "ACCEPT NON_ICC OUTGOING", enable); err != nil { - return err - } - } - - return nil -} - func programChainRule(rule iptables.Rule, ruleDescr string, insert bool) error { operation := "disable" fn := rule.Delete @@ -668,276 +263,3 @@ func appendOrDelChainRule(rule iptables.Rule, ruleDescr string, append bool) err } return nil } - -func setIcc(version iptables.IPVersion, bridgeIface string, iccEnable, internal, insert bool) error { - args := []string{"-i", bridgeIface, "-o", bridgeIface, "-j"} - acceptRule := iptables.Rule{IPVer: version, Table: iptables.Filter, Chain: DockerForwardChain, Args: append(args, "ACCEPT")} - dropRule := iptables.Rule{IPVer: version, Table: iptables.Filter, Chain: DockerForwardChain, Args: append(args, "DROP")} - - // The accept rule is no longer required for a bridge with external connectivity, because - // ICC traffic is allowed by the outgoing-packets rule created by setupIptablesInternal. - // The accept rule is still required for a --internal network because it has no outgoing - // rule. If insert and the rule is not required, an ACCEPT rule for an external network - // may have been left behind by an older version of the daemon so, delete it. - if insert && iccEnable && internal { - if err := acceptRule.Append(); err != nil { - return fmt.Errorf("Unable to allow intercontainer communication: %w", err) - } - } else { - if err := acceptRule.Delete(); err != nil { - log.G(context.TODO()).WithError(err).Warn("Failed to delete legacy ICC accept rule") - } - } - - if insert && !iccEnable { - if err := dropRule.Append(); err != nil { - return fmt.Errorf("Unable to prevent intercontainer communication: %w", err) - } - } else { - if err := dropRule.Delete(); err != nil { - log.G(context.TODO()).WithError(err).Warn("Failed to delete ICC drop rule") - } - } - - // Delete rules that may have been inserted into the FORWARD chain by moby 28.0.0 or older. - if insert { - if err := acceptRule.WithChain("FORWARD").Delete(); err != nil { - return fmt.Errorf("deleting FORWARD chain accept rule: %w", err) - } - if err := dropRule.WithChain("FORWARD").Delete(); err != nil { - return fmt.Errorf("deleting FORWARD chain drop rule: %w", err) - } - } - return nil -} - -// Control Inter-Network Communication. -// Install rules only if they aren't present, remove only if they are. -// If this method returns an error, it doesn't roll back any rules it has added. -// No error is returned if rules cannot be removed (errors are just logged). -func setINC(version iptables.IPVersion, iface string, routed, enable bool) (retErr error) { - iptable := iptables.GetIptable(version) - actionI, actionA := iptables.Insert, iptables.Append - actionMsg := "add" - if !enable { - actionI, actionA = iptables.Delete, iptables.Delete - actionMsg = "remove" - } - - if routed { - // Anything is allowed into a routed network at this stage, so RETURN. Port - // filtering rules in the DOCKER chain will drop anything that's not destined - // for an open port. - if err := iptable.ProgramRule(iptables.Filter, isolationChain1, actionI, []string{ - "-o", iface, - "-j", "RETURN", - }); err != nil { - log.G(context.TODO()).WithError(err).Warnf("Failed to %s inter-network communication rule", actionMsg) - if enable { - return fmt.Errorf("%s inter-network communication rule: %w", actionMsg, err) - } - } - - // Allow responses from the routed network into whichever network made the request. - if err := iptable.ProgramRule(iptables.Filter, isolationChain1, actionI, []string{ - "-i", iface, - "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", - "-j", "ACCEPT", - }); err != nil { - log.G(context.TODO()).WithError(err).Warnf("Failed to %s inter-network communication rule", actionMsg) - if enable { - return fmt.Errorf("%s inter-network communication rule: %w", actionMsg, err) - } - } - } - - if err := iptable.ProgramRule(iptables.Filter, isolationChain1, actionA, []string{ - "-i", iface, - "!", "-o", iface, - "-j", isolationChain2, - }); err != nil { - log.G(context.TODO()).WithError(err).Warnf("Failed to %s inter-network communication rule", actionMsg) - if enable { - return fmt.Errorf("%s inter-network communication rule: %w", actionMsg, err) - } - } - - if err := iptable.ProgramRule(iptables.Filter, isolationChain2, actionI, []string{ - "-o", iface, - "-j", "DROP", - }); err != nil { - log.G(context.TODO()).WithError(err).Warnf("Failed to %s inter-network communication rule", actionMsg) - if enable { - return fmt.Errorf("%s inter-network communication rule: %w", actionMsg, err) - } - } - - return nil -} - -// Obsolete chain from previous docker versions -const oldIsolationChain = "DOCKER-ISOLATION" - -func removeIPChains(version iptables.IPVersion) { - ipt := iptables.GetIptable(version) - - // Remove obsolete rules from default chains - ipt.ProgramRule(iptables.Filter, "FORWARD", iptables.Delete, []string{"-j", oldIsolationChain}) - - // Remove chains - for _, chainInfo := range []iptables.ChainInfo{ - {Name: dockerChain, Table: iptables.Nat, IPVersion: version}, - {Name: dockerChain, Table: iptables.Filter, IPVersion: version}, - {Name: DockerForwardChain, Table: iptables.Filter, IPVersion: version}, - {Name: dockerBridgeChain, Table: iptables.Filter, IPVersion: version}, - {Name: dockerCTChain, Table: iptables.Filter, IPVersion: version}, - {Name: isolationChain1, Table: iptables.Filter, IPVersion: version}, - {Name: isolationChain2, Table: iptables.Filter, IPVersion: version}, - {Name: oldIsolationChain, Table: iptables.Filter, IPVersion: version}, - } { - if err := chainInfo.Remove(); err != nil { - log.G(context.TODO()).Warnf("Failed to remove existing iptables entries in table %s chain %s : %v", chainInfo.Table, chainInfo.Name, err) - } - } -} - -func setupInternalNetworkRules(bridgeIface string, prefix netip.Prefix, icc, insert bool) error { - var version iptables.IPVersion - var inDropRule, outDropRule iptables.Rule - - // Either add or remove the interface from the firewalld zone, if firewalld is running. - if insert { - if err := iptables.AddInterfaceFirewalld(bridgeIface); err != nil { - return err - } - } else { - if err := iptables.DelInterfaceFirewalld(bridgeIface); err != nil && !errdefs.IsNotFound(err) { - return err - } - } - - if prefix.Addr().Is4() { - version = iptables.IPv4 - inDropRule = iptables.Rule{ - IPVer: version, - Table: iptables.Filter, - Chain: isolationChain1, - Args: []string{"-i", bridgeIface, "!", "-d", prefix.String(), "-j", "DROP"}, - } - outDropRule = iptables.Rule{ - IPVer: version, - Table: iptables.Filter, - Chain: isolationChain1, - Args: []string{"-o", bridgeIface, "!", "-s", prefix.String(), "-j", "DROP"}, - } - } else { - version = iptables.IPv6 - inDropRule = iptables.Rule{ - IPVer: version, - Table: iptables.Filter, - Chain: isolationChain1, - Args: []string{"-i", bridgeIface, "!", "-o", bridgeIface, "!", "-d", prefix.String(), "-j", "DROP"}, - } - outDropRule = iptables.Rule{ - IPVer: version, - Table: iptables.Filter, - Chain: isolationChain1, - Args: []string{"!", "-i", bridgeIface, "-o", bridgeIface, "!", "-s", prefix.String(), "-j", "DROP"}, - } - } - - if err := programChainRule(inDropRule, "DROP INCOMING", insert); err != nil { - return err - } - if err := programChainRule(outDropRule, "DROP OUTGOING", insert); err != nil { - return err - } - - // Set Inter Container Communication. - return setIcc(version, bridgeIface, icc, true, insert) -} - -// mirroredWSL2Workaround adds or removes an IPv4 NAT rule, depending on whether -// docker's host Linux appears to be a guest running under WSL2 in with mirrored -// mode networking. -// https://learn.microsoft.com/en-us/windows/wsl/networking#mirrored-mode-networking -// -// Without mirrored mode networking, or for a packet sent from Linux, packets -// sent to 127.0.0.1 are processed as outgoing - they hit the nat-OUTPUT chain, -// which does not jump to the nat-DOCKER chain because the rule has an exception -// for "-d 127.0.0.0/8". The default action on the nat-OUTPUT chain is ACCEPT (by -// default), so the packet is delivered to 127.0.0.1 on lo, where docker-proxy -// picks it up and acts as a man-in-the-middle; it receives the packet and -// re-sends it to the container (or acks a SYN and sets up a second TCP -// connection to the container). So, the container sees packets arrive with a -// source address belonging to the network's bridge, and it is able to reply to -// that address. -// -// In WSL2's mirrored networking mode, Linux has a loopback0 device as well as lo -// (which owns 127.0.0.1 as normal). Packets sent to 127.0.0.1 from Windows to a -// server listening on Linux's 127.0.0.1 are delivered via loopback0, and -// processed as packets arriving from outside the Linux host (which they are). -// -// So, these packets hit the nat-PREROUTING chain instead of nat-OUTPUT. It would -// normally be impossible for a packet ->127.0.0.1 to arrive from outside the -// host, so the nat-PREROUTING jump to nat-DOCKER has no exception for it. The -// packet is processed by a per-bridge DNAT rule in that chain, so it is -// delivered directly to the container (not via docker-proxy) with source address -// 127.0.0.1, so the container can't respond. -// -// DNAT is normally skipped by RETURN rules in the nat-DOCKER chain for packets -// arriving from any other bridge network. Similarly, this function adds (or -// removes) a rule to RETURN early for packets delivered via loopback0 with -// destination 127.0.0.0/8. -func mirroredWSL2Workaround(ipv iptables.IPVersion, hairpin bool) error { - // WSL2 does not (currently) support Windows<->Linux communication via ::1. - if ipv != iptables.IPv4 { - return nil - } - return programChainRule(mirroredWSL2Rule(), "WSL2 loopback", shouldInsertMirroredWSL2Rule(hairpin)) -} - -// shouldInsertMirroredWSL2Rule returns true if the NAT rule for mirrored WSL2 workaround -// is required. It is required if: -// - the userland proxy is running. If not, there's nothing on the host to catch -// the packet, so the loopback0 rule as wouldn't be useful. However, without -// the workaround, with improvements in WSL2 v2.3.11, and without userland proxy -// running - no workaround is needed, the normal DNAT/masquerading works. -// - and, the host Linux appears to be running under Windows WSL2 with mirrored -// mode networking. -func shouldInsertMirroredWSL2Rule(hairpin bool) bool { - if hairpin { - return false - } - return isRunningUnderWSL2MirroredMode() -} - -// isRunningUnderWSL2MirroredMode returns true if the host Linux appears to be -// running under Windows WSL2 with mirrored mode networking. If a loopback0 -// device exists, and there's an executable at /usr/bin/wslinfo, infer that -// this is WSL2 with mirrored networking. ("wslinfo --networking-mode" reports -// "mirrored", but applying the workaround for WSL2's loopback device when it's -// not needed is low risk, compared with executing wslinfo with dockerd's -// elevated permissions.) -func isRunningUnderWSL2MirroredMode() bool { - if _, err := nlwrap.LinkByName("loopback0"); err != nil { - if !errors.As(err, &netlink.LinkNotFoundError{}) { - log.G(context.TODO()).WithError(err).Warn("Failed to check for WSL interface") - } - return false - } - stat, err := os.Stat(wslinfoPath) - if err != nil { - return false - } - return stat.Mode().IsRegular() && (stat.Mode().Perm()&0o111) != 0 -} - -func mirroredWSL2Rule() iptables.Rule { - return iptables.Rule{ - IPVer: iptables.IPv4, - Table: iptables.Nat, - Chain: dockerChain, - Args: []string{"-i", "loopback0", "-d", "127.0.0.0/8", "-j", "RETURN"}, - } -} diff --git a/libnetwork/drivers/bridge/internal/iptabler/iptabler_test.go b/libnetwork/drivers/bridge/internal/iptabler/iptabler_test.go index 70e9cdd4da..06b567a438 100644 --- a/libnetwork/drivers/bridge/internal/iptabler/iptabler_test.go +++ b/libnetwork/drivers/bridge/internal/iptabler/iptabler_test.go @@ -3,25 +3,11 @@ package iptabler import ( - "context" - "net" - "net/netip" - "os" - "os/exec" - "path/filepath" - "strings" "testing" "github.com/docker/docker/internal/testutils/netnsutils" "github.com/docker/docker/libnetwork/iptables" - "github.com/docker/docker/libnetwork/types" - "github.com/vishvananda/netlink" "gotest.tools/v3/assert" - is "gotest.tools/v3/assert/cmp" -) - -const ( - defaultBridgeName = "testbridge" ) func TestCleanupIptableRules(t *testing.T) { @@ -81,437 +67,3 @@ func TestCleanupIptableRules(t *testing.T) { } } } - -func TestProgramIPTable(t *testing.T) { - // Create a test bridge with a basic bridge configuration (name + IPv4). - defer netnsutils.SetupTestOSContext(t)() - - _, err := iptables.GetIptable(iptables.IPv4).NewChain(DockerForwardChain, iptables.Filter) - assert.NilError(t, err) - - // Store various iptables chain rules we care for. - const iptablesTestBridgeIP = "192.168.42.1" - rules := []struct { - rule iptables.Rule - descr string - }{ - {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-d", "127.1.2.3", "-i", "lo", "-o", "lo", "-j", "DROP"}}, "Test Loopback"}, - {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", iptablesTestBridgeIP, "!", "-o", defaultBridgeName, "-j", "MASQUERADE"}}, "NAT Test"}, - {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-o", defaultBridgeName, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}}, "Test ACCEPT INCOMING"}, - {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-i", defaultBridgeName, "!", "-o", defaultBridgeName, "-j", "ACCEPT"}}, "Test ACCEPT NON_ICC OUTGOING"}, - {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-i", defaultBridgeName, "-o", defaultBridgeName, "-j", "ACCEPT"}}, "Test enable ICC"}, - {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-i", defaultBridgeName, "-o", defaultBridgeName, "-j", "DROP"}}, "Test disable ICC"}, - } - - // Assert the chain rules' insertion and removal. - for _, c := range rules { - // Add - if err := programChainRule(c.rule, c.descr, true); err != nil { - t.Fatalf("Failed to program iptable rule %s: %s", c.descr, err.Error()) - } - - if !c.rule.Exists() { - t.Fatalf("Failed to effectively program iptable rule: %s", c.descr) - } - - // Remove - if err := programChainRule(c.rule, c.descr, false); err != nil { - t.Fatalf("Failed to remove iptable rule %s: %s", c.descr, err.Error()) - } - if c.rule.Exists() { - t.Fatalf("Failed to effectively remove iptable rule: %s", c.descr) - } - } -} - -func TestSetupIPChains(t *testing.T) { - // Create a test bridge with a basic bridge configuration (name + IPv4). - defer netnsutils.SetupTestOSContext(t)() - - ipt, err := NewIptabler(FirewallConfig{IPv4: true}) - assert.NilError(t, err) - - nc := NetworkConfig{ - IfName: defaultBridgeName, - Config4: NetworkConfigFam{ - Prefix: netip.MustParsePrefix("192.168.42.0/24"), - }, - } - - assertBridgeConfig(t, ipt, nc) - - nc.Masquerade = true - assertBridgeConfig(t, ipt, nc) - - nc.ICC = true - assertBridgeConfig(t, ipt, nc) - - nc.Masquerade = false - assertBridgeConfig(t, ipt, nc) -} - -// Regression test for https://github.com/moby/moby/issues/46445 -func TestSetupIP6TablesWithHostIPv4(t *testing.T) { - defer netnsutils.SetupTestOSContext(t)() - - ipt, err := NewIptabler(FirewallConfig{ - IPv4: true, - IPv6: true, - }) - assert.NilError(t, err) - - nc := NetworkConfig{ - IfName: defaultBridgeName, - Masquerade: true, - Config4: NetworkConfigFam{ - HostIP: netip.MustParseAddr("192.0.2.2"), - Prefix: netip.MustParsePrefix("192.168.42.0/24"), - }, - Config6: NetworkConfigFam{ - Prefix: netip.MustParsePrefix("2001:db8::/64"), - }, - } - assertBridgeConfig(t, ipt, nc) -} - -// Assert function which pushes chains based on bridge config parameters. -func assertBridgeConfig(t *testing.T, ipt *Iptabler, nc NetworkConfig) { - t.Helper() - n, err := ipt.NewNetwork(nc) - assert.NilError(t, err) - err = n.DelNetworkLevelRules() - assert.NilError(t, err) -} - -func TestOutgoingNATRules(t *testing.T) { - const br = "br-nattest" - maskedBrIPv4 := netip.MustParsePrefix("192.168.42.1/16").Masked() - maskedBrIPv6 := netip.MustParsePrefix("2001:db8::1/64").Masked() - hostIPv4 := netip.MustParseAddr("192.0.2.2") - hostIPv6 := netip.MustParseAddr("2001:db8:1::1") - for _, tc := range []struct { - desc string - enableIPTables bool - enableIP6Tables bool - enableIPv4 bool - enableIPv6 bool - enableIPMasquerade bool - hostIPv4 netip.Addr - hostIPv6 netip.Addr - // Hairpin NAT rules are not tested here because they are orthogonal to outgoing NAT. They - // exist to support the port forwarding DNAT rules: without any port forwarding there would be - // no need for any hairpin NAT rules, and when there is port forwarding then hairpin NAT rules - // are needed even if outgoing NAT is disabled. Hairpin NAT tests belong with the port - // forwarding DNAT tests. - wantIPv4Masq bool - wantIPv4Snat bool - wantIPv6Masq bool - wantIPv6Snat bool - }{ - { - desc: "everything disabled except ipv4", - enableIPv4: true, // one of IPv4 or IPv6 must be enabled - }, - { - desc: "everything disabled except ipv6", - enableIPv6: true, - }, - { - desc: "iptables and ip6tables disabled", - enableIPv4: true, - enableIPv6: true, - enableIPMasquerade: true, - }, - { - desc: "host IP with iptables and ip6tables disabled", - enableIPv4: true, - enableIPv6: true, - enableIPMasquerade: true, - hostIPv4: hostIPv4, - hostIPv6: hostIPv6, - }, - { - desc: "masquerade disabled, no host IP", - enableIPTables: true, - enableIP6Tables: true, - enableIPv4: true, - enableIPv6: true, - }, - { - desc: "masquerade disabled, with host IP", - enableIPTables: true, - enableIP6Tables: true, - enableIPv4: true, - enableIPv6: true, - hostIPv4: hostIPv4, - hostIPv6: hostIPv6, - }, - { - desc: "IPv4 masquerade, IPv6 disabled", - enableIPv4: true, - enableIPTables: true, - enableIPMasquerade: true, - wantIPv4Masq: true, - }, - { - desc: "IPv6 masquerade, IPv4 disabled", - enableIPv6: true, - enableIP6Tables: true, - enableIPMasquerade: true, - wantIPv6Masq: true, - }, - { - desc: "IPv4 SNAT, IPv6 disabled", - enableIPv4: true, - enableIPTables: true, - enableIPMasquerade: true, - hostIPv4: hostIPv4, - wantIPv4Snat: true, - }, - { - desc: "IPv6 SNAT, IPv4 disabled", - enableIPv6: true, - enableIP6Tables: true, - enableIPMasquerade: true, - hostIPv6: hostIPv6, - wantIPv6Snat: true, - }, - { - desc: "IPv4 masquerade, IPv6 masquerade", - enableIPTables: true, - enableIP6Tables: true, - enableIPv4: true, - enableIPv6: true, - enableIPMasquerade: true, - wantIPv4Masq: true, - wantIPv6Masq: true, - }, - { - desc: "IPv4 masquerade, IPv6 SNAT", - enableIPTables: true, - enableIP6Tables: true, - enableIPv4: true, - enableIPv6: true, - enableIPMasquerade: true, - hostIPv6: hostIPv6, - wantIPv4Masq: true, - wantIPv6Snat: true, - }, - { - desc: "IPv4 SNAT, IPv6 masquerade", - enableIPTables: true, - enableIP6Tables: true, - enableIPv4: true, - enableIPv6: true, - enableIPMasquerade: true, - hostIPv4: hostIPv4, - wantIPv4Snat: true, - wantIPv6Masq: true, - }, - { - desc: "IPv4 SNAT, IPv6 SNAT", - enableIPTables: true, - enableIP6Tables: true, - enableIPv4: true, - enableIPv6: true, - enableIPMasquerade: true, - hostIPv4: hostIPv4, - hostIPv6: hostIPv6, - wantIPv4Snat: true, - wantIPv6Snat: true, - }, - } { - t.Run(tc.desc, func(t *testing.T) { - defer netnsutils.SetupTestOSContext(t)() - ipt, err := NewIptabler(FirewallConfig{ - IPv4: tc.enableIPTables, - IPv6: tc.enableIP6Tables, - }) - assert.NilError(t, err) - - nc := NetworkConfig{ - IfName: br, - Masquerade: tc.enableIPMasquerade, - Config4: NetworkConfigFam{ - HostIP: tc.hostIPv4, - Prefix: maskedBrIPv4, - }, - Config6: NetworkConfigFam{ - HostIP: tc.hostIPv6, - Prefix: maskedBrIPv6, - }, - } - n, err := ipt.NewNetwork(nc) - assert.NilError(t, err) - - defer func() { - err = n.DelNetworkLevelRules() - assert.NilError(t, err) - }() - - // Log the contents of all chains to aid troubleshooting. - for _, ipv := range []iptables.IPVersion{iptables.IPv4, iptables.IPv6} { - ipt := iptables.GetIptable(ipv) - for _, table := range []iptables.Table{iptables.Nat, iptables.Filter, iptables.Mangle} { - out, err := ipt.Raw("-t", string(table), "-S") - if err != nil { - t.Error(err) - } - t.Logf("%s: %s %s table rules:\n%s", tc.desc, ipv, table, string(out)) - } - } - for i, rc := range []struct { - want bool - rule iptables.Rule - }{ - // Rule order doesn't matter: At most one of the following IPv4 rules will exist, and the - // same goes for the IPv6 rules. - {tc.wantIPv4Masq, iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", maskedBrIPv4.String(), "!", "-o", br, "-j", "MASQUERADE"}}}, - {tc.wantIPv4Snat, iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", maskedBrIPv4.String(), "!", "-o", br, "-j", "SNAT", "--to-source", hostIPv4.String()}}}, - {tc.wantIPv6Masq, iptables.Rule{IPVer: iptables.IPv6, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", maskedBrIPv6.String(), "!", "-o", br, "-j", "MASQUERADE"}}}, - {tc.wantIPv6Snat, iptables.Rule{IPVer: iptables.IPv6, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", maskedBrIPv6.String(), "!", "-o", br, "-j", "SNAT", "--to-source", hostIPv6.String()}}}, - } { - assert.Check(t, is.Equal(rc.rule.Exists(), rc.want), "rule:%d", i) - } - }) - } -} - -func TestMirroredWSL2Workaround(t *testing.T) { - for _, tc := range []struct { - desc string - loopback0 bool - userlandProxy bool - wslinfoPerm os.FileMode // 0 for no-file - expLoopback0Rule bool - }{ - { - desc: "No loopback0", - }, - { - desc: "WSL2 mirrored", - loopback0: true, - userlandProxy: true, - wslinfoPerm: 0o777, - expLoopback0Rule: true, - }, - { - desc: "loopback0 but wslinfo not executable", - loopback0: true, - userlandProxy: true, - wslinfoPerm: 0o666, - }, - { - desc: "loopback0 but no wslinfo", - loopback0: true, - userlandProxy: true, - }, - { - desc: "loopback0 but no userland proxy", - loopback0: true, - wslinfoPerm: 0o777, - }, - } { - t.Run(tc.desc, func(t *testing.T) { - defer netnsutils.SetupTestOSContext(t)() - restoreWslinfoPath := simulateWSL2MirroredMode(t, tc.loopback0, tc.wslinfoPerm) - defer restoreWslinfoPath() - - _, err := NewIptabler(FirewallConfig{ - IPv4: true, - Hairpin: !tc.userlandProxy, - }) - assert.NilError(t, err) - assert.Check(t, is.Equal(mirroredWSL2Rule().Exists(), tc.expLoopback0Rule)) - }) - } -} - -// simulateWSL2MirroredMode simulates the WSL2 mirrored mode by creating a -// loopback0 interface and optionally creating a wslinfo file with the given -// permissions. -// A clean up function is returned and will restore the original wslinfoPath -// used within the 'bridge' package. The loopback0 interface isn't cleaned up. -// Instead this function should be called from a disposable network namespace. -func simulateWSL2MirroredMode(t *testing.T, loopback0 bool, wslinfoPerm os.FileMode) func() { - if loopback0 { - iface := &netlink.Dummy{ - LinkAttrs: netlink.LinkAttrs{ - Name: "loopback0", - }, - } - err := netlink.LinkAdd(iface) - assert.NilError(t, err) - } - - wslinfoPathOrig := wslinfoPath - if wslinfoPerm != 0 { - tmpdir := t.TempDir() - p := filepath.Join(tmpdir, "wslinfo") - err := os.WriteFile(p, []byte("#!/bin/sh\necho dummy file\n"), wslinfoPerm) - assert.NilError(t, err) - wslinfoPath = p - } - - return func() { - wslinfoPath = wslinfoPathOrig - } -} - -func TestMirroredWSL2LoopbackFiltering(t *testing.T) { - for _, tc := range []struct { - desc string - loopback0 bool - wslinfoPerm os.FileMode // 0 for no-file - expLoopback0Rule bool - }{ - { - desc: "No loopback0", - }, - { - desc: "WSL2 mirrored", - loopback0: true, - wslinfoPerm: 0o777, - expLoopback0Rule: true, - }, - { - desc: "loopback0 but wslinfo not executable", - loopback0: true, - wslinfoPerm: 0o666, - }, - { - desc: "loopback0 but no wslinfo", - loopback0: true, - }, - } { - t.Run(tc.desc, func(t *testing.T) { - defer netnsutils.SetupTestOSContext(t)() - restoreWslinfoPath := simulateWSL2MirroredMode(t, tc.loopback0, tc.wslinfoPerm) - defer restoreWslinfoPath() - - hostIP := net.ParseIP("127.0.0.1") - err := filterPortMappedOnLoopback(context.Background(), types.PortBinding{ - Proto: types.TCP, - IP: hostIP, - HostPort: 8000, - }, hostIP, true) - assert.NilError(t, err) - - out, err := exec.Command("iptables-save", "-t", "raw").CombinedOutput() - assert.NilError(t, err) - - // Checking this after trying to create rules, to make sure the init code in iptables/firewalld.go has run. - if fw, _ := iptables.UsingFirewalld(); fw { - t.Skip("firewalld is running in the host netns, it can't modify rules in the test's netns") - } - - if tc.expLoopback0Rule { - assert.Check(t, is.Equal(strings.Count(string(out), "-A PREROUTING"), 2)) - assert.Check(t, is.Contains(string(out), "-A PREROUTING -d 127.0.0.1/32 -i loopback0 -p tcp -m tcp --dport 8000 -j ACCEPT")) - } else { - assert.Check(t, is.Equal(strings.Count(string(out), "-A PREROUTING"), 1)) - assert.Check(t, !strings.Contains(string(out), "loopback0"), "There should be no rule in the raw-PREROUTING chain") - } - }) - } -} diff --git a/libnetwork/drivers/bridge/internal/iptabler/network.go b/libnetwork/drivers/bridge/internal/iptabler/network.go new file mode 100644 index 0000000000..4bc9c114e5 --- /dev/null +++ b/libnetwork/drivers/bridge/internal/iptabler/network.go @@ -0,0 +1,596 @@ +//go:build linux + +package iptabler + +import ( + "context" + "errors" + "fmt" + "net/netip" + + "github.com/containerd/log" + "github.com/docker/docker/errdefs" + "github.com/docker/docker/libnetwork/iptables" +) + +type ( + iptableCleanFunc func() error + iptablesCleanFuncs []iptableCleanFunc +) + +type NetworkConfigFam struct { + HostIP netip.Addr + Prefix netip.Prefix + Routed bool + Unprotected bool +} + +type NetworkConfig struct { + IfName string + Internal bool + ICC bool + Masquerade bool + Config4 NetworkConfigFam + Config6 NetworkConfigFam +} + +type Network struct { + NetworkConfig + ipt *Iptabler + cleanFuncs iptablesCleanFuncs +} + +func (ipt *Iptabler) NewNetwork(nc NetworkConfig) (_ *Network, retErr error) { + n := &Network{ + ipt: ipt, + NetworkConfig: nc, + } + defer func() { + if retErr != nil { + if err := n.DelNetworkLevelRules(); err != nil { + log.G(context.TODO()).WithError(err).Warnf("Failed to delete network level rules following earlier error") + } + } + }() + + if err := n.ReapplyNetworkLevelRules(); err != nil { + return nil, err + } + return n, nil +} + +func (n *Network) ReapplyNetworkLevelRules() error { + if n.ipt.IPv4 { + if err := n.configure(iptables.IPv4, n.Config4); err != nil { + return err + } + } + if n.ipt.IPv6 { + if err := n.configure(iptables.IPv6, n.Config6); err != nil { + return err + } + } + return nil +} + +func (n *Network) DelNetworkLevelRules() error { + var errs []error + for _, cleanFunc := range n.cleanFuncs { + if err := cleanFunc(); err != nil { + errs = append(errs, err) + } + } + n.cleanFuncs = nil + return errors.Join(errs...) +} + +func (n *Network) configure(ipv iptables.IPVersion, conf NetworkConfigFam) error { + if !conf.Prefix.IsValid() { + // Delete INC rules, in case they were created by a 28.0.0 daemon that didn't check + // whether the network had iptables/ip6tables enabled. + // This preserves https://github.com/moby/moby/commit/8cc4d1d4a2b6408232041f9ba4dff966eba80cc0 + return setINC(ipv, n.IfName, conf.Routed, false) + } + if err := n.setupIPTables(ipv, conf); err != nil { + return err + } + return nil +} + +func (n *Network) registerCleanFunc(clean iptableCleanFunc) { + n.cleanFuncs = append(n.cleanFuncs, clean) +} + +func (n *Network) setupIPTables(ipVersion iptables.IPVersion, config NetworkConfigFam) error { + if n.Internal { + if err := setupInternalNetworkRules(n.IfName, config.Prefix, n.ICC, true); err != nil { + return fmt.Errorf("Failed to Setup IP tables: %w", err) + } + n.registerCleanFunc(func() error { + return setupInternalNetworkRules(n.IfName, config.Prefix, n.ICC, false) + }) + } else { + if err := n.setupNonInternalNetworkRules(ipVersion, config, true); err != nil { + return fmt.Errorf("Failed to Setup IP tables: %w", err) + } + n.registerCleanFunc(func() error { + return n.setupNonInternalNetworkRules(ipVersion, config, false) + }) + + if err := iptables.AddInterfaceFirewalld(n.IfName); err != nil { + return err + } + n.registerCleanFunc(func() error { + if err := iptables.DelInterfaceFirewalld(n.IfName); err != nil && !errdefs.IsNotFound(err) { + return err + } + return nil + }) + + if err := deleteLegacyFilterRules(ipVersion, n.IfName); err != nil { + return fmt.Errorf("failed to delete legacy rules in filter-FORWARD: %w", err) + } + + err := setDefaultForwardRule(ipVersion, n.IfName, config.Unprotected, true) + if err != nil { + return err + } + n.registerCleanFunc(func() error { + return setDefaultForwardRule(ipVersion, n.IfName, config.Unprotected, false) + }) + + ctRule := iptables.Rule{IPVer: ipVersion, Table: iptables.Filter, Chain: dockerCTChain, Args: []string{ + "-o", n.IfName, + "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", + "-j", "ACCEPT", + }} + if err := appendOrDelChainRule(ctRule, "bridge ct related", true); err != nil { + return err + } + n.registerCleanFunc(func() error { + return appendOrDelChainRule(ctRule, "bridge ct related", false) + }) + jumpToDockerRule := iptables.Rule{IPVer: ipVersion, Table: iptables.Filter, Chain: dockerBridgeChain, Args: []string{ + "-o", n.IfName, + "-j", dockerChain, + }} + if err := appendOrDelChainRule(jumpToDockerRule, "jump to docker", true); err != nil { + return err + } + n.registerCleanFunc(func() error { + return appendOrDelChainRule(jumpToDockerRule, "jump to docker", false) + }) + + // Register the cleanup function first. Then, if setINC fails after creating + // some rules, they will be deleted. + n.registerCleanFunc(func() error { + return setINC(ipVersion, n.IfName, config.Routed, false) + }) + if err := setINC(ipVersion, n.IfName, config.Routed, true); err != nil { + return err + } + } + return nil +} + +func setICMP(ipv iptables.IPVersion, bridgeName string, enable bool) error { + icmpProto := "icmp" + if ipv == iptables.IPv6 { + icmpProto = "icmpv6" + } + icmpRule := iptables.Rule{IPVer: ipv, Table: iptables.Filter, Chain: dockerChain, Args: []string{ + "-o", bridgeName, + "-p", icmpProto, + "-j", "ACCEPT", + }} + return appendOrDelChainRule(icmpRule, "ICMP", enable) +} + +func addNATJumpRules(ipVer iptables.IPVersion, hairpinMode, enable bool) error { + preroute := iptables.Rule{IPVer: ipVer, Table: iptables.Nat, Chain: "PREROUTING", Args: []string{ + "-m", "addrtype", + "--dst-type", "LOCAL", + "-j", dockerChain, + }} + if enable { + if err := preroute.Append(); err != nil { + return fmt.Errorf("failed to append jump rules to nat-PREROUTING: %s", err) + } + } else { + if err := preroute.Delete(); err != nil { + return fmt.Errorf("failed to remove jump rules from nat-PREROUTING: %s", err) + } + } + + output := iptables.Rule{IPVer: ipVer, Table: iptables.Nat, Chain: "OUTPUT", Args: []string{ + "-m", "addrtype", + "--dst-type", "LOCAL", + "-j", dockerChain, + }} + if !hairpinMode { + output.Args = append(output.Args, "!", "--dst", loopbackAddress(ipVer)) + } + if enable { + if err := output.Append(); err != nil { + return fmt.Errorf("failed to append jump rules to nat-OUTPUT: %s", err) + } + } else { + if err := output.Delete(); err != nil { + return fmt.Errorf("failed to remove jump rules from nat-OUTPUT: %s", err) + } + } + + return nil +} + +// deleteLegacyFilterRules removes the legacy per-bridge rules from the filter-FORWARD +// chain. This is required for users upgrading the Engine to v28.0. +// TODO(aker): drop this function once Mirantis latest LTS is v28.0 (or higher). +func deleteLegacyFilterRules(ipVer iptables.IPVersion, bridgeName string) error { + iptable := iptables.GetIptable(ipVer) + // Delete legacy per-bridge jump to the DOCKER chain from the FORWARD chain, if it exists. + // These rules have been replaced by an ipset-matching rule. + link := []string{ + "-o", bridgeName, + "-j", dockerChain, + } + if iptable.Exists(iptables.Filter, "FORWARD", link...) { + del := append([]string{string(iptables.Delete), "FORWARD"}, link...) + if output, err := iptable.Raw(del...); err != nil { + return err + } else if len(output) != 0 { + return fmt.Errorf("could not delete linking rule from %s-%s: %s", iptables.Filter, dockerChain, output) + } + } + + // Delete legacy per-bridge related/established rule if it exists. These rules + // have been replaced by an ipset-matching rule. + establish := []string{ + "-o", bridgeName, + "-m", "conntrack", + "--ctstate", "RELATED,ESTABLISHED", + "-j", "ACCEPT", + } + if iptable.Exists(iptables.Filter, "FORWARD", establish...) { + del := append([]string{string(iptables.Delete), "FORWARD"}, establish...) + if output, err := iptable.Raw(del...); err != nil { + return err + } else if len(output) != 0 { + return fmt.Errorf("could not delete establish rule from %s-%s: %s", iptables.Filter, dockerChain, output) + } + } + + return nil +} + +// loopbackAddress returns the loopback address for the given IP version. +func loopbackAddress(version iptables.IPVersion) string { + switch version { + case iptables.IPv4, "": + // IPv4 (default for backward-compatibility) + return "127.0.0.0/8" + case iptables.IPv6: + return "::1/128" + default: + panic("unknown IP version: " + version) + } +} + +func setDefaultForwardRule(ipVersion iptables.IPVersion, ifName string, unprotected bool, enable bool) error { + // Normally, DROP anything that hasn't been ACCEPTed by a per-port/protocol + // rule. This prevents direct access to un-mapped ports from remote hosts + // that can route directly to the container's address (by setting up a + // route via the host's address). + action := "DROP" + if unprotected { + // If the user really wants to allow all access from the wider network, + // explicitly ACCEPT anything so that the filter-FORWARD chain's + // default policy can't interfere. + action = "ACCEPT" + } + + rule := iptables.Rule{IPVer: ipVersion, Table: iptables.Filter, Chain: dockerChain, Args: []string{ + "!", "-i", ifName, + "-o", ifName, + "-j", action, + }} + + // Append to the filter table's DOCKER chain (the default rule must follow + // per-port ACCEPT rules, which will be inserted at the top of the chain). + if err := appendOrDelChainRule(rule, "DEFAULT FWD", enable); err != nil { + return fmt.Errorf("failed to add default-drop rule: %w", err) + } + return nil +} + +func (n *Network) setupNonInternalNetworkRules(ipVer iptables.IPVersion, config NetworkConfigFam, enable bool) error { + var natArgs, hpNatArgs []string + if config.HostIP.IsValid() { + // The user wants IPv4/IPv6 SNAT with the given address. + hostAddr := config.HostIP.String() + natArgs = []string{"-s", config.Prefix.String(), "!", "-o", n.IfName, "-j", "SNAT", "--to-source", hostAddr} + hpNatArgs = []string{"-m", "addrtype", "--src-type", "LOCAL", "-o", n.IfName, "-j", "SNAT", "--to-source", hostAddr} + } else { + // Use MASQUERADE, which picks the src-ip based on next-hop from the route table + natArgs = []string{"-s", config.Prefix.String(), "!", "-o", n.IfName, "-j", "MASQUERADE"} + hpNatArgs = []string{"-m", "addrtype", "--src-type", "LOCAL", "-o", n.IfName, "-j", "MASQUERADE"} + } + natRule := iptables.Rule{IPVer: ipVer, Table: iptables.Nat, Chain: "POSTROUTING", Args: natArgs} + hpNatRule := iptables.Rule{IPVer: ipVer, Table: iptables.Nat, Chain: "POSTROUTING", Args: hpNatArgs} + + // Set NAT. + nat := !config.Routed + if n.Masquerade { + if nat { + if err := programChainRule(natRule, "NAT", enable); err != nil { + return err + } + } + // If the userland proxy is running (!hairpin), skip DNAT for packets originating from + // this new network. Then, the proxy can pick up the packet from the host address the dest + // port is published to. Otherwise, if the packet is DNAT'd, it's forwarded straight to the + // target network, and will be dropped by network isolation rules if it didn't originate in + // the same bridge network. (So, with the proxy enabled, this skip allows a container in one + // network to reach a port published by a container in another bridge network.) + // + // If the userland proxy is disabled, don't skip, so packets will be DNAT'd. That will + // enable access to ports published by containers in the same network. But, the INC rules + // will block access to that published port from containers in other networks. (However, + // users may add a rule to DOCKER-USER to work around the INC rules if needed.) + if !n.ipt.Hairpin { + skipDNAT := iptables.Rule{IPVer: ipVer, Table: iptables.Nat, Chain: dockerChain, Args: []string{ + "-i", n.IfName, + "-j", "RETURN", + }} + if err := programChainRule(skipDNAT, "SKIP DNAT", enable); err != nil { + return err + } + } + } + + // In hairpin mode, masquerade traffic from localhost. If hairpin is disabled or if we're tearing down + // that bridge, make sure the iptables rule isn't lying around. + if err := programChainRule(hpNatRule, "MASQ LOCAL HOST", enable && n.ipt.Hairpin); err != nil { + return err + } + + // Set Inter Container Communication. + if err := setIcc(ipVer, n.IfName, n.ICC, false, enable); err != nil { + return err + } + + // Allow ICMP in routed mode. + if !nat { + if err := setICMP(ipVer, n.IfName, enable); err != nil { + return err + } + } + + // Handle outgoing packets. This rule was previously added unconditionally + // to ACCEPT packets that weren't ICC - an extra rule was needed to enable + // ICC if needed. Those rules are now combined. So, outRuleNoICC is only + // needed for ICC=false, along with the DROP rule for ICC added by setIcc. + outRuleNoICC := iptables.Rule{IPVer: ipVer, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{ + "-i", n.IfName, + "!", "-o", n.IfName, + "-j", "ACCEPT", + }} + // If there's a version of outRuleNoICC in the FORWARD chain, created by moby 28.0.0 or older, delete it. + if enable { + if err := outRuleNoICC.WithChain("FORWARD").Delete(); err != nil { + return fmt.Errorf("deleting FORWARD chain outRuleNoICC: %w", err) + } + } + if n.ICC { + // Accept outgoing traffic to anywhere, including other containers on this bridge. + outRuleICC := iptables.Rule{IPVer: ipVer, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{ + "-i", n.IfName, + "-j", "ACCEPT", + }} + if err := appendOrDelChainRule(outRuleICC, "ACCEPT OUTGOING", enable); err != nil { + return err + } + // If there's a version of outRuleICC in the FORWARD chain, created by moby 28.0.0 or older, delete it. + if enable { + if err := outRuleICC.WithChain("FORWARD").Delete(); err != nil { + return fmt.Errorf("deleting FORWARD chain outRuleICC: %w", err) + } + } + } else { + // Accept outgoing traffic to anywhere, apart from other containers on this bridge. + // setIcc added a DROP rule for ICC traffic. + if err := appendOrDelChainRule(outRuleNoICC, "ACCEPT NON_ICC OUTGOING", enable); err != nil { + return err + } + } + + return nil +} + +func setIcc(version iptables.IPVersion, bridgeIface string, iccEnable, internal, insert bool) error { + args := []string{"-i", bridgeIface, "-o", bridgeIface, "-j"} + acceptRule := iptables.Rule{IPVer: version, Table: iptables.Filter, Chain: DockerForwardChain, Args: append(args, "ACCEPT")} + dropRule := iptables.Rule{IPVer: version, Table: iptables.Filter, Chain: DockerForwardChain, Args: append(args, "DROP")} + + // The accept rule is no longer required for a bridge with external connectivity, because + // ICC traffic is allowed by the outgoing-packets rule created by setupIptablesInternal. + // The accept rule is still required for a --internal network because it has no outgoing + // rule. If insert and the rule is not required, an ACCEPT rule for an external network + // may have been left behind by an older version of the daemon so, delete it. + if insert && iccEnable && internal { + if err := acceptRule.Append(); err != nil { + return fmt.Errorf("Unable to allow intercontainer communication: %w", err) + } + } else { + if err := acceptRule.Delete(); err != nil { + log.G(context.TODO()).WithError(err).Warn("Failed to delete legacy ICC accept rule") + } + } + + if insert && !iccEnable { + if err := dropRule.Append(); err != nil { + return fmt.Errorf("Unable to prevent intercontainer communication: %w", err) + } + } else { + if err := dropRule.Delete(); err != nil { + log.G(context.TODO()).WithError(err).Warn("Failed to delete ICC drop rule") + } + } + + // Delete rules that may have been inserted into the FORWARD chain by moby 28.0.0 or older. + if insert { + if err := acceptRule.WithChain("FORWARD").Delete(); err != nil { + return fmt.Errorf("deleting FORWARD chain accept rule: %w", err) + } + if err := dropRule.WithChain("FORWARD").Delete(); err != nil { + return fmt.Errorf("deleting FORWARD chain drop rule: %w", err) + } + } + return nil +} + +// Control Inter-Network Communication. +// Install rules only if they aren't present, remove only if they are. +// If this method returns an error, it doesn't roll back any rules it has added. +// No error is returned if rules cannot be removed (errors are just logged). +func setINC(version iptables.IPVersion, iface string, routed, enable bool) (retErr error) { + iptable := iptables.GetIptable(version) + actionI, actionA := iptables.Insert, iptables.Append + actionMsg := "add" + if !enable { + actionI, actionA = iptables.Delete, iptables.Delete + actionMsg = "remove" + } + + if routed { + // Anything is allowed into a routed network at this stage, so RETURN. Port + // filtering rules in the DOCKER chain will drop anything that's not destined + // for an open port. + if err := iptable.ProgramRule(iptables.Filter, isolationChain1, actionI, []string{ + "-o", iface, + "-j", "RETURN", + }); err != nil { + log.G(context.TODO()).WithError(err).Warnf("Failed to %s inter-network communication rule", actionMsg) + if enable { + return fmt.Errorf("%s inter-network communication rule: %w", actionMsg, err) + } + } + + // Allow responses from the routed network into whichever network made the request. + if err := iptable.ProgramRule(iptables.Filter, isolationChain1, actionI, []string{ + "-i", iface, + "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", + "-j", "ACCEPT", + }); err != nil { + log.G(context.TODO()).WithError(err).Warnf("Failed to %s inter-network communication rule", actionMsg) + if enable { + return fmt.Errorf("%s inter-network communication rule: %w", actionMsg, err) + } + } + } + + if err := iptable.ProgramRule(iptables.Filter, isolationChain1, actionA, []string{ + "-i", iface, + "!", "-o", iface, + "-j", isolationChain2, + }); err != nil { + log.G(context.TODO()).WithError(err).Warnf("Failed to %s inter-network communication rule", actionMsg) + if enable { + return fmt.Errorf("%s inter-network communication rule: %w", actionMsg, err) + } + } + + if err := iptable.ProgramRule(iptables.Filter, isolationChain2, actionI, []string{ + "-o", iface, + "-j", "DROP", + }); err != nil { + log.G(context.TODO()).WithError(err).Warnf("Failed to %s inter-network communication rule", actionMsg) + if enable { + return fmt.Errorf("%s inter-network communication rule: %w", actionMsg, err) + } + } + + return nil +} + +// Obsolete chain from previous docker versions +const oldIsolationChain = "DOCKER-ISOLATION" + +func removeIPChains(version iptables.IPVersion) { + ipt := iptables.GetIptable(version) + + // Remove obsolete rules from default chains + ipt.ProgramRule(iptables.Filter, "FORWARD", iptables.Delete, []string{"-j", oldIsolationChain}) + + // Remove chains + for _, chainInfo := range []iptables.ChainInfo{ + {Name: dockerChain, Table: iptables.Nat, IPVersion: version}, + {Name: dockerChain, Table: iptables.Filter, IPVersion: version}, + {Name: DockerForwardChain, Table: iptables.Filter, IPVersion: version}, + {Name: dockerBridgeChain, Table: iptables.Filter, IPVersion: version}, + {Name: dockerCTChain, Table: iptables.Filter, IPVersion: version}, + {Name: isolationChain1, Table: iptables.Filter, IPVersion: version}, + {Name: isolationChain2, Table: iptables.Filter, IPVersion: version}, + {Name: oldIsolationChain, Table: iptables.Filter, IPVersion: version}, + } { + if err := chainInfo.Remove(); err != nil { + log.G(context.TODO()).Warnf("Failed to remove existing iptables entries in table %s chain %s : %v", chainInfo.Table, chainInfo.Name, err) + } + } +} + +func setupInternalNetworkRules(bridgeIface string, prefix netip.Prefix, icc, insert bool) error { + var version iptables.IPVersion + var inDropRule, outDropRule iptables.Rule + + // Either add or remove the interface from the firewalld zone, if firewalld is running. + if insert { + if err := iptables.AddInterfaceFirewalld(bridgeIface); err != nil { + return err + } + } else { + if err := iptables.DelInterfaceFirewalld(bridgeIface); err != nil && !errdefs.IsNotFound(err) { + return err + } + } + + if prefix.Addr().Is4() { + version = iptables.IPv4 + inDropRule = iptables.Rule{ + IPVer: version, + Table: iptables.Filter, + Chain: isolationChain1, + Args: []string{"-i", bridgeIface, "!", "-d", prefix.String(), "-j", "DROP"}, + } + outDropRule = iptables.Rule{ + IPVer: version, + Table: iptables.Filter, + Chain: isolationChain1, + Args: []string{"-o", bridgeIface, "!", "-s", prefix.String(), "-j", "DROP"}, + } + } else { + version = iptables.IPv6 + inDropRule = iptables.Rule{ + IPVer: version, + Table: iptables.Filter, + Chain: isolationChain1, + Args: []string{"-i", bridgeIface, "!", "-o", bridgeIface, "!", "-d", prefix.String(), "-j", "DROP"}, + } + outDropRule = iptables.Rule{ + IPVer: version, + Table: iptables.Filter, + Chain: isolationChain1, + Args: []string{"!", "-i", bridgeIface, "-o", bridgeIface, "!", "-s", prefix.String(), "-j", "DROP"}, + } + } + + if err := programChainRule(inDropRule, "DROP INCOMING", insert); err != nil { + return err + } + if err := programChainRule(outDropRule, "DROP OUTGOING", insert); err != nil { + return err + } + + // Set Inter Container Communication. + return setIcc(version, bridgeIface, icc, true, insert) +} diff --git a/libnetwork/drivers/bridge/internal/iptabler/network_test.go b/libnetwork/drivers/bridge/internal/iptabler/network_test.go new file mode 100644 index 0000000000..f7b6cd5808 --- /dev/null +++ b/libnetwork/drivers/bridge/internal/iptabler/network_test.go @@ -0,0 +1,312 @@ +//go:build linux + +package iptabler + +import ( + "net/netip" + "testing" + + "github.com/docker/docker/internal/testutils/netnsutils" + "github.com/docker/docker/libnetwork/iptables" + "gotest.tools/v3/assert" + is "gotest.tools/v3/assert/cmp" +) + +const ( + defaultBridgeName = "testbridge" +) + +func TestProgramIPTable(t *testing.T) { + // Create a test bridge with a basic bridge configuration (name + IPv4). + defer netnsutils.SetupTestOSContext(t)() + + _, err := iptables.GetIptable(iptables.IPv4).NewChain(DockerForwardChain, iptables.Filter) + assert.NilError(t, err) + + // Store various iptables chain rules we care for. + const iptablesTestBridgeIP = "192.168.42.1" + rules := []struct { + rule iptables.Rule + descr string + }{ + {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-d", "127.1.2.3", "-i", "lo", "-o", "lo", "-j", "DROP"}}, "Test Loopback"}, + {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", iptablesTestBridgeIP, "!", "-o", defaultBridgeName, "-j", "MASQUERADE"}}, "NAT Test"}, + {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-o", defaultBridgeName, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}}, "Test ACCEPT INCOMING"}, + {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-i", defaultBridgeName, "!", "-o", defaultBridgeName, "-j", "ACCEPT"}}, "Test ACCEPT NON_ICC OUTGOING"}, + {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-i", defaultBridgeName, "-o", defaultBridgeName, "-j", "ACCEPT"}}, "Test enable ICC"}, + {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-i", defaultBridgeName, "-o", defaultBridgeName, "-j", "DROP"}}, "Test disable ICC"}, + } + + // Assert the chain rules' insertion and removal. + for _, c := range rules { + // Add + if err := programChainRule(c.rule, c.descr, true); err != nil { + t.Fatalf("Failed to program iptable rule %s: %s", c.descr, err.Error()) + } + + if !c.rule.Exists() { + t.Fatalf("Failed to effectively program iptable rule: %s", c.descr) + } + + // Remove + if err := programChainRule(c.rule, c.descr, false); err != nil { + t.Fatalf("Failed to remove iptable rule %s: %s", c.descr, err.Error()) + } + if c.rule.Exists() { + t.Fatalf("Failed to effectively remove iptable rule: %s", c.descr) + } + } +} + +func TestSetupIPChains(t *testing.T) { + // Create a test bridge with a basic bridge configuration (name + IPv4). + defer netnsutils.SetupTestOSContext(t)() + + ipt, err := NewIptabler(FirewallConfig{IPv4: true}) + assert.NilError(t, err) + + nc := NetworkConfig{ + IfName: defaultBridgeName, + Config4: NetworkConfigFam{ + Prefix: netip.MustParsePrefix("192.168.42.0/24"), + }, + } + + assertBridgeConfig(t, ipt, nc) + + nc.Masquerade = true + assertBridgeConfig(t, ipt, nc) + + nc.ICC = true + assertBridgeConfig(t, ipt, nc) + + nc.Masquerade = false + assertBridgeConfig(t, ipt, nc) +} + +// Regression test for https://github.com/moby/moby/issues/46445 +func TestSetupIP6TablesWithHostIPv4(t *testing.T) { + defer netnsutils.SetupTestOSContext(t)() + + ipt, err := NewIptabler(FirewallConfig{ + IPv4: true, + IPv6: true, + }) + assert.NilError(t, err) + + nc := NetworkConfig{ + IfName: defaultBridgeName, + Masquerade: true, + Config4: NetworkConfigFam{ + HostIP: netip.MustParseAddr("192.0.2.2"), + Prefix: netip.MustParsePrefix("192.168.42.0/24"), + }, + Config6: NetworkConfigFam{ + Prefix: netip.MustParsePrefix("2001:db8::/64"), + }, + } + assertBridgeConfig(t, ipt, nc) +} + +// Assert function which pushes chains based on bridge config parameters. +func assertBridgeConfig(t *testing.T, ipt *Iptabler, nc NetworkConfig) { + t.Helper() + n, err := ipt.NewNetwork(nc) + assert.NilError(t, err) + err = n.DelNetworkLevelRules() + assert.NilError(t, err) +} + +func TestOutgoingNATRules(t *testing.T) { + const br = "br-nattest" + maskedBrIPv4 := netip.MustParsePrefix("192.168.42.1/16").Masked() + maskedBrIPv6 := netip.MustParsePrefix("2001:db8::1/64").Masked() + hostIPv4 := netip.MustParseAddr("192.0.2.2") + hostIPv6 := netip.MustParseAddr("2001:db8:1::1") + for _, tc := range []struct { + desc string + enableIPTables bool + enableIP6Tables bool + enableIPv4 bool + enableIPv6 bool + enableIPMasquerade bool + hostIPv4 netip.Addr + hostIPv6 netip.Addr + // Hairpin NAT rules are not tested here because they are orthogonal to outgoing NAT. They + // exist to support the port forwarding DNAT rules: without any port forwarding there would be + // no need for any hairpin NAT rules, and when there is port forwarding then hairpin NAT rules + // are needed even if outgoing NAT is disabled. Hairpin NAT tests belong with the port + // forwarding DNAT tests. + wantIPv4Masq bool + wantIPv4Snat bool + wantIPv6Masq bool + wantIPv6Snat bool + }{ + { + desc: "everything disabled except ipv4", + enableIPv4: true, // one of IPv4 or IPv6 must be enabled + }, + { + desc: "everything disabled except ipv6", + enableIPv6: true, + }, + { + desc: "iptables and ip6tables disabled", + enableIPv4: true, + enableIPv6: true, + enableIPMasquerade: true, + }, + { + desc: "host IP with iptables and ip6tables disabled", + enableIPv4: true, + enableIPv6: true, + enableIPMasquerade: true, + hostIPv4: hostIPv4, + hostIPv6: hostIPv6, + }, + { + desc: "masquerade disabled, no host IP", + enableIPTables: true, + enableIP6Tables: true, + enableIPv4: true, + enableIPv6: true, + }, + { + desc: "masquerade disabled, with host IP", + enableIPTables: true, + enableIP6Tables: true, + enableIPv4: true, + enableIPv6: true, + hostIPv4: hostIPv4, + hostIPv6: hostIPv6, + }, + { + desc: "IPv4 masquerade, IPv6 disabled", + enableIPv4: true, + enableIPTables: true, + enableIPMasquerade: true, + wantIPv4Masq: true, + }, + { + desc: "IPv6 masquerade, IPv4 disabled", + enableIPv6: true, + enableIP6Tables: true, + enableIPMasquerade: true, + wantIPv6Masq: true, + }, + { + desc: "IPv4 SNAT, IPv6 disabled", + enableIPv4: true, + enableIPTables: true, + enableIPMasquerade: true, + hostIPv4: hostIPv4, + wantIPv4Snat: true, + }, + { + desc: "IPv6 SNAT, IPv4 disabled", + enableIPv6: true, + enableIP6Tables: true, + enableIPMasquerade: true, + hostIPv6: hostIPv6, + wantIPv6Snat: true, + }, + { + desc: "IPv4 masquerade, IPv6 masquerade", + enableIPTables: true, + enableIP6Tables: true, + enableIPv4: true, + enableIPv6: true, + enableIPMasquerade: true, + wantIPv4Masq: true, + wantIPv6Masq: true, + }, + { + desc: "IPv4 masquerade, IPv6 SNAT", + enableIPTables: true, + enableIP6Tables: true, + enableIPv4: true, + enableIPv6: true, + enableIPMasquerade: true, + hostIPv6: hostIPv6, + wantIPv4Masq: true, + wantIPv6Snat: true, + }, + { + desc: "IPv4 SNAT, IPv6 masquerade", + enableIPTables: true, + enableIP6Tables: true, + enableIPv4: true, + enableIPv6: true, + enableIPMasquerade: true, + hostIPv4: hostIPv4, + wantIPv4Snat: true, + wantIPv6Masq: true, + }, + { + desc: "IPv4 SNAT, IPv6 SNAT", + enableIPTables: true, + enableIP6Tables: true, + enableIPv4: true, + enableIPv6: true, + enableIPMasquerade: true, + hostIPv4: hostIPv4, + hostIPv6: hostIPv6, + wantIPv4Snat: true, + wantIPv6Snat: true, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + defer netnsutils.SetupTestOSContext(t)() + ipt, err := NewIptabler(FirewallConfig{ + IPv4: tc.enableIPTables, + IPv6: tc.enableIP6Tables, + }) + assert.NilError(t, err) + + nc := NetworkConfig{ + IfName: br, + Masquerade: tc.enableIPMasquerade, + Config4: NetworkConfigFam{ + HostIP: tc.hostIPv4, + Prefix: maskedBrIPv4, + }, + Config6: NetworkConfigFam{ + HostIP: tc.hostIPv6, + Prefix: maskedBrIPv6, + }, + } + n, err := ipt.NewNetwork(nc) + assert.NilError(t, err) + + defer func() { + err = n.DelNetworkLevelRules() + assert.NilError(t, err) + }() + + // Log the contents of all chains to aid troubleshooting. + for _, ipv := range []iptables.IPVersion{iptables.IPv4, iptables.IPv6} { + ipt := iptables.GetIptable(ipv) + for _, table := range []iptables.Table{iptables.Nat, iptables.Filter, iptables.Mangle} { + out, err := ipt.Raw("-t", string(table), "-S") + if err != nil { + t.Error(err) + } + t.Logf("%s: %s %s table rules:\n%s", tc.desc, ipv, table, string(out)) + } + } + for i, rc := range []struct { + want bool + rule iptables.Rule + }{ + // Rule order doesn't matter: At most one of the following IPv4 rules will exist, and the + // same goes for the IPv6 rules. + {tc.wantIPv4Masq, iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", maskedBrIPv4.String(), "!", "-o", br, "-j", "MASQUERADE"}}}, + {tc.wantIPv4Snat, iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", maskedBrIPv4.String(), "!", "-o", br, "-j", "SNAT", "--to-source", hostIPv4.String()}}}, + {tc.wantIPv6Masq, iptables.Rule{IPVer: iptables.IPv6, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", maskedBrIPv6.String(), "!", "-o", br, "-j", "MASQUERADE"}}}, + {tc.wantIPv6Snat, iptables.Rule{IPVer: iptables.IPv6, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", maskedBrIPv6.String(), "!", "-o", br, "-j", "SNAT", "--to-source", hostIPv6.String()}}}, + } { + assert.Check(t, is.Equal(rc.rule.Exists(), rc.want), "rule:%d", i) + } + }) + } +} diff --git a/libnetwork/drivers/bridge/internal/iptabler/port.go b/libnetwork/drivers/bridge/internal/iptabler/port.go index a109787cb1..1cad6c62fa 100644 --- a/libnetwork/drivers/bridge/internal/iptabler/port.go +++ b/libnetwork/drivers/bridge/internal/iptabler/port.go @@ -30,6 +30,8 @@ func (n *Network) modPorts(ctx context.Context, pbs []types.PortBinding, enable return nil } +// setPerPortIptables configures rules required by port binding b. Rules are added if +// enable is true, else removed. func (n *Network) setPerPortIptables(ctx context.Context, b types.PortBinding, enable bool) error { v := iptables.IPv4 enabled := n.ipt.IPv4 @@ -72,6 +74,8 @@ func (n *Network) setPerPortIptables(ctx context.Context, b types.PortBinding, e return nil } +// setPerPortNAT configures DNAT and MASQUERADE rules for port binding b. Rules are added if +// enable is true, else removed. func (n *Network) setPerPortNAT(ipv iptables.IPVersion, b types.PortBinding, enable bool) error { if b.HostPort == 0 { // NAT is disabled. @@ -116,6 +120,9 @@ func (n *Network) setPerPortNAT(ipv iptables.IPVersion, b types.PortBinding, ena return nil } +// setPerPortForwarding opens access to a container's published port, as described by binding b. +// It also does something weird, broken, and disabled-by-default related to SCTP. Rules are added +// if enable is true, else removed. func setPerPortForwarding(b types.PortBinding, ipv iptables.IPVersion, bridgeName string, enable bool) error { // Insert rules for open ports at the top of the filter table's DOCKER // chain (a per-network DROP rule, which must come after these per-port @@ -133,6 +140,7 @@ func setPerPortForwarding(b types.PortBinding, ipv iptables.IPVersion, bridgeNam return err } + // TODO(robmry) - remove, see https://github.com/moby/moby/pull/48149 if b.Proto == types.SCTP && os.Getenv("DOCKER_IPTABLES_SCTP_CHECKSUM") == "1" { // Linux kernel v4.9 and below enables NETIF_F_SCTP_CRC for veth by // the following commit. diff --git a/libnetwork/drivers/bridge/internal/iptabler/wsl2.go b/libnetwork/drivers/bridge/internal/iptabler/wsl2.go new file mode 100644 index 0000000000..601203bf31 --- /dev/null +++ b/libnetwork/drivers/bridge/internal/iptabler/wsl2.go @@ -0,0 +1,104 @@ +//go:build linux + +package iptabler + +import ( + "context" + "errors" + "os" + + "github.com/containerd/log" + "github.com/docker/docker/internal/nlwrap" + "github.com/docker/docker/libnetwork/iptables" + "github.com/vishvananda/netlink" +) + +// Path to the executable installed in Linux under WSL2 that reports on +// WSL config. https://github.com/microsoft/WSL/releases/tag/2.0.4 +// Can be modified by tests. +var wslinfoPath = "/usr/bin/wslinfo" + +// mirroredWSL2Workaround adds or removes an IPv4 NAT rule, depending on whether +// docker's host Linux appears to be a guest running under WSL2 in with mirrored +// mode networking. +// https://learn.microsoft.com/en-us/windows/wsl/networking#mirrored-mode-networking +// +// Without mirrored mode networking, or for a packet sent from Linux, packets +// sent to 127.0.0.1 are processed as outgoing - they hit the nat-OUTPUT chain, +// which does not jump to the nat-DOCKER chain because the rule has an exception +// for "-d 127.0.0.0/8". The default action on the nat-OUTPUT chain is ACCEPT (by +// default), so the packet is delivered to 127.0.0.1 on lo, where docker-proxy +// picks it up and acts as a man-in-the-middle; it receives the packet and +// re-sends it to the container (or acks a SYN and sets up a second TCP +// connection to the container). So, the container sees packets arrive with a +// source address belonging to the network's bridge, and it is able to reply to +// that address. +// +// In WSL2's mirrored networking mode, Linux has a loopback0 device as well as lo +// (which owns 127.0.0.1 as normal). Packets sent to 127.0.0.1 from Windows to a +// server listening on Linux's 127.0.0.1 are delivered via loopback0, and +// processed as packets arriving from outside the Linux host (which they are). +// +// So, these packets hit the nat-PREROUTING chain instead of nat-OUTPUT. It would +// normally be impossible for a packet ->127.0.0.1 to arrive from outside the +// host, so the nat-PREROUTING jump to nat-DOCKER has no exception for it. The +// packet is processed by a per-bridge DNAT rule in that chain, so it is +// delivered directly to the container (not via docker-proxy) with source address +// 127.0.0.1, so the container can't respond. +// +// DNAT is normally skipped by RETURN rules in the nat-DOCKER chain for packets +// arriving from any other bridge network. Similarly, this function adds (or +// removes) a rule to RETURN early for packets delivered via loopback0 with +// destination 127.0.0.0/8. +func mirroredWSL2Workaround(ipv iptables.IPVersion, hairpin bool) error { + // WSL2 does not (currently) support Windows<->Linux communication via ::1. + if ipv != iptables.IPv4 { + return nil + } + return programChainRule(mirroredWSL2Rule(), "WSL2 loopback", shouldInsertMirroredWSL2Rule(hairpin)) +} + +// shouldInsertMirroredWSL2Rule returns true if the NAT rule for mirrored WSL2 workaround +// is required. It is required if: +// - the userland proxy is running. If not, there's nothing on the host to catch +// the packet, so the loopback0 rule as wouldn't be useful. However, without +// the workaround, with improvements in WSL2 v2.3.11, and without userland proxy +// running - no workaround is needed, the normal DNAT/masquerading works. +// - and, the host Linux appears to be running under Windows WSL2 with mirrored +// mode networking. +func shouldInsertMirroredWSL2Rule(hairpin bool) bool { + if hairpin { + return false + } + return isRunningUnderWSL2MirroredMode() +} + +// isRunningUnderWSL2MirroredMode returns true if the host Linux appears to be +// running under Windows WSL2 with mirrored mode networking. If a loopback0 +// device exists, and there's an executable at /usr/bin/wslinfo, infer that +// this is WSL2 with mirrored networking. ("wslinfo --networking-mode" reports +// "mirrored", but applying the workaround for WSL2's loopback device when it's +// not needed is low risk, compared with executing wslinfo with dockerd's +// elevated permissions.) +func isRunningUnderWSL2MirroredMode() bool { + if _, err := nlwrap.LinkByName("loopback0"); err != nil { + if !errors.As(err, &netlink.LinkNotFoundError{}) { + log.G(context.TODO()).WithError(err).Warn("Failed to check for WSL interface") + } + return false + } + stat, err := os.Stat(wslinfoPath) + if err != nil { + return false + } + return stat.Mode().IsRegular() && (stat.Mode().Perm()&0o111) != 0 +} + +func mirroredWSL2Rule() iptables.Rule { + return iptables.Rule{ + IPVer: iptables.IPv4, + Table: iptables.Nat, + Chain: dockerChain, + Args: []string{"-i", "loopback0", "-d", "127.0.0.0/8", "-j", "RETURN"}, + } +} diff --git a/libnetwork/drivers/bridge/internal/iptabler/wsl2_test.go b/libnetwork/drivers/bridge/internal/iptabler/wsl2_test.go new file mode 100644 index 0000000000..5c0d20f029 --- /dev/null +++ b/libnetwork/drivers/bridge/internal/iptabler/wsl2_test.go @@ -0,0 +1,160 @@ +//go:build linux + +package iptabler + +import ( + "context" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/docker/docker/libnetwork/types" + "github.com/vishvananda/netlink" + + "github.com/docker/docker/internal/testutils/netnsutils" + "github.com/docker/docker/libnetwork/iptables" + "gotest.tools/v3/assert" + is "gotest.tools/v3/assert/cmp" +) + +func TestMirroredWSL2Workaround(t *testing.T) { + for _, tc := range []struct { + desc string + loopback0 bool + userlandProxy bool + wslinfoPerm os.FileMode // 0 for no-file + expLoopback0Rule bool + }{ + { + desc: "No loopback0", + }, + { + desc: "WSL2 mirrored", + loopback0: true, + userlandProxy: true, + wslinfoPerm: 0o777, + expLoopback0Rule: true, + }, + { + desc: "loopback0 but wslinfo not executable", + loopback0: true, + userlandProxy: true, + wslinfoPerm: 0o666, + }, + { + desc: "loopback0 but no wslinfo", + loopback0: true, + userlandProxy: true, + }, + { + desc: "loopback0 but no userland proxy", + loopback0: true, + wslinfoPerm: 0o777, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + defer netnsutils.SetupTestOSContext(t)() + restoreWslinfoPath := simulateWSL2MirroredMode(t, tc.loopback0, tc.wslinfoPerm) + defer restoreWslinfoPath() + + _, err := NewIptabler(FirewallConfig{ + IPv4: true, + Hairpin: !tc.userlandProxy, + }) + assert.NilError(t, err) + assert.Check(t, is.Equal(mirroredWSL2Rule().Exists(), tc.expLoopback0Rule)) + }) + } +} + +// simulateWSL2MirroredMode simulates the WSL2 mirrored mode by creating a +// loopback0 interface and optionally creating a wslinfo file with the given +// permissions. +// A clean up function is returned and will restore the original wslinfoPath +// used within the 'bridge' package. The loopback0 interface isn't cleaned up. +// Instead this function should be called from a disposable network namespace. +func simulateWSL2MirroredMode(t *testing.T, loopback0 bool, wslinfoPerm os.FileMode) func() { + if loopback0 { + iface := &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: "loopback0", + }, + } + err := netlink.LinkAdd(iface) + assert.NilError(t, err) + } + + wslinfoPathOrig := wslinfoPath + if wslinfoPerm != 0 { + tmpdir := t.TempDir() + p := filepath.Join(tmpdir, "wslinfo") + err := os.WriteFile(p, []byte("#!/bin/sh\necho dummy file\n"), wslinfoPerm) + assert.NilError(t, err) + wslinfoPath = p + } + + return func() { + wslinfoPath = wslinfoPathOrig + } +} + +func TestMirroredWSL2LoopbackFiltering(t *testing.T) { + for _, tc := range []struct { + desc string + loopback0 bool + wslinfoPerm os.FileMode // 0 for no-file + expLoopback0Rule bool + }{ + { + desc: "No loopback0", + }, + { + desc: "WSL2 mirrored", + loopback0: true, + wslinfoPerm: 0o777, + expLoopback0Rule: true, + }, + { + desc: "loopback0 but wslinfo not executable", + loopback0: true, + wslinfoPerm: 0o666, + }, + { + desc: "loopback0 but no wslinfo", + loopback0: true, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + defer netnsutils.SetupTestOSContext(t)() + restoreWslinfoPath := simulateWSL2MirroredMode(t, tc.loopback0, tc.wslinfoPerm) + defer restoreWslinfoPath() + + hostIP := net.ParseIP("127.0.0.1") + err := filterPortMappedOnLoopback(context.Background(), types.PortBinding{ + Proto: types.TCP, + IP: hostIP, + HostPort: 8000, + }, hostIP, true) + assert.NilError(t, err) + + out, err := exec.Command("iptables-save", "-t", "raw").CombinedOutput() + assert.NilError(t, err) + + // Checking this after trying to create rules, to make sure the init code in iptables/firewalld.go has run. + if fw, _ := iptables.UsingFirewalld(); fw { + t.Skip("firewalld is running in the host netns, it can't modify rules in the test's netns") + } + + if tc.expLoopback0Rule { + assert.Check(t, is.Equal(strings.Count(string(out), "-A PREROUTING"), 2)) + assert.Check(t, is.Contains(string(out), "-A PREROUTING -d 127.0.0.1/32 -i loopback0 -p tcp -m tcp --dport 8000 -j ACCEPT")) + } else { + assert.Check(t, is.Equal(strings.Count(string(out), "-A PREROUTING"), 1)) + assert.Check(t, !strings.Contains(string(out), "loopback0"), "There should be no rule in the raw-PREROUTING chain") + } + }) + } +}