diff --git a/libnetwork/drivers/bridge/bridge_linux.go b/libnetwork/drivers/bridge/bridge_linux.go index a38b8390a7..3f7a9e0579 100644 --- a/libnetwork/drivers/bridge/bridge_linux.go +++ b/libnetwork/drivers/bridge/bridge_linux.go @@ -12,11 +12,11 @@ import ( "github.com/containerd/log" "github.com/docker/docker/errdefs" - "github.com/docker/docker/internal/modprobe" "github.com/docker/docker/internal/nlwrap" "github.com/docker/docker/internal/otelutil" "github.com/docker/docker/libnetwork/datastore" "github.com/docker/docker/libnetwork/driverapi" + "github.com/docker/docker/libnetwork/drivers/bridge/internal/iptabler" "github.com/docker/docker/libnetwork/drivers/bridge/internal/rlkclient" "github.com/docker/docker/libnetwork/internal/netiputil" "github.com/docker/docker/libnetwork/iptables" @@ -51,10 +51,10 @@ const ( const spanPrefix = "libnetwork.drivers.bridge" -type ( - iptableCleanFunc func() error - iptablesCleanFuncs []iptableCleanFunc -) +// DockerForwardChain is where libnetwork.programIngress puts Swarm's jump to DOCKER-INGRESS. +// +// FIXME(robmry) - it doesn't belong here. +const DockerForwardChain = iptabler.DockerForwardChain // configuration info for the "bridge" driver. type configuration struct { @@ -67,12 +67,6 @@ type configuration struct { Rootless bool } -type firewaller struct { - IPv4 bool - IPv6 bool - Hairpin bool -} - // networkConfiguration for network specific configuration type networkConfiguration struct { ID string @@ -144,7 +138,7 @@ type bridgeNetwork struct { config *networkConfiguration endpoints map[string]*bridgeEndpoint // key: endpoint id driver *driver // The network's driver - iptablesNetwork *iptablesNetwork + iptablesNetwork *iptabler.Network sync.Mutex } @@ -165,7 +159,7 @@ type driver struct { nlh nlwrap.Handle portDriverClient portDriverClient configNetwork sync.Mutex - firewaller firewaller + firewaller *iptabler.Iptabler sync.Mutex } @@ -401,7 +395,7 @@ func parseErr(label, value, errString string) error { return types.InvalidParameterErrorf("failed to parse %s value: %v (%s)", label, value, errString) } -func (n *bridgeNetwork) newIptablesNetwork() (*iptablesNetwork, error) { +func (n *bridgeNetwork) newIptablesNetwork() (*iptabler.Network, error) { config4, err := makeNetworkConfigFam(n.config.HostIPv4, n.bridge.bridgeIPv4, n.gwMode(iptables.IPv4)) if err != nil { return nil, err @@ -410,7 +404,7 @@ func (n *bridgeNetwork) newIptablesNetwork() (*iptablesNetwork, error) { if err != nil { return nil, err } - return n.driver.firewaller.NewNetwork(networkConfig{ + return n.driver.firewaller.NewNetwork(iptabler.NetworkConfig{ IfName: n.config.BridgeName, Internal: n.config.Internal, ICC: n.config.EnableICC, @@ -420,8 +414,8 @@ func (n *bridgeNetwork) newIptablesNetwork() (*iptablesNetwork, error) { }) } -func makeNetworkConfigFam(hostIP net.IP, bridgePrefix *net.IPNet, gwm gwMode) (networkConfigFam, error) { - c := networkConfigFam{ +func makeNetworkConfigFam(hostIP net.IP, bridgePrefix *net.IPNet, gwm gwMode) (iptabler.NetworkConfigFam, error) { + c := iptabler.NetworkConfigFam{ Routed: gwm.routed(), Unprotected: gwm.unprotected(), } @@ -429,14 +423,14 @@ func makeNetworkConfigFam(hostIP net.IP, bridgePrefix *net.IPNet, gwm gwMode) (n var ok bool c.HostIP, ok = netip.AddrFromSlice(hostIP) if !ok { - return networkConfigFam{}, fmt.Errorf("invalid host address %q", hostIP) + return iptabler.NetworkConfigFam{}, fmt.Errorf("invalid host address %q", hostIP) } c.HostIP = c.HostIP.Unmap() } if bridgePrefix != nil { p, ok := netiputil.ToPrefix(bridgePrefix) if !ok { - return networkConfigFam{}, fmt.Errorf("invalid bridge prefix %s", bridgePrefix) + return iptabler.NetworkConfigFam{}, fmt.Errorf("invalid bridge prefix %s", bridgePrefix) } c.Prefix = p.Masked() } @@ -507,12 +501,13 @@ func (d *driver) configure(option map[string]interface{}) error { return errdefs.InvalidParameter(fmt.Errorf("invalid configuration type (%T) passed", opt)) } - d.firewaller = firewaller{ + var err error + d.firewaller, err = iptabler.NewIptabler(iptabler.FirewallConfig{ IPv4: config.EnableIPTables, IPv6: config.EnableIP6Tables, Hairpin: !config.EnableUserlandProxy || config.UserlandProxyPath == "", - } - if err := d.firewaller.init(); err != nil { + }) + if err != nil { return err } iptables.OnReloaded(d.handleFirewalldReload) @@ -534,56 +529,6 @@ func (d *driver) configure(option map[string]interface{}) error { return d.initStore() } -func (fw *firewaller) init() error { - if fw.IPv4 { - removeIPChains(iptables.IPv4) - - if err := setupIPChains(iptables.IPv4, fw.Hairpin); err != nil { - return err - } - - // Make sure on firewall reload, first thing being re-played is chains creation - iptables.OnReloaded(func() { - log.G(context.TODO()).Debugf("Recreating iptables chains on firewall reload") - if err := setupIPChains(iptables.IPv4, fw.Hairpin); err != nil { - log.G(context.TODO()).WithError(err).Error("Error reloading iptables chains") - } - }) - } - - if fw.IPv6 { - if err := modprobe.LoadModules(context.TODO(), func() error { - iptable := iptables.GetIptable(iptables.IPv6) - _, err := iptable.Raw("-t", "filter", "-n", "-L", "FORWARD") - return err - }, "ip6_tables"); err != nil { - log.G(context.TODO()).WithError(err).Debug("Loading ip6_tables") - } - - removeIPChains(iptables.IPv6) - - err := setupIPChains(iptables.IPv6, fw.Hairpin) - if err != nil { - // If the chains couldn't be set up, it's probably because the kernel has no IPv6 - // support, or it doesn't have module ip6_tables loaded. It won't be possible to - // create IPv6 networks without enabling ip6_tables in the kernel, or disabling - // ip6tables in the daemon config. But, allow the daemon to start because IPv4 - // will work. So, log the problem, and continue. - log.G(context.TODO()).WithError(err).Warn("ip6tables is enabled, but cannot set up ip6tables chains") - } else { - // Make sure on firewall reload, first thing being re-played is chains creation - iptables.OnReloaded(func() { - log.G(context.TODO()).Debugf("Recreating ip6tables chains on firewall reload") - if err := setupIPChains(iptables.IPv6, fw.Hairpin); err != nil { - log.G(context.TODO()).WithError(err).Error("Error reloading ip6tables chains") - } - }) - } - } - - return nil -} - func (d *driver) getNetwork(id string) (*bridgeNetwork, error) { d.Lock() defer d.Unlock() @@ -1051,7 +996,7 @@ func (d *driver) deleteNetwork(nid string) error { // Don't delete the bridge interface if it was not created by libnetwork. } - if err := n.iptablesNetwork.delNetworkLevelRules(); err != nil { + if err := n.iptablesNetwork.DelNetworkLevelRules(); err != nil { log.G(context.TODO()).WithError(err).Warnf("Failed to clean iptables rules for bridge network") } @@ -1607,6 +1552,39 @@ func (d *driver) RevokeExternalConnectivity(nid, eid string) error { return nil } +// clearConntrackEntries flushes conntrack entries matching endpoint IP address +// or matching one of the exposed UDP port. +// In the first case, this could happen if packets were received by the host +// between userland proxy startup and iptables setup. +// In the latter case, this could happen if packets were received whereas there +// were nowhere to route them, as netfilter creates entries in such case. +// This is required because iptables NAT rules are evaluated by netfilter only +// when creating a new conntrack entry. When Docker latter adds NAT rules, +// netfilter ignore them for any packet matching a pre-existing conntrack entry. +// As such, we need to flush all those conntrack entries to make sure NAT rules +// are correctly applied to all packets. +// See: #8795, #44688 & #44742. +func clearConntrackEntries(nlh nlwrap.Handle, ep *bridgeEndpoint) { + var ipv4List []net.IP + var ipv6List []net.IP + var udpPorts []uint16 + + if ep.addr != nil { + ipv4List = append(ipv4List, ep.addr.IP) + } + if ep.addrv6 != nil { + ipv6List = append(ipv6List, ep.addrv6.IP) + } + for _, pb := range ep.portMapping { + if pb.Proto == types.UDP { + udpPorts = append(udpPorts, pb.HostPort) + } + } + + iptables.DeleteConntrackEntries(nlh, ipv4List, ipv6List) + iptables.DeleteConntrackEntriesByPort(nlh, types.UDP, udpPorts) +} + func (d *driver) handleFirewalldReload() { if !d.config.EnableIPTables && !d.config.EnableIP6Tables { return @@ -1643,7 +1621,7 @@ func (d *driver) handleFirewalldReloadNw(nid string) { return } - if err := nw.iptablesNetwork.reapplyNetworkLevelRules(); err != nil { + if err := nw.iptablesNetwork.ReapplyNetworkLevelRules(); err != nil { log.G(context.Background()).WithFields(log.Fields{ "nid": nw.id, "error": err, diff --git a/libnetwork/drivers/bridge/bridge_linux_test.go b/libnetwork/drivers/bridge/bridge_linux_test.go index b354903d3b..4d13b9e501 100644 --- a/libnetwork/drivers/bridge/bridge_linux_test.go +++ b/libnetwork/drivers/bridge/bridge_linux_test.go @@ -32,6 +32,13 @@ import ( "gotest.tools/v3/icmd" ) +const ( + // FIXME(robmry) - remove these and make the tests work for non-iptables firewalls. + dockerChain = "DOCKER" + isolationChain1 = "DOCKER-ISOLATION-STAGE-1" + isolationChain2 = "DOCKER-ISOLATION-STAGE-2" +) + func TestEndpointMarshalling(t *testing.T) { ip1, _ := types.ParseCIDR("172.22.0.9/16") ip2, _ := types.ParseCIDR("2001:db8::9") @@ -301,6 +308,8 @@ func TestCreateFullOptions(t *testing.T) { func TestCreateNoConfig(t *testing.T) { defer netnsutils.SetupTestOSContext(t)() d := newDriver(storeutils.NewTempStore(t)) + err := d.configure(nil) + assert.NilError(t, err) netconfig := &networkConfiguration{BridgeName: DefaultBridgeName, EnableIPv4: true} genericOption := make(map[string]interface{}) @@ -613,17 +622,17 @@ func TestCreateMultipleNetworks(t *testing.T) { // Verify the network isolation rules are installed for each network func verifyV4INCEntries(networks map[string]*bridgeNetwork, t *testing.T) { iptable := iptables.GetIptable(iptables.IPv4) - out1, err := iptable.Raw("-S", IsolationChain1) + out1, err := iptable.Raw("-S", isolationChain1) if err != nil { t.Fatal(err) } - out2, err := iptable.Raw("-S", IsolationChain2) + out2, err := iptable.Raw("-S", isolationChain2) if err != nil { t.Fatal(err) } for _, n := range networks { - re := regexp.MustCompile(fmt.Sprintf("-i %s ! -o %s -j %s", n.config.BridgeName, n.config.BridgeName, IsolationChain2)) + re := regexp.MustCompile(fmt.Sprintf("-i %s ! -o %s -j %s", n.config.BridgeName, n.config.BridgeName, isolationChain2)) matches := re.FindAllString(string(out1[:]), -1) if len(matches) != 1 { t.Fatalf("Cannot find expected inter-network isolation rules in IP Tables for network %s:\n%s.", n.id, string(out1[:])) @@ -964,7 +973,7 @@ func TestLinkContainers(t *testing.T) { t.Fatalf("Failed to program external connectivity: %v", err) } - out, _ := iptable.Raw("-L", DockerChain) + out, _ := iptable.Raw("-L", dockerChain) for _, pm := range exposedPorts { regex := fmt.Sprintf("%s dpt:%d", pm.Proto.String(), pm.Port) re := regexp.MustCompile(regex) @@ -990,7 +999,7 @@ func TestLinkContainers(t *testing.T) { t.Fatal("Failed to unlink ep1 and ep2") } - out, _ = iptable.Raw("-L", DockerChain) + out, _ = iptable.Raw("-L", dockerChain) for _, pm := range exposedPorts { regex := fmt.Sprintf("%s dpt:%d", pm.Proto.String(), pm.Port) re := regexp.MustCompile(regex) @@ -1018,7 +1027,7 @@ func TestLinkContainers(t *testing.T) { } err = d.ProgramExternalConnectivity(context.Background(), "net1", "ep2", sbOptions) if err != nil { - out, _ = iptable.Raw("-L", DockerChain) + out, _ = iptable.Raw("-L", dockerChain) for _, pm := range exposedPorts { regex := fmt.Sprintf("%s dpt:%d", pm.Proto.String(), pm.Port) re := regexp.MustCompile(regex) @@ -1243,57 +1252,6 @@ func TestSetDefaultGw(t *testing.T) { } } -func TestCleanupIptableRules(t *testing.T) { - defer netnsutils.SetupTestOSContext(t)() - bridgeChains := []struct { - name string - table iptables.Table - expRemoved bool - }{ - {name: DockerChain, table: iptables.Nat, expRemoved: true}, - // The filter-FORWARD chain has references to DockerChain and IsolationChain1, - // so the chains won't be removed - but they should be flushed. (This has - // long/always been the case for the daemon, its filter-FORWARD rules aren't - // removed.) - {name: DockerChain, table: iptables.Filter}, - {name: IsolationChain1, table: iptables.Filter}, - } - - ipVersions := []iptables.IPVersion{iptables.IPv4, iptables.IPv6} - - for _, version := range ipVersions { - err := setupIPChains(version, true) - assert.NilError(t, err, "version:%s", version) - - iptable := iptables.GetIptable(version) - for _, chainInfo := range bridgeChains { - exists := iptable.ExistChain(chainInfo.name, chainInfo.table) - assert.Check(t, exists, "version:%s chain:%s table:%v", - version, chainInfo.name, chainInfo.table) - } - - // Insert RETURN rules so that there's something to flush. - for _, chainInfo := range bridgeChains { - out, err := iptable.Raw("-t", string(chainInfo.table), "-A", chainInfo.name, "-j", "RETURN") - assert.NilError(t, err, "version:%s chain:%s table:%v out:%s", - version, chainInfo.name, chainInfo.table, out) - } - - removeIPChains(version) - - for _, chainInfo := range bridgeChains { - exists := iptable.Exists(chainInfo.table, chainInfo.name, "-A", chainInfo.name, "-j", "RETURN") - assert.Check(t, !exists, "version:%s chain:%s table:%v", - version, chainInfo.name, chainInfo.table) - if chainInfo.expRemoved { - exists := iptable.ExistChain(chainInfo.name, chainInfo.table) - assert.Check(t, !exists, "version:%s chain:%s table:%v", - version, chainInfo.name, chainInfo.table) - } - } - } -} - func TestCreateWithExistingBridge(t *testing.T) { defer netnsutils.SetupTestOSContext(t)() d := newDriver(storeutils.NewTempStore(t)) @@ -1407,3 +1365,51 @@ func TestCreateParallel(t *testing.T) { t.Fatalf("Success should be 1 instead: %d", success) } } + +// Regression test for https://github.com/moby/moby/issues/46445 +func TestSetupIP6TablesWithHostIPv4(t *testing.T) { + defer netnsutils.SetupTestOSContext(t)() + d := newDriver(storeutils.NewTempStore(t)) + dc := &configuration{ + EnableIPTables: true, + EnableIP6Tables: true, + } + if err := d.configure(map[string]interface{}{netlabel.GenericData: dc}); err != nil { + t.Fatal(err) + } + nc := &networkConfiguration{ + BridgeName: DefaultBridgeName, + AddressIPv4: &net.IPNet{IP: net.ParseIP("192.168.42.1"), Mask: net.CIDRMask(16, 32)}, + EnableIPMasquerade: true, + EnableIPv4: true, + EnableIPv6: true, + AddressIPv6: &net.IPNet{IP: net.ParseIP("2001:db8::1"), Mask: net.CIDRMask(64, 128)}, + HostIPv4: net.ParseIP("192.0.2.2"), + } + + // Create test bridge. + nh, err := nlwrap.NewHandle() + if err != nil { + t.Fatal(err) + } + br := &bridgeInterface{nlh: nh} + if err := setupDevice(nc, br); err != nil { + t.Fatalf("Failed to create the testing Bridge: %s", err.Error()) + } + if err := setupBridgeIPv4(nc, br); err != nil { + t.Fatalf("Failed to bring up the testing Bridge: %s", err.Error()) + } + if err := setupBridgeIPv6(nc, br); err != nil { + t.Fatalf("Failed to bring up the testing Bridge: %s", err.Error()) + } + + // Check firewall configuration succeeds. + nw := bridgeNetwork{ + config: nc, + driver: d, + bridge: br, + } + fwn, err := nw.newIptablesNetwork() + assert.NilError(t, err) + assert.Check(t, fwn != nil, "no firewaller network") +} diff --git a/libnetwork/drivers/bridge/internal/iptabler/iptabler.go b/libnetwork/drivers/bridge/internal/iptabler/iptabler.go new file mode 100644 index 0000000000..b88e2739a6 --- /dev/null +++ b/libnetwork/drivers/bridge/internal/iptabler/iptabler.go @@ -0,0 +1,265 @@ +//go:build linux + +package iptabler + +import ( + "context" + "fmt" + + "github.com/containerd/log" + "github.com/docker/docker/internal/modprobe" + "github.com/docker/docker/libnetwork/iptables" +) + +const ( + // dockerChain: DOCKER iptable chain name + dockerChain = "DOCKER" + // DockerForwardChain contains Docker's filter-FORWARD rules. + // + // FIXME(robmry) - only exported because it's used to set up the jump to swarm's DOCKER-INGRESS chain. + DockerForwardChain = "DOCKER-FORWARD" + dockerBridgeChain = "DOCKER-BRIDGE" + dockerCTChain = "DOCKER-CT" + + // Isolation between bridge networks is achieved in two stages by means + // of the following two chains in the filter table. The first chain matches + // on the source interface being a bridge network's bridge and the + // destination being a different interface. A positive match leads to the + // second isolation chain. No match returns to the parent chain. The second + // isolation chain matches on destination interface being a bridge network's + // bridge. A positive match identifies a packet originated from one bridge + // network's bridge destined to another bridge network's bridge and will + // result in the packet being dropped. No match returns to the parent chain. + isolationChain1 = "DOCKER-ISOLATION-STAGE-1" + isolationChain2 = "DOCKER-ISOLATION-STAGE-2" +) + +type FirewallConfig struct { + IPv4 bool + IPv6 bool + Hairpin bool +} + +type Iptabler struct { + FirewallConfig +} + +func NewIptabler(config FirewallConfig) (*Iptabler, error) { + ipt := &Iptabler{FirewallConfig: config} + + if ipt.IPv4 { + removeIPChains(iptables.IPv4) + + if err := setupIPChains(iptables.IPv4, ipt.Hairpin); err != nil { + return nil, err + } + + // Make sure on firewall reload, first thing being re-played is chains creation + iptables.OnReloaded(func() { + log.G(context.TODO()).Debugf("Recreating iptables chains on firewall reload") + if err := setupIPChains(iptables.IPv4, ipt.Hairpin); err != nil { + log.G(context.TODO()).WithError(err).Error("Error reloading iptables chains") + } + }) + } + + if ipt.IPv6 { + if err := modprobe.LoadModules(context.TODO(), func() error { + iptable := iptables.GetIptable(iptables.IPv6) + _, err := iptable.Raw("-t", "filter", "-n", "-L", "FORWARD") + return err + }, "ip6_tables"); err != nil { + log.G(context.TODO()).WithError(err).Debug("Loading ip6_tables") + } + + removeIPChains(iptables.IPv6) + + err := setupIPChains(iptables.IPv6, ipt.Hairpin) + if err != nil { + // If the chains couldn't be set up, it's probably because the kernel has no IPv6 + // support, or it doesn't have module ip6_tables loaded. It won't be possible to + // create IPv6 networks without enabling ip6_tables in the kernel, or disabling + // ip6tables in the daemon config. But, allow the daemon to start because IPv4 + // will work. So, log the problem, and continue. + log.G(context.TODO()).WithError(err).Warn("ip6tables is enabled, but cannot set up ip6tables chains") + } else { + // Make sure on firewall reload, first thing being re-played is chains creation + iptables.OnReloaded(func() { + log.G(context.TODO()).Debugf("Recreating ip6tables chains on firewall reload") + if err := setupIPChains(iptables.IPv6, ipt.Hairpin); err != nil { + log.G(context.TODO()).WithError(err).Error("Error reloading ip6tables chains") + } + }) + } + } + + return ipt, nil +} + +func setupIPChains(version iptables.IPVersion, hairpin bool) (retErr error) { + iptable := iptables.GetIptable(version) + + _, err := iptable.NewChain(dockerChain, iptables.Nat) + if err != nil { + return fmt.Errorf("failed to create NAT chain %s: %v", dockerChain, err) + } + defer func() { + if retErr != nil { + if err := iptable.RemoveExistingChain(dockerChain, iptables.Nat); err != nil { + log.G(context.TODO()).Warnf("failed on removing iptables NAT chain %s on cleanup: %v", dockerChain, err) + } + } + }() + + _, err = iptable.NewChain(dockerChain, iptables.Filter) + if err != nil { + return fmt.Errorf("failed to create FILTER chain %s: %v", dockerChain, err) + } + defer func() { + if retErr != nil { + if err := iptable.RemoveExistingChain(dockerChain, iptables.Filter); err != nil { + log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", dockerChain, err) + } + } + }() + + _, err = iptable.NewChain(DockerForwardChain, iptables.Filter) + if err != nil { + return fmt.Errorf("failed to create FILTER chain %s: %v", DockerForwardChain, err) + } + defer func() { + if retErr != nil { + if err := iptable.RemoveExistingChain(DockerForwardChain, iptables.Filter); err != nil { + log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", DockerForwardChain, err) + } + } + }() + + _, err = iptable.NewChain(dockerBridgeChain, iptables.Filter) + if err != nil { + return fmt.Errorf("failed to create FILTER chain %s: %v", dockerBridgeChain, err) + } + defer func() { + if retErr != nil { + if err := iptable.RemoveExistingChain(dockerBridgeChain, iptables.Filter); err != nil { + log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", dockerBridgeChain, err) + } + } + }() + + _, err = iptable.NewChain(dockerCTChain, iptables.Filter) + if err != nil { + return fmt.Errorf("failed to create FILTER chain %s: %v", dockerCTChain, err) + } + defer func() { + if retErr != nil { + if err := iptable.RemoveExistingChain(dockerCTChain, iptables.Filter); err != nil { + log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", dockerCTChain, err) + } + } + }() + + _, err = iptable.NewChain(isolationChain1, iptables.Filter) + if err != nil { + return fmt.Errorf("failed to create FILTER isolation chain: %v", err) + } + defer func() { + if retErr != nil { + if err := iptable.RemoveExistingChain(isolationChain1, iptables.Filter); err != nil { + log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", isolationChain1, err) + } + } + }() + + _, err = iptable.NewChain(isolationChain2, iptables.Filter) + if err != nil { + return fmt.Errorf("failed to create FILTER isolation chain: %v", err) + } + defer func() { + if retErr != nil { + if err := iptable.RemoveExistingChain(isolationChain2, iptables.Filter); err != nil { + log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", isolationChain2, err) + } + } + }() + + if err := addNATJumpRules(version, hairpin, true); err != nil { + return fmt.Errorf("failed to add jump rules to %s NAT table: %w", version, err) + } + defer func() { + if retErr != nil { + if err := addNATJumpRules(version, hairpin, false); err != nil { + log.G(context.TODO()).Warnf("failed on removing jump rules from %s NAT table: %v", version, err) + } + } + }() + + // Make sure the filter-FORWARD chain has rules to accept related packets and + // jump to the isolation and docker chains. (Re-)insert at the top of the table, + // in reverse order. + if err := iptable.EnsureJumpRule("FORWARD", DockerForwardChain); err != nil { + return err + } + if err := iptable.EnsureJumpRule(DockerForwardChain, dockerBridgeChain); err != nil { + return err + } + if err := iptable.EnsureJumpRule(DockerForwardChain, isolationChain1); err != nil { + return err + } + if err := iptable.EnsureJumpRule(DockerForwardChain, dockerCTChain); err != nil { + return err + } + + if err := mirroredWSL2Workaround(version, hairpin); err != nil { + return err + } + + // Delete rules that may have been added to the FORWARD chain by moby 28.0.0. + ipsetName := "docker-ext-bridges-v4" + if version == iptables.IPv6 { + ipsetName = "docker-ext-bridges-v6" + } + if err := iptable.DeleteJumpRule("FORWARD", dockerChain, + "-m", "set", "--match-set", ipsetName, "dst"); err != nil { + log.G(context.TODO()).WithFields(log.Fields{"error": err, "set": ipsetName}).Debug( + "deleting legacy ipset dest match rule") + } + if err := iptable.DeleteJumpRule("FORWARD", isolationChain1); err != nil { + return err + } + if err := iptable.DeleteJumpRule("FORWARD", "ACCEPT", + "-m", "set", "--match-set", ipsetName, "dst", + "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", + ); err != nil { + log.G(context.TODO()).WithFields(log.Fields{"error": err, "set": ipsetName}).Debug( + "deleting legacy ipset conntrack rule") + } + + return nil +} + +func programChainRule(rule iptables.Rule, ruleDescr string, insert bool) error { + operation := "disable" + fn := rule.Delete + if insert { + operation = "enable" + fn = rule.Insert + } + if err := fn(); err != nil { + return fmt.Errorf("Unable to %s %s rule: %w", operation, ruleDescr, err) + } + return nil +} + +func appendOrDelChainRule(rule iptables.Rule, ruleDescr string, append bool) error { + operation := "disable" + fn := rule.Delete + if append { + operation = "enable" + fn = rule.Append + } + if err := fn(); err != nil { + return fmt.Errorf("Unable to %s %s rule: %w", operation, ruleDescr, err) + } + return nil +} diff --git a/libnetwork/drivers/bridge/internal/iptabler/iptabler_test.go b/libnetwork/drivers/bridge/internal/iptabler/iptabler_test.go new file mode 100644 index 0000000000..06b567a438 --- /dev/null +++ b/libnetwork/drivers/bridge/internal/iptabler/iptabler_test.go @@ -0,0 +1,69 @@ +//go:build linux + +package iptabler + +import ( + "testing" + + "github.com/docker/docker/internal/testutils/netnsutils" + "github.com/docker/docker/libnetwork/iptables" + "gotest.tools/v3/assert" +) + +func TestCleanupIptableRules(t *testing.T) { + // Check for existence of a dummy rule to make sure iptables is initialised - then can + // check whether firewalld is running. + _ = iptables.GetIptable(iptables.IPv4).Exists(iptables.Filter, "FORWARD", "-j", "DROP") + if fw, _ := iptables.UsingFirewalld(); fw { + t.Skip("firewalld is running in the host netns, it can't modify rules in the test's netns") + } + + defer netnsutils.SetupTestOSContext(t)() + bridgeChains := []struct { + name string + table iptables.Table + expRemoved bool + }{ + {name: dockerChain, table: iptables.Nat, expRemoved: true}, + // The filter-FORWARD chain has references to dockerChain and isolationChain1, + // so the chains won't be removed - but they should be flushed. (This has + // long/always been the case for the daemon, its filter-FORWARD rules aren't + // removed.) + {name: dockerChain, table: iptables.Filter}, + {name: isolationChain1, table: iptables.Filter}, + } + + ipVersions := []iptables.IPVersion{iptables.IPv4, iptables.IPv6} + + for _, version := range ipVersions { + err := setupIPChains(version, true) + assert.NilError(t, err, "version:%s", version) + + iptable := iptables.GetIptable(version) + for _, chainInfo := range bridgeChains { + exists := iptable.ExistChain(chainInfo.name, chainInfo.table) + assert.Check(t, exists, "version:%s chain:%s table:%v", + version, chainInfo.name, chainInfo.table) + } + + // Insert RETURN rules so that there's something to flush. + for _, chainInfo := range bridgeChains { + out, err := iptable.Raw("-t", string(chainInfo.table), "-A", chainInfo.name, "-j", "RETURN") + assert.NilError(t, err, "version:%s chain:%s table:%v out:%s", + version, chainInfo.name, chainInfo.table, out) + } + + removeIPChains(version) + + for _, chainInfo := range bridgeChains { + exists := iptable.Exists(chainInfo.table, chainInfo.name, "-A", chainInfo.name, "-j", "RETURN") + assert.Check(t, !exists, "version:%s chain:%s table:%v", + version, chainInfo.name, chainInfo.table) + if chainInfo.expRemoved { + exists := iptable.ExistChain(chainInfo.name, chainInfo.table) + assert.Check(t, !exists, "version:%s chain:%s table:%v", + version, chainInfo.name, chainInfo.table) + } + } + } +} diff --git a/libnetwork/drivers/bridge/link.go b/libnetwork/drivers/bridge/internal/iptabler/link.go similarity index 74% rename from libnetwork/drivers/bridge/link.go rename to libnetwork/drivers/bridge/internal/iptabler/link.go index 63be801fb1..90e9d5bb08 100644 --- a/libnetwork/drivers/bridge/link.go +++ b/libnetwork/drivers/bridge/internal/iptabler/link.go @@ -1,6 +1,6 @@ //go:build linux -package bridge +package iptabler import ( "context" @@ -12,7 +12,7 @@ import ( "github.com/docker/docker/libnetwork/types" ) -func (n *iptablesNetwork) AddLink(ctx context.Context, parentIP, childIP netip.Addr, ports []types.TransportPort) error { +func (n *Network) AddLink(ctx context.Context, parentIP, childIP netip.Addr, ports []types.TransportPort) error { if !parentIP.IsValid() || parentIP.IsUnspecified() { return fmt.Errorf("cannot link to a container with an empty parent IP address") } @@ -20,7 +20,7 @@ func (n *iptablesNetwork) AddLink(ctx context.Context, parentIP, childIP netip.A return fmt.Errorf("cannot link to a container with an empty child IP address") } - chain := iptables.ChainInfo{Name: DockerChain} + chain := iptables.ChainInfo{Name: dockerChain} for _, port := range ports { if err := chain.Link(iptables.Append, parentIP, childIP, int(port.Port), port.Proto.String(), n.IfName); err != nil { return err @@ -29,8 +29,8 @@ func (n *iptablesNetwork) AddLink(ctx context.Context, parentIP, childIP netip.A return nil } -func (n *iptablesNetwork) DelLink(ctx context.Context, parentIP, childIP netip.Addr, ports []types.TransportPort) { - chain := iptables.ChainInfo{Name: DockerChain} +func (n *Network) DelLink(ctx context.Context, parentIP, childIP netip.Addr, ports []types.TransportPort) { + chain := iptables.ChainInfo{Name: dockerChain} for _, port := range ports { if err := chain.Link(iptables.Delete, parentIP, childIP, int(port.Port), port.Proto.String(), n.IfName); err != nil { log.G(ctx).WithFields(log.Fields{ diff --git a/libnetwork/drivers/bridge/setup_ip_tables_linux.go b/libnetwork/drivers/bridge/internal/iptabler/network.go similarity index 55% rename from libnetwork/drivers/bridge/setup_ip_tables_linux.go rename to libnetwork/drivers/bridge/internal/iptabler/network.go index a7e89f2eef..4bc9c114e5 100644 --- a/libnetwork/drivers/bridge/setup_ip_tables_linux.go +++ b/libnetwork/drivers/bridge/internal/iptabler/network.go @@ -1,237 +1,71 @@ -package bridge +//go:build linux + +package iptabler import ( "context" "errors" "fmt" - "net" "net/netip" - "os" "github.com/containerd/log" "github.com/docker/docker/errdefs" - "github.com/docker/docker/internal/nlwrap" "github.com/docker/docker/libnetwork/iptables" - "github.com/docker/docker/libnetwork/types" - "github.com/vishvananda/netlink" ) -// DockerChain: DOCKER iptable chain name -const ( - DockerChain = "DOCKER" - DockerForwardChain = "DOCKER-FORWARD" - DockerBridgeChain = "DOCKER-BRIDGE" - DockerCTChain = "DOCKER-CT" - - // Isolation between bridge networks is achieved in two stages by means - // of the following two chains in the filter table. The first chain matches - // on the source interface being a bridge network's bridge and the - // destination being a different interface. A positive match leads to the - // second isolation chain. No match returns to the parent chain. The second - // isolation chain matches on destination interface being a bridge network's - // bridge. A positive match identifies a packet originated from one bridge - // network's bridge destined to another bridge network's bridge and will - // result in the packet being dropped. No match returns to the parent chain. - - IsolationChain1 = "DOCKER-ISOLATION-STAGE-1" - IsolationChain2 = "DOCKER-ISOLATION-STAGE-2" +type ( + iptableCleanFunc func() error + iptablesCleanFuncs []iptableCleanFunc ) -// Path to the executable installed in Linux under WSL2 that reports on -// WSL config. https://github.com/microsoft/WSL/releases/tag/2.0.4 -// Can be modified by tests. -var wslinfoPath = "/usr/bin/wslinfo" - -func setupIPChains(version iptables.IPVersion, hairpin bool) (retErr error) { - iptable := iptables.GetIptable(version) - - _, err := iptable.NewChain(DockerChain, iptables.Nat) - if err != nil { - return fmt.Errorf("failed to create NAT chain %s: %v", DockerChain, err) - } - defer func() { - if retErr != nil { - if err := iptable.RemoveExistingChain(DockerChain, iptables.Nat); err != nil { - log.G(context.TODO()).Warnf("failed on removing iptables NAT chain %s on cleanup: %v", DockerChain, err) - } - } - }() - - _, err = iptable.NewChain(DockerChain, iptables.Filter) - if err != nil { - return fmt.Errorf("failed to create FILTER chain %s: %v", DockerChain, err) - } - defer func() { - if retErr != nil { - if err := iptable.RemoveExistingChain(DockerChain, iptables.Filter); err != nil { - log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", DockerChain, err) - } - } - }() - - _, err = iptable.NewChain(DockerForwardChain, iptables.Filter) - if err != nil { - return fmt.Errorf("failed to create FILTER chain %s: %v", DockerForwardChain, err) - } - defer func() { - if retErr != nil { - if err := iptable.RemoveExistingChain(DockerForwardChain, iptables.Filter); err != nil { - log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", DockerForwardChain, err) - } - } - }() - - _, err = iptable.NewChain(DockerBridgeChain, iptables.Filter) - if err != nil { - return fmt.Errorf("failed to create FILTER chain %s: %v", DockerBridgeChain, err) - } - defer func() { - if retErr != nil { - if err := iptable.RemoveExistingChain(DockerBridgeChain, iptables.Filter); err != nil { - log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", DockerBridgeChain, err) - } - } - }() - - _, err = iptable.NewChain(DockerCTChain, iptables.Filter) - if err != nil { - return fmt.Errorf("failed to create FILTER chain %s: %v", DockerCTChain, err) - } - defer func() { - if retErr != nil { - if err := iptable.RemoveExistingChain(DockerCTChain, iptables.Filter); err != nil { - log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", DockerCTChain, err) - } - } - }() - - _, err = iptable.NewChain(IsolationChain1, iptables.Filter) - if err != nil { - return fmt.Errorf("failed to create FILTER isolation chain: %v", err) - } - defer func() { - if retErr != nil { - if err := iptable.RemoveExistingChain(IsolationChain1, iptables.Filter); err != nil { - log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", IsolationChain1, err) - } - } - }() - - _, err = iptable.NewChain(IsolationChain2, iptables.Filter) - if err != nil { - return fmt.Errorf("failed to create FILTER isolation chain: %v", err) - } - defer func() { - if retErr != nil { - if err := iptable.RemoveExistingChain(IsolationChain2, iptables.Filter); err != nil { - log.G(context.TODO()).Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", IsolationChain2, err) - } - } - }() - - if err := addNATJumpRules(version, hairpin, true); err != nil { - return fmt.Errorf("failed to add jump rules to %s NAT table: %w", version, err) - } - defer func() { - if retErr != nil { - if err := addNATJumpRules(version, hairpin, false); err != nil { - log.G(context.TODO()).Warnf("failed on removing jump rules from %s NAT table: %v", version, err) - } - } - }() - - // Make sure the filter-FORWARD chain has rules to accept related packets and - // jump to the isolation and docker chains. (Re-)insert at the top of the table, - // in reverse order. - if err := iptable.EnsureJumpRule("FORWARD", DockerForwardChain); err != nil { - return err - } - if err := iptable.EnsureJumpRule(DockerForwardChain, DockerBridgeChain); err != nil { - return err - } - if err := iptable.EnsureJumpRule(DockerForwardChain, IsolationChain1); err != nil { - return err - } - if err := iptable.EnsureJumpRule(DockerForwardChain, DockerCTChain); err != nil { - return err - } - - if err := mirroredWSL2Workaround(version, hairpin); err != nil { - return err - } - - // Delete rules that may have been added to the FORWARD chain by moby 28.0.0. - ipsetName := "docker-ext-bridges-v4" - if version == iptables.IPv6 { - ipsetName = "docker-ext-bridges-v6" - } - if err := iptable.DeleteJumpRule("FORWARD", DockerChain, - "-m", "set", "--match-set", ipsetName, "dst"); err != nil { - log.G(context.TODO()).WithFields(log.Fields{"error": err, "set": ipsetName}).Debug( - "deleting legacy ipset dest match rule") - } - if err := iptable.DeleteJumpRule("FORWARD", IsolationChain1); err != nil { - return err - } - if err := iptable.DeleteJumpRule("FORWARD", "ACCEPT", - "-m", "set", "--match-set", ipsetName, "dst", - "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", - ); err != nil { - log.G(context.TODO()).WithFields(log.Fields{"error": err, "set": ipsetName}).Debug( - "deleting legacy ipset conntrack rule") - } - - return nil -} - -type networkConfigFam struct { +type NetworkConfigFam struct { HostIP netip.Addr Prefix netip.Prefix Routed bool Unprotected bool } -type networkConfig struct { +type NetworkConfig struct { IfName string Internal bool ICC bool Masquerade bool - Config4 networkConfigFam - Config6 networkConfigFam + Config4 NetworkConfigFam + Config6 NetworkConfigFam } -type iptablesNetwork struct { - networkConfig - fw *firewaller +type Network struct { + NetworkConfig + ipt *Iptabler cleanFuncs iptablesCleanFuncs } -func (fw *firewaller) NewNetwork(nc networkConfig) (_ *iptablesNetwork, retErr error) { - n := &iptablesNetwork{ - fw: fw, - networkConfig: nc, +func (ipt *Iptabler) NewNetwork(nc NetworkConfig) (_ *Network, retErr error) { + n := &Network{ + ipt: ipt, + NetworkConfig: nc, } defer func() { if retErr != nil { - if err := n.delNetworkLevelRules(); err != nil { + if err := n.DelNetworkLevelRules(); err != nil { log.G(context.TODO()).WithError(err).Warnf("Failed to delete network level rules following earlier error") } } }() - if err := n.reapplyNetworkLevelRules(); err != nil { + if err := n.ReapplyNetworkLevelRules(); err != nil { return nil, err } return n, nil } -func (n *iptablesNetwork) reapplyNetworkLevelRules() error { - if n.fw.IPv4 { +func (n *Network) ReapplyNetworkLevelRules() error { + if n.ipt.IPv4 { if err := n.configure(iptables.IPv4, n.Config4); err != nil { return err } } - if n.fw.IPv6 { + if n.ipt.IPv6 { if err := n.configure(iptables.IPv6, n.Config6); err != nil { return err } @@ -239,7 +73,7 @@ func (n *iptablesNetwork) reapplyNetworkLevelRules() error { return nil } -func (n *iptablesNetwork) delNetworkLevelRules() error { +func (n *Network) DelNetworkLevelRules() error { var errs []error for _, cleanFunc := range n.cleanFuncs { if err := cleanFunc(); err != nil { @@ -250,7 +84,7 @@ func (n *iptablesNetwork) delNetworkLevelRules() error { return errors.Join(errs...) } -func (n *iptablesNetwork) configure(ipv iptables.IPVersion, conf networkConfigFam) error { +func (n *Network) configure(ipv iptables.IPVersion, conf NetworkConfigFam) error { if !conf.Prefix.IsValid() { // Delete INC rules, in case they were created by a 28.0.0 daemon that didn't check // whether the network had iptables/ip6tables enabled. @@ -263,11 +97,11 @@ func (n *iptablesNetwork) configure(ipv iptables.IPVersion, conf networkConfigFa return nil } -func (n *iptablesNetwork) registerCleanFunc(clean iptableCleanFunc) { +func (n *Network) registerCleanFunc(clean iptableCleanFunc) { n.cleanFuncs = append(n.cleanFuncs, clean) } -func (n *iptablesNetwork) setupIPTables(ipVersion iptables.IPVersion, config networkConfigFam) error { +func (n *Network) setupIPTables(ipVersion iptables.IPVersion, config NetworkConfigFam) error { if n.Internal { if err := setupInternalNetworkRules(n.IfName, config.Prefix, n.ICC, true); err != nil { return fmt.Errorf("Failed to Setup IP tables: %w", err) @@ -305,7 +139,7 @@ func (n *iptablesNetwork) setupIPTables(ipVersion iptables.IPVersion, config net return setDefaultForwardRule(ipVersion, n.IfName, config.Unprotected, false) }) - ctRule := iptables.Rule{IPVer: ipVersion, Table: iptables.Filter, Chain: DockerCTChain, Args: []string{ + ctRule := iptables.Rule{IPVer: ipVersion, Table: iptables.Filter, Chain: dockerCTChain, Args: []string{ "-o", n.IfName, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT", @@ -316,9 +150,9 @@ func (n *iptablesNetwork) setupIPTables(ipVersion iptables.IPVersion, config net n.registerCleanFunc(func() error { return appendOrDelChainRule(ctRule, "bridge ct related", false) }) - jumpToDockerRule := iptables.Rule{IPVer: ipVersion, Table: iptables.Filter, Chain: DockerBridgeChain, Args: []string{ + jumpToDockerRule := iptables.Rule{IPVer: ipVersion, Table: iptables.Filter, Chain: dockerBridgeChain, Args: []string{ "-o", n.IfName, - "-j", DockerChain, + "-j", dockerChain, }} if err := appendOrDelChainRule(jumpToDockerRule, "jump to docker", true); err != nil { return err @@ -344,7 +178,7 @@ func setICMP(ipv iptables.IPVersion, bridgeName string, enable bool) error { if ipv == iptables.IPv6 { icmpProto = "icmpv6" } - icmpRule := iptables.Rule{IPVer: ipv, Table: iptables.Filter, Chain: DockerChain, Args: []string{ + icmpRule := iptables.Rule{IPVer: ipv, Table: iptables.Filter, Chain: dockerChain, Args: []string{ "-o", bridgeName, "-p", icmpProto, "-j", "ACCEPT", @@ -356,7 +190,7 @@ func addNATJumpRules(ipVer iptables.IPVersion, hairpinMode, enable bool) error { preroute := iptables.Rule{IPVer: ipVer, Table: iptables.Nat, Chain: "PREROUTING", Args: []string{ "-m", "addrtype", "--dst-type", "LOCAL", - "-j", DockerChain, + "-j", dockerChain, }} if enable { if err := preroute.Append(); err != nil { @@ -371,7 +205,7 @@ func addNATJumpRules(ipVer iptables.IPVersion, hairpinMode, enable bool) error { output := iptables.Rule{IPVer: ipVer, Table: iptables.Nat, Chain: "OUTPUT", Args: []string{ "-m", "addrtype", "--dst-type", "LOCAL", - "-j", DockerChain, + "-j", dockerChain, }} if !hairpinMode { output.Args = append(output.Args, "!", "--dst", loopbackAddress(ipVer)) @@ -398,14 +232,14 @@ func deleteLegacyFilterRules(ipVer iptables.IPVersion, bridgeName string) error // These rules have been replaced by an ipset-matching rule. link := []string{ "-o", bridgeName, - "-j", DockerChain, + "-j", dockerChain, } if iptable.Exists(iptables.Filter, "FORWARD", link...) { del := append([]string{string(iptables.Delete), "FORWARD"}, link...) if output, err := iptable.Raw(del...); err != nil { return err } else if len(output) != 0 { - return fmt.Errorf("could not delete linking rule from %s-%s: %s", iptables.Filter, DockerChain, output) + return fmt.Errorf("could not delete linking rule from %s-%s: %s", iptables.Filter, dockerChain, output) } } @@ -422,7 +256,7 @@ func deleteLegacyFilterRules(ipVer iptables.IPVersion, bridgeName string) error if output, err := iptable.Raw(del...); err != nil { return err } else if len(output) != 0 { - return fmt.Errorf("could not delete establish rule from %s-%s: %s", iptables.Filter, DockerChain, output) + return fmt.Errorf("could not delete establish rule from %s-%s: %s", iptables.Filter, dockerChain, output) } } @@ -455,7 +289,7 @@ func setDefaultForwardRule(ipVersion iptables.IPVersion, ifName string, unprotec action = "ACCEPT" } - rule := iptables.Rule{IPVer: ipVersion, Table: iptables.Filter, Chain: DockerChain, Args: []string{ + rule := iptables.Rule{IPVer: ipVersion, Table: iptables.Filter, Chain: dockerChain, Args: []string{ "!", "-i", ifName, "-o", ifName, "-j", action, @@ -469,7 +303,7 @@ func setDefaultForwardRule(ipVersion iptables.IPVersion, ifName string, unprotec return nil } -func (n *iptablesNetwork) setupNonInternalNetworkRules(ipVer iptables.IPVersion, config networkConfigFam, enable bool) error { +func (n *Network) setupNonInternalNetworkRules(ipVer iptables.IPVersion, config NetworkConfigFam, enable bool) error { var natArgs, hpNatArgs []string if config.HostIP.IsValid() { // The user wants IPv4/IPv6 SNAT with the given address. @@ -503,8 +337,8 @@ func (n *iptablesNetwork) setupNonInternalNetworkRules(ipVer iptables.IPVersion, // enable access to ports published by containers in the same network. But, the INC rules // will block access to that published port from containers in other networks. (However, // users may add a rule to DOCKER-USER to work around the INC rules if needed.) - if !n.fw.Hairpin { - skipDNAT := iptables.Rule{IPVer: ipVer, Table: iptables.Nat, Chain: DockerChain, Args: []string{ + if !n.ipt.Hairpin { + skipDNAT := iptables.Rule{IPVer: ipVer, Table: iptables.Nat, Chain: dockerChain, Args: []string{ "-i", n.IfName, "-j", "RETURN", }} @@ -516,7 +350,7 @@ func (n *iptablesNetwork) setupNonInternalNetworkRules(ipVer iptables.IPVersion, // In hairpin mode, masquerade traffic from localhost. If hairpin is disabled or if we're tearing down // that bridge, make sure the iptables rule isn't lying around. - if err := programChainRule(hpNatRule, "MASQ LOCAL HOST", enable && n.fw.Hairpin); err != nil { + if err := programChainRule(hpNatRule, "MASQ LOCAL HOST", enable && n.ipt.Hairpin); err != nil { return err } @@ -573,32 +407,6 @@ func (n *iptablesNetwork) setupNonInternalNetworkRules(ipVer iptables.IPVersion, return nil } -func programChainRule(rule iptables.Rule, ruleDescr string, insert bool) error { - operation := "disable" - fn := rule.Delete - if insert { - operation = "enable" - fn = rule.Insert - } - if err := fn(); err != nil { - return fmt.Errorf("Unable to %s %s rule: %w", operation, ruleDescr, err) - } - return nil -} - -func appendOrDelChainRule(rule iptables.Rule, ruleDescr string, append bool) error { - operation := "disable" - fn := rule.Delete - if append { - operation = "enable" - fn = rule.Append - } - if err := fn(); err != nil { - return fmt.Errorf("Unable to %s %s rule: %w", operation, ruleDescr, err) - } - return nil -} - func setIcc(version iptables.IPVersion, bridgeIface string, iccEnable, internal, insert bool) error { args := []string{"-i", bridgeIface, "-o", bridgeIface, "-j"} acceptRule := iptables.Rule{IPVer: version, Table: iptables.Filter, Chain: DockerForwardChain, Args: append(args, "ACCEPT")} @@ -658,7 +466,7 @@ func setINC(version iptables.IPVersion, iface string, routed, enable bool) (retE // Anything is allowed into a routed network at this stage, so RETURN. Port // filtering rules in the DOCKER chain will drop anything that's not destined // for an open port. - if err := iptable.ProgramRule(iptables.Filter, IsolationChain1, actionI, []string{ + if err := iptable.ProgramRule(iptables.Filter, isolationChain1, actionI, []string{ "-o", iface, "-j", "RETURN", }); err != nil { @@ -669,7 +477,7 @@ func setINC(version iptables.IPVersion, iface string, routed, enable bool) (retE } // Allow responses from the routed network into whichever network made the request. - if err := iptable.ProgramRule(iptables.Filter, IsolationChain1, actionI, []string{ + if err := iptable.ProgramRule(iptables.Filter, isolationChain1, actionI, []string{ "-i", iface, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT", @@ -681,10 +489,10 @@ func setINC(version iptables.IPVersion, iface string, routed, enable bool) (retE } } - if err := iptable.ProgramRule(iptables.Filter, IsolationChain1, actionA, []string{ + if err := iptable.ProgramRule(iptables.Filter, isolationChain1, actionA, []string{ "-i", iface, "!", "-o", iface, - "-j", IsolationChain2, + "-j", isolationChain2, }); err != nil { log.G(context.TODO()).WithError(err).Warnf("Failed to %s inter-network communication rule", actionMsg) if enable { @@ -692,7 +500,7 @@ func setINC(version iptables.IPVersion, iface string, routed, enable bool) (retE } } - if err := iptable.ProgramRule(iptables.Filter, IsolationChain2, actionI, []string{ + if err := iptable.ProgramRule(iptables.Filter, isolationChain2, actionI, []string{ "-o", iface, "-j", "DROP", }); err != nil { @@ -716,13 +524,13 @@ func removeIPChains(version iptables.IPVersion) { // Remove chains for _, chainInfo := range []iptables.ChainInfo{ - {Name: DockerChain, Table: iptables.Nat, IPVersion: version}, - {Name: DockerChain, Table: iptables.Filter, IPVersion: version}, + {Name: dockerChain, Table: iptables.Nat, IPVersion: version}, + {Name: dockerChain, Table: iptables.Filter, IPVersion: version}, {Name: DockerForwardChain, Table: iptables.Filter, IPVersion: version}, - {Name: DockerBridgeChain, Table: iptables.Filter, IPVersion: version}, - {Name: DockerCTChain, Table: iptables.Filter, IPVersion: version}, - {Name: IsolationChain1, Table: iptables.Filter, IPVersion: version}, - {Name: IsolationChain2, Table: iptables.Filter, IPVersion: version}, + {Name: dockerBridgeChain, Table: iptables.Filter, IPVersion: version}, + {Name: dockerCTChain, Table: iptables.Filter, IPVersion: version}, + {Name: isolationChain1, Table: iptables.Filter, IPVersion: version}, + {Name: isolationChain2, Table: iptables.Filter, IPVersion: version}, {Name: oldIsolationChain, Table: iptables.Filter, IPVersion: version}, } { if err := chainInfo.Remove(); err != nil { @@ -751,13 +559,13 @@ func setupInternalNetworkRules(bridgeIface string, prefix netip.Prefix, icc, ins inDropRule = iptables.Rule{ IPVer: version, Table: iptables.Filter, - Chain: IsolationChain1, + Chain: isolationChain1, Args: []string{"-i", bridgeIface, "!", "-d", prefix.String(), "-j", "DROP"}, } outDropRule = iptables.Rule{ IPVer: version, Table: iptables.Filter, - Chain: IsolationChain1, + Chain: isolationChain1, Args: []string{"-o", bridgeIface, "!", "-s", prefix.String(), "-j", "DROP"}, } } else { @@ -765,13 +573,13 @@ func setupInternalNetworkRules(bridgeIface string, prefix netip.Prefix, icc, ins inDropRule = iptables.Rule{ IPVer: version, Table: iptables.Filter, - Chain: IsolationChain1, + Chain: isolationChain1, Args: []string{"-i", bridgeIface, "!", "-o", bridgeIface, "!", "-d", prefix.String(), "-j", "DROP"}, } outDropRule = iptables.Rule{ IPVer: version, Table: iptables.Filter, - Chain: IsolationChain1, + Chain: isolationChain1, Args: []string{"!", "-i", bridgeIface, "-o", bridgeIface, "!", "-s", prefix.String(), "-j", "DROP"}, } } @@ -786,121 +594,3 @@ func setupInternalNetworkRules(bridgeIface string, prefix netip.Prefix, icc, ins // Set Inter Container Communication. return setIcc(version, bridgeIface, icc, true, insert) } - -// clearConntrackEntries flushes conntrack entries matching endpoint IP address -// or matching one of the exposed UDP port. -// In the first case, this could happen if packets were received by the host -// between userland proxy startup and iptables setup. -// In the latter case, this could happen if packets were received whereas there -// were nowhere to route them, as netfilter creates entries in such case. -// This is required because iptables NAT rules are evaluated by netfilter only -// when creating a new conntrack entry. When Docker latter adds NAT rules, -// netfilter ignore them for any packet matching a pre-existing conntrack entry. -// As such, we need to flush all those conntrack entries to make sure NAT rules -// are correctly applied to all packets. -// See: #8795, #44688 & #44742. -func clearConntrackEntries(nlh nlwrap.Handle, ep *bridgeEndpoint) { - var ipv4List []net.IP - var ipv6List []net.IP - var udpPorts []uint16 - - if ep.addr != nil { - ipv4List = append(ipv4List, ep.addr.IP) - } - if ep.addrv6 != nil { - ipv6List = append(ipv6List, ep.addrv6.IP) - } - for _, pb := range ep.portMapping { - if pb.Proto == types.UDP { - udpPorts = append(udpPorts, pb.HostPort) - } - } - - iptables.DeleteConntrackEntries(nlh, ipv4List, ipv6List) - iptables.DeleteConntrackEntriesByPort(nlh, types.UDP, udpPorts) -} - -// mirroredWSL2Workaround adds or removes an IPv4 NAT rule, depending on whether -// docker's host Linux appears to be a guest running under WSL2 in with mirrored -// mode networking. -// https://learn.microsoft.com/en-us/windows/wsl/networking#mirrored-mode-networking -// -// Without mirrored mode networking, or for a packet sent from Linux, packets -// sent to 127.0.0.1 are processed as outgoing - they hit the nat-OUTPUT chain, -// which does not jump to the nat-DOCKER chain because the rule has an exception -// for "-d 127.0.0.0/8". The default action on the nat-OUTPUT chain is ACCEPT (by -// default), so the packet is delivered to 127.0.0.1 on lo, where docker-proxy -// picks it up and acts as a man-in-the-middle; it receives the packet and -// re-sends it to the container (or acks a SYN and sets up a second TCP -// connection to the container). So, the container sees packets arrive with a -// source address belonging to the network's bridge, and it is able to reply to -// that address. -// -// In WSL2's mirrored networking mode, Linux has a loopback0 device as well as lo -// (which owns 127.0.0.1 as normal). Packets sent to 127.0.0.1 from Windows to a -// server listening on Linux's 127.0.0.1 are delivered via loopback0, and -// processed as packets arriving from outside the Linux host (which they are). -// -// So, these packets hit the nat-PREROUTING chain instead of nat-OUTPUT. It would -// normally be impossible for a packet ->127.0.0.1 to arrive from outside the -// host, so the nat-PREROUTING jump to nat-DOCKER has no exception for it. The -// packet is processed by a per-bridge DNAT rule in that chain, so it is -// delivered directly to the container (not via docker-proxy) with source address -// 127.0.0.1, so the container can't respond. -// -// DNAT is normally skipped by RETURN rules in the nat-DOCKER chain for packets -// arriving from any other bridge network. Similarly, this function adds (or -// removes) a rule to RETURN early for packets delivered via loopback0 with -// destination 127.0.0.0/8. -func mirroredWSL2Workaround(ipv iptables.IPVersion, hairpin bool) error { - // WSL2 does not (currently) support Windows<->Linux communication via ::1. - if ipv != iptables.IPv4 { - return nil - } - return programChainRule(mirroredWSL2Rule(), "WSL2 loopback", insertMirroredWSL2Rule(hairpin)) -} - -// insertMirroredWSL2Rule returns true if the NAT rule for mirrored WSL2 workaround -// is required. It is required if: -// - the userland proxy is running. If not, there's nothing on the host to catch -// the packet, so the loopback0 rule as wouldn't be useful. However, without -// the workaround, with improvements in WSL2 v2.3.11, and without userland proxy -// running - no workaround is needed, the normal DNAT/masquerading works. -// - and, the host Linux appears to be running under Windows WSL2 with mirrored -// mode networking. -func insertMirroredWSL2Rule(hairpin bool) bool { - if hairpin { - return false - } - return isRunningUnderWSL2MirroredMode() -} - -// isRunningUnderWSL2MirroredMode returns true if the host Linux appears to be -// running under Windows WSL2 with mirrored mode networking. If a loopback0 -// device exists, and there's an executable at /usr/bin/wslinfo, infer that -// this is WSL2 with mirrored networking. ("wslinfo --networking-mode" reports -// "mirrored", but applying the workaround for WSL2's loopback device when it's -// not needed is low risk, compared with executing wslinfo with dockerd's -// elevated permissions.) -func isRunningUnderWSL2MirroredMode() bool { - if _, err := nlwrap.LinkByName("loopback0"); err != nil { - if !errors.As(err, &netlink.LinkNotFoundError{}) { - log.G(context.TODO()).WithError(err).Warn("Failed to check for WSL interface") - } - return false - } - stat, err := os.Stat(wslinfoPath) - if err != nil { - return false - } - return stat.Mode().IsRegular() && (stat.Mode().Perm()&0o111) != 0 -} - -func mirroredWSL2Rule() iptables.Rule { - return iptables.Rule{ - IPVer: iptables.IPv4, - Table: iptables.Nat, - Chain: DockerChain, - Args: []string{"-i", "loopback0", "-d", "127.0.0.0/8", "-j", "RETURN"}, - } -} diff --git a/libnetwork/drivers/bridge/internal/iptabler/network_test.go b/libnetwork/drivers/bridge/internal/iptabler/network_test.go new file mode 100644 index 0000000000..f7b6cd5808 --- /dev/null +++ b/libnetwork/drivers/bridge/internal/iptabler/network_test.go @@ -0,0 +1,312 @@ +//go:build linux + +package iptabler + +import ( + "net/netip" + "testing" + + "github.com/docker/docker/internal/testutils/netnsutils" + "github.com/docker/docker/libnetwork/iptables" + "gotest.tools/v3/assert" + is "gotest.tools/v3/assert/cmp" +) + +const ( + defaultBridgeName = "testbridge" +) + +func TestProgramIPTable(t *testing.T) { + // Create a test bridge with a basic bridge configuration (name + IPv4). + defer netnsutils.SetupTestOSContext(t)() + + _, err := iptables.GetIptable(iptables.IPv4).NewChain(DockerForwardChain, iptables.Filter) + assert.NilError(t, err) + + // Store various iptables chain rules we care for. + const iptablesTestBridgeIP = "192.168.42.1" + rules := []struct { + rule iptables.Rule + descr string + }{ + {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-d", "127.1.2.3", "-i", "lo", "-o", "lo", "-j", "DROP"}}, "Test Loopback"}, + {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", iptablesTestBridgeIP, "!", "-o", defaultBridgeName, "-j", "MASQUERADE"}}, "NAT Test"}, + {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-o", defaultBridgeName, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}}, "Test ACCEPT INCOMING"}, + {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-i", defaultBridgeName, "!", "-o", defaultBridgeName, "-j", "ACCEPT"}}, "Test ACCEPT NON_ICC OUTGOING"}, + {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-i", defaultBridgeName, "-o", defaultBridgeName, "-j", "ACCEPT"}}, "Test enable ICC"}, + {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-i", defaultBridgeName, "-o", defaultBridgeName, "-j", "DROP"}}, "Test disable ICC"}, + } + + // Assert the chain rules' insertion and removal. + for _, c := range rules { + // Add + if err := programChainRule(c.rule, c.descr, true); err != nil { + t.Fatalf("Failed to program iptable rule %s: %s", c.descr, err.Error()) + } + + if !c.rule.Exists() { + t.Fatalf("Failed to effectively program iptable rule: %s", c.descr) + } + + // Remove + if err := programChainRule(c.rule, c.descr, false); err != nil { + t.Fatalf("Failed to remove iptable rule %s: %s", c.descr, err.Error()) + } + if c.rule.Exists() { + t.Fatalf("Failed to effectively remove iptable rule: %s", c.descr) + } + } +} + +func TestSetupIPChains(t *testing.T) { + // Create a test bridge with a basic bridge configuration (name + IPv4). + defer netnsutils.SetupTestOSContext(t)() + + ipt, err := NewIptabler(FirewallConfig{IPv4: true}) + assert.NilError(t, err) + + nc := NetworkConfig{ + IfName: defaultBridgeName, + Config4: NetworkConfigFam{ + Prefix: netip.MustParsePrefix("192.168.42.0/24"), + }, + } + + assertBridgeConfig(t, ipt, nc) + + nc.Masquerade = true + assertBridgeConfig(t, ipt, nc) + + nc.ICC = true + assertBridgeConfig(t, ipt, nc) + + nc.Masquerade = false + assertBridgeConfig(t, ipt, nc) +} + +// Regression test for https://github.com/moby/moby/issues/46445 +func TestSetupIP6TablesWithHostIPv4(t *testing.T) { + defer netnsutils.SetupTestOSContext(t)() + + ipt, err := NewIptabler(FirewallConfig{ + IPv4: true, + IPv6: true, + }) + assert.NilError(t, err) + + nc := NetworkConfig{ + IfName: defaultBridgeName, + Masquerade: true, + Config4: NetworkConfigFam{ + HostIP: netip.MustParseAddr("192.0.2.2"), + Prefix: netip.MustParsePrefix("192.168.42.0/24"), + }, + Config6: NetworkConfigFam{ + Prefix: netip.MustParsePrefix("2001:db8::/64"), + }, + } + assertBridgeConfig(t, ipt, nc) +} + +// Assert function which pushes chains based on bridge config parameters. +func assertBridgeConfig(t *testing.T, ipt *Iptabler, nc NetworkConfig) { + t.Helper() + n, err := ipt.NewNetwork(nc) + assert.NilError(t, err) + err = n.DelNetworkLevelRules() + assert.NilError(t, err) +} + +func TestOutgoingNATRules(t *testing.T) { + const br = "br-nattest" + maskedBrIPv4 := netip.MustParsePrefix("192.168.42.1/16").Masked() + maskedBrIPv6 := netip.MustParsePrefix("2001:db8::1/64").Masked() + hostIPv4 := netip.MustParseAddr("192.0.2.2") + hostIPv6 := netip.MustParseAddr("2001:db8:1::1") + for _, tc := range []struct { + desc string + enableIPTables bool + enableIP6Tables bool + enableIPv4 bool + enableIPv6 bool + enableIPMasquerade bool + hostIPv4 netip.Addr + hostIPv6 netip.Addr + // Hairpin NAT rules are not tested here because they are orthogonal to outgoing NAT. They + // exist to support the port forwarding DNAT rules: without any port forwarding there would be + // no need for any hairpin NAT rules, and when there is port forwarding then hairpin NAT rules + // are needed even if outgoing NAT is disabled. Hairpin NAT tests belong with the port + // forwarding DNAT tests. + wantIPv4Masq bool + wantIPv4Snat bool + wantIPv6Masq bool + wantIPv6Snat bool + }{ + { + desc: "everything disabled except ipv4", + enableIPv4: true, // one of IPv4 or IPv6 must be enabled + }, + { + desc: "everything disabled except ipv6", + enableIPv6: true, + }, + { + desc: "iptables and ip6tables disabled", + enableIPv4: true, + enableIPv6: true, + enableIPMasquerade: true, + }, + { + desc: "host IP with iptables and ip6tables disabled", + enableIPv4: true, + enableIPv6: true, + enableIPMasquerade: true, + hostIPv4: hostIPv4, + hostIPv6: hostIPv6, + }, + { + desc: "masquerade disabled, no host IP", + enableIPTables: true, + enableIP6Tables: true, + enableIPv4: true, + enableIPv6: true, + }, + { + desc: "masquerade disabled, with host IP", + enableIPTables: true, + enableIP6Tables: true, + enableIPv4: true, + enableIPv6: true, + hostIPv4: hostIPv4, + hostIPv6: hostIPv6, + }, + { + desc: "IPv4 masquerade, IPv6 disabled", + enableIPv4: true, + enableIPTables: true, + enableIPMasquerade: true, + wantIPv4Masq: true, + }, + { + desc: "IPv6 masquerade, IPv4 disabled", + enableIPv6: true, + enableIP6Tables: true, + enableIPMasquerade: true, + wantIPv6Masq: true, + }, + { + desc: "IPv4 SNAT, IPv6 disabled", + enableIPv4: true, + enableIPTables: true, + enableIPMasquerade: true, + hostIPv4: hostIPv4, + wantIPv4Snat: true, + }, + { + desc: "IPv6 SNAT, IPv4 disabled", + enableIPv6: true, + enableIP6Tables: true, + enableIPMasquerade: true, + hostIPv6: hostIPv6, + wantIPv6Snat: true, + }, + { + desc: "IPv4 masquerade, IPv6 masquerade", + enableIPTables: true, + enableIP6Tables: true, + enableIPv4: true, + enableIPv6: true, + enableIPMasquerade: true, + wantIPv4Masq: true, + wantIPv6Masq: true, + }, + { + desc: "IPv4 masquerade, IPv6 SNAT", + enableIPTables: true, + enableIP6Tables: true, + enableIPv4: true, + enableIPv6: true, + enableIPMasquerade: true, + hostIPv6: hostIPv6, + wantIPv4Masq: true, + wantIPv6Snat: true, + }, + { + desc: "IPv4 SNAT, IPv6 masquerade", + enableIPTables: true, + enableIP6Tables: true, + enableIPv4: true, + enableIPv6: true, + enableIPMasquerade: true, + hostIPv4: hostIPv4, + wantIPv4Snat: true, + wantIPv6Masq: true, + }, + { + desc: "IPv4 SNAT, IPv6 SNAT", + enableIPTables: true, + enableIP6Tables: true, + enableIPv4: true, + enableIPv6: true, + enableIPMasquerade: true, + hostIPv4: hostIPv4, + hostIPv6: hostIPv6, + wantIPv4Snat: true, + wantIPv6Snat: true, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + defer netnsutils.SetupTestOSContext(t)() + ipt, err := NewIptabler(FirewallConfig{ + IPv4: tc.enableIPTables, + IPv6: tc.enableIP6Tables, + }) + assert.NilError(t, err) + + nc := NetworkConfig{ + IfName: br, + Masquerade: tc.enableIPMasquerade, + Config4: NetworkConfigFam{ + HostIP: tc.hostIPv4, + Prefix: maskedBrIPv4, + }, + Config6: NetworkConfigFam{ + HostIP: tc.hostIPv6, + Prefix: maskedBrIPv6, + }, + } + n, err := ipt.NewNetwork(nc) + assert.NilError(t, err) + + defer func() { + err = n.DelNetworkLevelRules() + assert.NilError(t, err) + }() + + // Log the contents of all chains to aid troubleshooting. + for _, ipv := range []iptables.IPVersion{iptables.IPv4, iptables.IPv6} { + ipt := iptables.GetIptable(ipv) + for _, table := range []iptables.Table{iptables.Nat, iptables.Filter, iptables.Mangle} { + out, err := ipt.Raw("-t", string(table), "-S") + if err != nil { + t.Error(err) + } + t.Logf("%s: %s %s table rules:\n%s", tc.desc, ipv, table, string(out)) + } + } + for i, rc := range []struct { + want bool + rule iptables.Rule + }{ + // Rule order doesn't matter: At most one of the following IPv4 rules will exist, and the + // same goes for the IPv6 rules. + {tc.wantIPv4Masq, iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", maskedBrIPv4.String(), "!", "-o", br, "-j", "MASQUERADE"}}}, + {tc.wantIPv4Snat, iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", maskedBrIPv4.String(), "!", "-o", br, "-j", "SNAT", "--to-source", hostIPv4.String()}}}, + {tc.wantIPv6Masq, iptables.Rule{IPVer: iptables.IPv6, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", maskedBrIPv6.String(), "!", "-o", br, "-j", "MASQUERADE"}}}, + {tc.wantIPv6Snat, iptables.Rule{IPVer: iptables.IPv6, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", maskedBrIPv6.String(), "!", "-o", br, "-j", "SNAT", "--to-source", hostIPv6.String()}}}, + } { + assert.Check(t, is.Equal(rc.rule.Exists(), rc.want), "rule:%d", i) + } + }) + } +} diff --git a/libnetwork/drivers/bridge/internal/iptabler/port.go b/libnetwork/drivers/bridge/internal/iptabler/port.go new file mode 100644 index 0000000000..1cad6c62fa --- /dev/null +++ b/libnetwork/drivers/bridge/internal/iptabler/port.go @@ -0,0 +1,248 @@ +//go:build linux + +package iptabler + +import ( + "context" + "net" + "os" + "strconv" + + "github.com/containerd/log" + "github.com/docker/docker/libnetwork/iptables" + "github.com/docker/docker/libnetwork/types" +) + +func (n *Network) AddPorts(ctx context.Context, pbs []types.PortBinding) error { + return n.modPorts(ctx, pbs, true) +} + +func (n *Network) DelPorts(ctx context.Context, pbs []types.PortBinding) error { + return n.modPorts(ctx, pbs, false) +} + +func (n *Network) modPorts(ctx context.Context, pbs []types.PortBinding, enable bool) error { + for _, pb := range pbs { + if err := n.setPerPortIptables(ctx, pb, enable); err != nil { + return err + } + } + return nil +} + +// setPerPortIptables configures rules required by port binding b. Rules are added if +// enable is true, else removed. +func (n *Network) setPerPortIptables(ctx context.Context, b types.PortBinding, enable bool) error { + v := iptables.IPv4 + enabled := n.ipt.IPv4 + config := n.Config4 + if b.IP.To4() == nil { + v = iptables.IPv6 + enabled = n.ipt.IPv6 + config = n.Config6 + } + + if !enabled || n.Internal { + // Nothing to do. + return nil + } + + if err := filterPortMappedOnLoopback(ctx, b, b.HostIP, enable); err != nil { + return err + } + + if err := n.filterDirectAccess(ctx, b, enable); err != nil { + return err + } + + if (b.IP.To4() != nil) != (b.HostIP.To4() != nil) { + // The binding is between containerV4 and hostV6 (not vice versa as that + // will have been rejected earlier). It's handled by docker-proxy. So, no + // further iptables rules are required. + return nil + } + + if err := n.setPerPortNAT(v, b, enable); err != nil { + return err + } + + if !config.Unprotected { + if err := setPerPortForwarding(b, v, n.IfName, enable); err != nil { + return err + } + } + return nil +} + +// setPerPortNAT configures DNAT and MASQUERADE rules for port binding b. Rules are added if +// enable is true, else removed. +func (n *Network) setPerPortNAT(ipv iptables.IPVersion, b types.PortBinding, enable bool) error { + if b.HostPort == 0 { + // NAT is disabled. + return nil + } + // iptables interprets "0.0.0.0" as "0.0.0.0/32", whereas we + // want "0.0.0.0/0". "0/0" is correctly interpreted as "any + // value" by both iptables and ip6tables. + hostIP := "0/0" + if !b.HostIP.IsUnspecified() { + hostIP = b.HostIP.String() + } + args := []string{ + "-p", b.Proto.String(), + "-d", hostIP, + "--dport", strconv.Itoa(int(b.HostPort)), + "-j", "DNAT", + "--to-destination", net.JoinHostPort(b.IP.String(), strconv.Itoa(int(b.Port))), + } + if !n.ipt.Hairpin { + args = append(args, "!", "-i", n.IfName) + } + if ipv == iptables.IPv6 { + args = append(args, "!", "-s", "fe80::/10") + } + rule := iptables.Rule{IPVer: ipv, Table: iptables.Nat, Chain: dockerChain, Args: args} + if err := appendOrDelChainRule(rule, "DNAT", enable); err != nil { + return err + } + + rule = iptables.Rule{IPVer: ipv, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{ + "-p", b.Proto.String(), + "-s", b.IP.String(), + "-d", b.IP.String(), + "--dport", strconv.Itoa(int(b.Port)), + "-j", "MASQUERADE", + }} + if err := appendOrDelChainRule(rule, "MASQUERADE", n.ipt.Hairpin && enable); err != nil { + return err + } + + return nil +} + +// setPerPortForwarding opens access to a container's published port, as described by binding b. +// It also does something weird, broken, and disabled-by-default related to SCTP. Rules are added +// if enable is true, else removed. +func setPerPortForwarding(b types.PortBinding, ipv iptables.IPVersion, bridgeName string, enable bool) error { + // Insert rules for open ports at the top of the filter table's DOCKER + // chain (a per-network DROP rule, which must come after these per-port + // per-container ACCEPT rules, is appended to the chain when the network + // is created). + rule := iptables.Rule{IPVer: ipv, Table: iptables.Filter, Chain: dockerChain, Args: []string{ + "!", "-i", bridgeName, + "-o", bridgeName, + "-p", b.Proto.String(), + "-d", b.IP.String(), + "--dport", strconv.Itoa(int(b.Port)), + "-j", "ACCEPT", + }} + if err := programChainRule(rule, "OPEN PORT", enable); err != nil { + return err + } + + // TODO(robmry) - remove, see https://github.com/moby/moby/pull/48149 + if b.Proto == types.SCTP && os.Getenv("DOCKER_IPTABLES_SCTP_CHECKSUM") == "1" { + // Linux kernel v4.9 and below enables NETIF_F_SCTP_CRC for veth by + // the following commit. + // This introduces a problem when combined with a physical NIC without + // NETIF_F_SCTP_CRC. As for a workaround, here we add an iptables entry + // to fill the checksum. + // + // https://github.com/torvalds/linux/commit/c80fafbbb59ef9924962f83aac85531039395b18 + rule := iptables.Rule{IPVer: ipv, Table: iptables.Mangle, Chain: "POSTROUTING", Args: []string{ + "-p", b.Proto.String(), + "--sport", strconv.Itoa(int(b.Port)), + "-j", "CHECKSUM", + "--checksum-fill", + }} + if err := appendOrDelChainRule(rule, "SCTP CHECKSUM", enable); err != nil { + return err + } + } + + return nil +} + +// filterPortMappedOnLoopback adds an iptables rule that drops remote +// connections to ports mapped on loopback addresses. +// +// This is a no-op if the portBinding is for IPv6 (IPv6 loopback address is +// non-routable), or over a network with gw_mode=routed (PBs in routed mode +// don't map ports on the host). +func filterPortMappedOnLoopback(ctx context.Context, b types.PortBinding, hostIP net.IP, enable bool) error { + if rawRulesDisabled(ctx) { + return nil + } + if b.HostPort == 0 || !hostIP.IsLoopback() || hostIP.To4() == nil { + return nil + } + + acceptMirrored := iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Raw, Chain: "PREROUTING", Args: []string{ + "-p", b.Proto.String(), + "-d", hostIP.String(), + "--dport", strconv.Itoa(int(b.HostPort)), + "-i", "loopback0", + "-j", "ACCEPT", + }} + enableMirrored := enable && isRunningUnderWSL2MirroredMode() + if err := appendOrDelChainRule(acceptMirrored, "LOOPBACK FILTERING - ACCEPT MIRRORED", enableMirrored); err != nil { + return err + } + + drop := iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Raw, Chain: "PREROUTING", Args: []string{ + "-p", b.Proto.String(), + "-d", hostIP.String(), + "--dport", strconv.Itoa(int(b.HostPort)), + "!", "-i", "lo", + "-j", "DROP", + }} + if err := appendOrDelChainRule(drop, "LOOPBACK FILTERING - DROP", enable); err != nil { + return err + } + + return nil +} + +// filterDirectAccess adds an iptables rule that drops 'direct' remote +// connections made to the container's IP address, when the network gateway +// mode is "nat". +// +// This is a no-op if the gw_mode is "nat-unprotected" or "routed". +func (n *Network) filterDirectAccess(ctx context.Context, b types.PortBinding, enable bool) error { + if rawRulesDisabled(ctx) { + return nil + } + ipv := iptables.IPv4 + config := n.Config4 + if b.IP.To4() == nil { + ipv = iptables.IPv6 + config = n.Config6 + } + + // gw_mode=nat-unprotected means there's minimal security for NATed ports, + // so don't filter direct access. + if config.Unprotected || config.Routed { + return nil + } + + drop := iptables.Rule{IPVer: ipv, Table: iptables.Raw, Chain: "PREROUTING", Args: []string{ + "-p", b.Proto.String(), + "-d", b.IP.String(), // Container IP address + "--dport", strconv.Itoa(int(b.Port)), // Container port + "!", "-i", n.IfName, + "-j", "DROP", + }} + if err := appendOrDelChainRule(drop, "DIRECT ACCESS FILTERING - DROP", enable); err != nil { + return err + } + + return nil +} + +func rawRulesDisabled(ctx context.Context) bool { + if os.Getenv("DOCKER_INSECURE_NO_IPTABLES_RAW") == "1" { + log.G(ctx).Debug("DOCKER_INSECURE_NO_IPTABLES_RAW=1 - skipping raw rules") + return true + } + return false +} diff --git a/libnetwork/drivers/bridge/internal/iptabler/wsl2.go b/libnetwork/drivers/bridge/internal/iptabler/wsl2.go new file mode 100644 index 0000000000..601203bf31 --- /dev/null +++ b/libnetwork/drivers/bridge/internal/iptabler/wsl2.go @@ -0,0 +1,104 @@ +//go:build linux + +package iptabler + +import ( + "context" + "errors" + "os" + + "github.com/containerd/log" + "github.com/docker/docker/internal/nlwrap" + "github.com/docker/docker/libnetwork/iptables" + "github.com/vishvananda/netlink" +) + +// Path to the executable installed in Linux under WSL2 that reports on +// WSL config. https://github.com/microsoft/WSL/releases/tag/2.0.4 +// Can be modified by tests. +var wslinfoPath = "/usr/bin/wslinfo" + +// mirroredWSL2Workaround adds or removes an IPv4 NAT rule, depending on whether +// docker's host Linux appears to be a guest running under WSL2 in with mirrored +// mode networking. +// https://learn.microsoft.com/en-us/windows/wsl/networking#mirrored-mode-networking +// +// Without mirrored mode networking, or for a packet sent from Linux, packets +// sent to 127.0.0.1 are processed as outgoing - they hit the nat-OUTPUT chain, +// which does not jump to the nat-DOCKER chain because the rule has an exception +// for "-d 127.0.0.0/8". The default action on the nat-OUTPUT chain is ACCEPT (by +// default), so the packet is delivered to 127.0.0.1 on lo, where docker-proxy +// picks it up and acts as a man-in-the-middle; it receives the packet and +// re-sends it to the container (or acks a SYN and sets up a second TCP +// connection to the container). So, the container sees packets arrive with a +// source address belonging to the network's bridge, and it is able to reply to +// that address. +// +// In WSL2's mirrored networking mode, Linux has a loopback0 device as well as lo +// (which owns 127.0.0.1 as normal). Packets sent to 127.0.0.1 from Windows to a +// server listening on Linux's 127.0.0.1 are delivered via loopback0, and +// processed as packets arriving from outside the Linux host (which they are). +// +// So, these packets hit the nat-PREROUTING chain instead of nat-OUTPUT. It would +// normally be impossible for a packet ->127.0.0.1 to arrive from outside the +// host, so the nat-PREROUTING jump to nat-DOCKER has no exception for it. The +// packet is processed by a per-bridge DNAT rule in that chain, so it is +// delivered directly to the container (not via docker-proxy) with source address +// 127.0.0.1, so the container can't respond. +// +// DNAT is normally skipped by RETURN rules in the nat-DOCKER chain for packets +// arriving from any other bridge network. Similarly, this function adds (or +// removes) a rule to RETURN early for packets delivered via loopback0 with +// destination 127.0.0.0/8. +func mirroredWSL2Workaround(ipv iptables.IPVersion, hairpin bool) error { + // WSL2 does not (currently) support Windows<->Linux communication via ::1. + if ipv != iptables.IPv4 { + return nil + } + return programChainRule(mirroredWSL2Rule(), "WSL2 loopback", shouldInsertMirroredWSL2Rule(hairpin)) +} + +// shouldInsertMirroredWSL2Rule returns true if the NAT rule for mirrored WSL2 workaround +// is required. It is required if: +// - the userland proxy is running. If not, there's nothing on the host to catch +// the packet, so the loopback0 rule as wouldn't be useful. However, without +// the workaround, with improvements in WSL2 v2.3.11, and without userland proxy +// running - no workaround is needed, the normal DNAT/masquerading works. +// - and, the host Linux appears to be running under Windows WSL2 with mirrored +// mode networking. +func shouldInsertMirroredWSL2Rule(hairpin bool) bool { + if hairpin { + return false + } + return isRunningUnderWSL2MirroredMode() +} + +// isRunningUnderWSL2MirroredMode returns true if the host Linux appears to be +// running under Windows WSL2 with mirrored mode networking. If a loopback0 +// device exists, and there's an executable at /usr/bin/wslinfo, infer that +// this is WSL2 with mirrored networking. ("wslinfo --networking-mode" reports +// "mirrored", but applying the workaround for WSL2's loopback device when it's +// not needed is low risk, compared with executing wslinfo with dockerd's +// elevated permissions.) +func isRunningUnderWSL2MirroredMode() bool { + if _, err := nlwrap.LinkByName("loopback0"); err != nil { + if !errors.As(err, &netlink.LinkNotFoundError{}) { + log.G(context.TODO()).WithError(err).Warn("Failed to check for WSL interface") + } + return false + } + stat, err := os.Stat(wslinfoPath) + if err != nil { + return false + } + return stat.Mode().IsRegular() && (stat.Mode().Perm()&0o111) != 0 +} + +func mirroredWSL2Rule() iptables.Rule { + return iptables.Rule{ + IPVer: iptables.IPv4, + Table: iptables.Nat, + Chain: dockerChain, + Args: []string{"-i", "loopback0", "-d", "127.0.0.0/8", "-j", "RETURN"}, + } +} diff --git a/libnetwork/drivers/bridge/internal/iptabler/wsl2_test.go b/libnetwork/drivers/bridge/internal/iptabler/wsl2_test.go new file mode 100644 index 0000000000..5c0d20f029 --- /dev/null +++ b/libnetwork/drivers/bridge/internal/iptabler/wsl2_test.go @@ -0,0 +1,160 @@ +//go:build linux + +package iptabler + +import ( + "context" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/docker/docker/libnetwork/types" + "github.com/vishvananda/netlink" + + "github.com/docker/docker/internal/testutils/netnsutils" + "github.com/docker/docker/libnetwork/iptables" + "gotest.tools/v3/assert" + is "gotest.tools/v3/assert/cmp" +) + +func TestMirroredWSL2Workaround(t *testing.T) { + for _, tc := range []struct { + desc string + loopback0 bool + userlandProxy bool + wslinfoPerm os.FileMode // 0 for no-file + expLoopback0Rule bool + }{ + { + desc: "No loopback0", + }, + { + desc: "WSL2 mirrored", + loopback0: true, + userlandProxy: true, + wslinfoPerm: 0o777, + expLoopback0Rule: true, + }, + { + desc: "loopback0 but wslinfo not executable", + loopback0: true, + userlandProxy: true, + wslinfoPerm: 0o666, + }, + { + desc: "loopback0 but no wslinfo", + loopback0: true, + userlandProxy: true, + }, + { + desc: "loopback0 but no userland proxy", + loopback0: true, + wslinfoPerm: 0o777, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + defer netnsutils.SetupTestOSContext(t)() + restoreWslinfoPath := simulateWSL2MirroredMode(t, tc.loopback0, tc.wslinfoPerm) + defer restoreWslinfoPath() + + _, err := NewIptabler(FirewallConfig{ + IPv4: true, + Hairpin: !tc.userlandProxy, + }) + assert.NilError(t, err) + assert.Check(t, is.Equal(mirroredWSL2Rule().Exists(), tc.expLoopback0Rule)) + }) + } +} + +// simulateWSL2MirroredMode simulates the WSL2 mirrored mode by creating a +// loopback0 interface and optionally creating a wslinfo file with the given +// permissions. +// A clean up function is returned and will restore the original wslinfoPath +// used within the 'bridge' package. The loopback0 interface isn't cleaned up. +// Instead this function should be called from a disposable network namespace. +func simulateWSL2MirroredMode(t *testing.T, loopback0 bool, wslinfoPerm os.FileMode) func() { + if loopback0 { + iface := &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: "loopback0", + }, + } + err := netlink.LinkAdd(iface) + assert.NilError(t, err) + } + + wslinfoPathOrig := wslinfoPath + if wslinfoPerm != 0 { + tmpdir := t.TempDir() + p := filepath.Join(tmpdir, "wslinfo") + err := os.WriteFile(p, []byte("#!/bin/sh\necho dummy file\n"), wslinfoPerm) + assert.NilError(t, err) + wslinfoPath = p + } + + return func() { + wslinfoPath = wslinfoPathOrig + } +} + +func TestMirroredWSL2LoopbackFiltering(t *testing.T) { + for _, tc := range []struct { + desc string + loopback0 bool + wslinfoPerm os.FileMode // 0 for no-file + expLoopback0Rule bool + }{ + { + desc: "No loopback0", + }, + { + desc: "WSL2 mirrored", + loopback0: true, + wslinfoPerm: 0o777, + expLoopback0Rule: true, + }, + { + desc: "loopback0 but wslinfo not executable", + loopback0: true, + wslinfoPerm: 0o666, + }, + { + desc: "loopback0 but no wslinfo", + loopback0: true, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + defer netnsutils.SetupTestOSContext(t)() + restoreWslinfoPath := simulateWSL2MirroredMode(t, tc.loopback0, tc.wslinfoPerm) + defer restoreWslinfoPath() + + hostIP := net.ParseIP("127.0.0.1") + err := filterPortMappedOnLoopback(context.Background(), types.PortBinding{ + Proto: types.TCP, + IP: hostIP, + HostPort: 8000, + }, hostIP, true) + assert.NilError(t, err) + + out, err := exec.Command("iptables-save", "-t", "raw").CombinedOutput() + assert.NilError(t, err) + + // Checking this after trying to create rules, to make sure the init code in iptables/firewalld.go has run. + if fw, _ := iptables.UsingFirewalld(); fw { + t.Skip("firewalld is running in the host netns, it can't modify rules in the test's netns") + } + + if tc.expLoopback0Rule { + assert.Check(t, is.Equal(strings.Count(string(out), "-A PREROUTING"), 2)) + assert.Check(t, is.Contains(string(out), "-A PREROUTING -d 127.0.0.1/32 -i loopback0 -p tcp -m tcp --dport 8000 -j ACCEPT")) + } else { + assert.Check(t, is.Equal(strings.Count(string(out), "-A PREROUTING"), 1)) + assert.Check(t, !strings.Contains(string(out), "loopback0"), "There should be no rule in the raw-PREROUTING chain") + } + }) + } +} diff --git a/libnetwork/drivers/bridge/port_mapping_linux.go b/libnetwork/drivers/bridge/port_mapping_linux.go index 667353466d..386aa5c6d9 100644 --- a/libnetwork/drivers/bridge/port_mapping_linux.go +++ b/libnetwork/drivers/bridge/port_mapping_linux.go @@ -16,7 +16,6 @@ import ( "github.com/containerd/log" "github.com/docker/docker/libnetwork/drivers/bridge/internal/rlkclient" - "github.com/docker/docker/libnetwork/iptables" "github.com/docker/docker/libnetwork/netutils" "github.com/docker/docker/libnetwork/portallocator" "github.com/docker/docker/libnetwork/portmapper" @@ -773,232 +772,6 @@ func (n *bridgeNetwork) releasePortBindings(pbs []portBinding) error { return errors.Join(errs...) } -func (n *iptablesNetwork) AddPorts(ctx context.Context, pbs []types.PortBinding) error { - return n.modPorts(ctx, pbs, true) -} - -func (n *iptablesNetwork) DelPorts(ctx context.Context, pbs []types.PortBinding) error { - return n.modPorts(ctx, pbs, false) -} - -func (n *iptablesNetwork) modPorts(ctx context.Context, pbs []types.PortBinding, enable bool) error { - for _, pb := range pbs { - if err := n.setPerPortIptables(ctx, pb, enable); err != nil { - return err - } - } - return nil -} - -func (n *iptablesNetwork) setPerPortIptables(ctx context.Context, b types.PortBinding, enable bool) error { - v := iptables.IPv4 - enabled := n.fw.IPv4 - config := n.Config4 - if b.IP.To4() == nil { - v = iptables.IPv6 - enabled = n.fw.IPv6 - config = n.Config6 - } - - if !enabled { - // Nothing to do, iptables/ip6tables is not enabled. - return nil - } - - if err := filterPortMappedOnLoopback(ctx, b, b.HostIP, enable); err != nil { - return err - } - - if err := n.filterDirectAccess(ctx, b, enable); err != nil { - return err - } - - if (b.IP.To4() != nil) != (b.HostIP.To4() != nil) { - // The binding is between containerV4 and hostV6 (not vice versa as that - // will have been rejected earlier). It's handled by docker-proxy. So, no - // further iptables rules are required. - return nil - } - - if err := n.setPerPortNAT(v, b, enable); err != nil { - return err - } - - if !config.Unprotected { - if err := setPerPortForwarding(b, v, n.IfName, enable); err != nil { - return err - } - } - return nil -} - -func (n *iptablesNetwork) setPerPortNAT(ipv iptables.IPVersion, b types.PortBinding, enable bool) error { - if b.HostPort == 0 { - // NAT is disabled. - return nil - } - // iptables interprets "0.0.0.0" as "0.0.0.0/32", whereas we - // want "0.0.0.0/0". "0/0" is correctly interpreted as "any - // value" by both iptables and ip6tables. - hostIP := "0/0" - if !b.HostIP.IsUnspecified() { - hostIP = b.HostIP.String() - } - args := []string{ - "-p", b.Proto.String(), - "-d", hostIP, - "--dport", strconv.Itoa(int(b.HostPort)), - "-j", "DNAT", - "--to-destination", net.JoinHostPort(b.IP.String(), strconv.Itoa(int(b.Port))), - } - if !n.fw.Hairpin { - args = append(args, "!", "-i", n.IfName) - } - if ipv == iptables.IPv6 { - args = append(args, "!", "-s", "fe80::/10") - } - rule := iptables.Rule{IPVer: ipv, Table: iptables.Nat, Chain: DockerChain, Args: args} - if err := appendOrDelChainRule(rule, "DNAT", enable); err != nil { - return err - } - - rule = iptables.Rule{IPVer: ipv, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{ - "-p", b.Proto.String(), - "-s", b.IP.String(), - "-d", b.IP.String(), - "--dport", strconv.Itoa(int(b.Port)), - "-j", "MASQUERADE", - }} - if err := appendOrDelChainRule(rule, "MASQUERADE", n.fw.Hairpin && enable); err != nil { - return err - } - - return nil -} - -func setPerPortForwarding(b types.PortBinding, ipv iptables.IPVersion, bridgeName string, enable bool) error { - // Insert rules for open ports at the top of the filter table's DOCKER - // chain (a per-network DROP rule, which must come after these per-port - // per-container ACCEPT rules, is appended to the chain when the network - // is created). - rule := iptables.Rule{IPVer: ipv, Table: iptables.Filter, Chain: DockerChain, Args: []string{ - "!", "-i", bridgeName, - "-o", bridgeName, - "-p", b.Proto.String(), - "-d", b.IP.String(), - "--dport", strconv.Itoa(int(b.Port)), - "-j", "ACCEPT", - }} - if err := programChainRule(rule, "OPEN PORT", enable); err != nil { - return err - } - - if b.Proto == types.SCTP && os.Getenv("DOCKER_IPTABLES_SCTP_CHECKSUM") == "1" { - // Linux kernel v4.9 and below enables NETIF_F_SCTP_CRC for veth by - // the following commit. - // This introduces a problem when combined with a physical NIC without - // NETIF_F_SCTP_CRC. As for a workaround, here we add an iptables entry - // to fill the checksum. - // - // https://github.com/torvalds/linux/commit/c80fafbbb59ef9924962f83aac85531039395b18 - rule := iptables.Rule{IPVer: ipv, Table: iptables.Mangle, Chain: "POSTROUTING", Args: []string{ - "-p", b.Proto.String(), - "--sport", strconv.Itoa(int(b.Port)), - "-j", "CHECKSUM", - "--checksum-fill", - }} - if err := appendOrDelChainRule(rule, "SCTP CHECKSUM", enable); err != nil { - return err - } - } - - return nil -} - -// filterPortMappedOnLoopback adds an iptables rule that drops remote -// connections to ports mapped on loopback addresses. -// -// This is a no-op if the portBinding is for IPv6 (IPv6 loopback address is -// non-routable), or over a network with gw_mode=routed (PBs in routed mode -// don't map ports on the host). -func filterPortMappedOnLoopback(ctx context.Context, b types.PortBinding, hostIP net.IP, enable bool) error { - if rawRulesDisabled(ctx) { - return nil - } - if b.HostPort == 0 || !hostIP.IsLoopback() || hostIP.To4() == nil { - return nil - } - - acceptMirrored := iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Raw, Chain: "PREROUTING", Args: []string{ - "-p", b.Proto.String(), - "-d", hostIP.String(), - "--dport", strconv.Itoa(int(b.HostPort)), - "-i", "loopback0", - "-j", "ACCEPT", - }} - enableMirrored := enable && isRunningUnderWSL2MirroredMode() - if err := appendOrDelChainRule(acceptMirrored, "LOOPBACK FILTERING - ACCEPT MIRRORED", enableMirrored); err != nil { - return err - } - - drop := iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Raw, Chain: "PREROUTING", Args: []string{ - "-p", b.Proto.String(), - "-d", hostIP.String(), - "--dport", strconv.Itoa(int(b.HostPort)), - "!", "-i", "lo", - "-j", "DROP", - }} - if err := appendOrDelChainRule(drop, "LOOPBACK FILTERING - DROP", enable); err != nil { - return err - } - - return nil -} - -// filterDirectAccess adds an iptables rule that drops 'direct' remote -// connections made to the container's IP address, when the network gateway -// mode is "nat". -// -// This is a no-op if the gw_mode is "nat-unprotected" or "routed". -func (n *iptablesNetwork) filterDirectAccess(ctx context.Context, b types.PortBinding, enable bool) error { - if rawRulesDisabled(ctx) { - return nil - } - ipv := iptables.IPv4 - config := n.Config4 - if b.IP.To4() == nil { - ipv = iptables.IPv6 - config = n.Config6 - } - - // gw_mode=nat-unprotected means there's minimal security for NATed ports, - // so don't filter direct access. - if config.Unprotected || config.Routed { - return nil - } - - drop := iptables.Rule{IPVer: ipv, Table: iptables.Raw, Chain: "PREROUTING", Args: []string{ - "-p", b.Proto.String(), - "-d", b.IP.String(), // Container IP address - "--dport", strconv.Itoa(int(b.Port)), // Container port - "!", "-i", n.IfName, - "-j", "DROP", - }} - if err := appendOrDelChainRule(drop, "DIRECT ACCESS FILTERING - DROP", enable); err != nil { - return err - } - - return nil -} - -func rawRulesDisabled(ctx context.Context) bool { - if os.Getenv("DOCKER_INSECURE_NO_IPTABLES_RAW") == "1" { - log.G(ctx).Debug("DOCKER_INSECURE_NO_IPTABLES_RAW=1 - skipping raw rules") - return true - } - return false -} - func (n *bridgeNetwork) reapplyPerPortIptables() { n.Lock() var allPBs []portBinding diff --git a/libnetwork/drivers/bridge/setup_ip_tables_linux_test.go b/libnetwork/drivers/bridge/setup_ip_tables_linux_test.go deleted file mode 100644 index aa11b85a05..0000000000 --- a/libnetwork/drivers/bridge/setup_ip_tables_linux_test.go +++ /dev/null @@ -1,559 +0,0 @@ -package bridge - -import ( - "context" - "net" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - - "github.com/docker/docker/internal/nlwrap" - "github.com/docker/docker/internal/testutils/netnsutils" - "github.com/docker/docker/internal/testutils/storeutils" - "github.com/docker/docker/libnetwork/driverapi" - "github.com/docker/docker/libnetwork/iptables" - "github.com/docker/docker/libnetwork/netlabel" - "github.com/docker/docker/libnetwork/types" - "github.com/vishvananda/netlink" - "gotest.tools/v3/assert" - is "gotest.tools/v3/assert/cmp" -) - -const ( - iptablesTestBridgeIP = "192.168.42.1" -) - -// A testRegisterer implements the driverapi.Registerer interface. -type testRegisterer struct { - t *testing.T - d *driver -} - -func (r *testRegisterer) RegisterDriver(name string, di driverapi.Driver, _ driverapi.Capability) error { - if got, want := name, "bridge"; got != want { - r.t.Fatalf("got driver name %s, want %s", got, want) - } - d, ok := di.(*driver) - if !ok { - r.t.Fatalf("got driver type %T, want %T", di, &driver{}) - } - r.d = d - return nil -} - -func TestProgramIPTable(t *testing.T) { - // Create a test bridge with a basic bridge configuration (name + IPv4). - defer netnsutils.SetupTestOSContext(t)() - - nh, err := nlwrap.NewHandle() - if err != nil { - t.Fatal(err) - } - - createTestBridge(getBasicTestConfig(), &bridgeInterface{nlh: nh}, t) - _, err = iptables.GetIptable(iptables.IPv4).NewChain(DockerForwardChain, iptables.Filter) - assert.NilError(t, err) - - // Store various iptables chain rules we care for. - rules := []struct { - rule iptables.Rule - descr string - }{ - {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-d", "127.1.2.3", "-i", "lo", "-o", "lo", "-j", "DROP"}}, "Test Loopback"}, - {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", iptablesTestBridgeIP, "!", "-o", DefaultBridgeName, "-j", "MASQUERADE"}}, "NAT Test"}, - {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-o", DefaultBridgeName, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}}, "Test ACCEPT INCOMING"}, - {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-i", DefaultBridgeName, "!", "-o", DefaultBridgeName, "-j", "ACCEPT"}}, "Test ACCEPT NON_ICC OUTGOING"}, - {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-i", DefaultBridgeName, "-o", DefaultBridgeName, "-j", "ACCEPT"}}, "Test enable ICC"}, - {iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{"-i", DefaultBridgeName, "-o", DefaultBridgeName, "-j", "DROP"}}, "Test disable ICC"}, - } - - // Assert the chain rules' insertion and removal. - for _, c := range rules { - assertIPTableChainProgramming(c.rule, c.descr, t) - } -} - -func TestSetupIPChains(t *testing.T) { - // Create a test bridge with a basic bridge configuration (name + IPv4). - defer netnsutils.SetupTestOSContext(t)() - - nh, err := nlwrap.NewHandle() - if err != nil { - t.Fatal(err) - } - - driverconfig := configuration{ - EnableIPTables: true, - } - d := &driver{ - config: driverconfig, - } - assertChainConfig(d, t) - - config := getBasicTestConfig() - br := &bridgeInterface{nlh: nh} - createTestBridge(config, br, t) - - assertBridgeConfig(config, br, d, t) - - config.EnableIPMasquerade = true - assertBridgeConfig(config, br, d, t) - - config.EnableICC = true - assertBridgeConfig(config, br, d, t) - - config.EnableIPMasquerade = false - assertBridgeConfig(config, br, d, t) -} - -func getBasicTestConfig() *networkConfiguration { - config := &networkConfiguration{ - BridgeName: DefaultBridgeName, - EnableIPv4: true, - AddressIPv4: &net.IPNet{IP: net.ParseIP(iptablesTestBridgeIP), Mask: net.CIDRMask(16, 32)}, - } - return config -} - -func createTestBridge(config *networkConfiguration, br *bridgeInterface, t *testing.T) { - if err := setupDevice(config, br); err != nil { - t.Fatalf("Failed to create the testing Bridge: %s", err.Error()) - } - if err := setupBridgeIPv4(config, br); err != nil { - t.Fatalf("Failed to bring up the testing Bridge: %s", err.Error()) - } - if config.EnableIPv6 { - if err := setupBridgeIPv6(config, br); err != nil { - t.Fatalf("Failed to bring up the testing Bridge: %s", err.Error()) - } - } -} - -// Assert base function which pushes iptables chain rules on insertion and removal. -func assertIPTableChainProgramming(rule iptables.Rule, descr string, t *testing.T) { - t.Helper() - - // Add - if err := programChainRule(rule, descr, true); err != nil { - t.Fatalf("Failed to program iptable rule %s: %s", descr, err.Error()) - } - - if !rule.Exists() { - t.Fatalf("Failed to effectively program iptable rule: %s", descr) - } - - // Remove - if err := programChainRule(rule, descr, false); err != nil { - t.Fatalf("Failed to remove iptable rule %s: %s", descr, err.Error()) - } - if rule.Exists() { - t.Fatalf("Failed to effectively remove iptable rule: %s", descr) - } -} - -// Assert function which create chains. -func assertChainConfig(d *driver, t *testing.T) { - var err error - - err = setupIPChains(iptables.IPv4, !d.config.EnableUserlandProxy) - assert.NilError(t, err) - - if d.config.EnableIP6Tables { - err = setupIPChains(iptables.IPv6, !d.config.EnableUserlandProxy) - assert.NilError(t, err) - } -} - -// Assert function which pushes chains based on bridge config parameters. -func assertBridgeConfig(config *networkConfiguration, br *bridgeInterface, d *driver, t *testing.T) { - nw := bridgeNetwork{ - config: config, - driver: d, - bridge: br, - } - - fwn, err := nw.newIptablesNetwork() - assert.NilError(t, err) - assert.Check(t, fwn != nil, "no firewaller network") -} - -// Regression test for https://github.com/moby/moby/issues/46445 -func TestSetupIP6TablesWithHostIPv4(t *testing.T) { - defer netnsutils.SetupTestOSContext(t)() - d := newDriver(storeutils.NewTempStore(t)) - dc := &configuration{ - EnableIPTables: true, - EnableIP6Tables: true, - } - if err := d.configure(map[string]interface{}{netlabel.GenericData: dc}); err != nil { - t.Fatal(err) - } - nc := &networkConfiguration{ - BridgeName: DefaultBridgeName, - AddressIPv4: &net.IPNet{IP: net.ParseIP(iptablesTestBridgeIP), Mask: net.CIDRMask(16, 32)}, - EnableIPMasquerade: true, - EnableIPv4: true, - EnableIPv6: true, - AddressIPv6: &net.IPNet{IP: net.ParseIP("2001:db8::1"), Mask: net.CIDRMask(64, 128)}, - HostIPv4: net.ParseIP("192.0.2.2"), - } - nh, err := nlwrap.NewHandle() - if err != nil { - t.Fatal(err) - } - br := &bridgeInterface{nlh: nh} - createTestBridge(nc, br, t) - assertBridgeConfig(nc, br, d, t) -} - -func TestOutgoingNATRules(t *testing.T) { - br := "br-nattest" - brIPv4 := &net.IPNet{IP: net.ParseIP(iptablesTestBridgeIP), Mask: net.CIDRMask(16, 32)} - brIPv6 := &net.IPNet{IP: net.ParseIP("2001:db8::1"), Mask: net.CIDRMask(64, 128)} - maskedBrIPv4 := &net.IPNet{IP: brIPv4.IP.Mask(brIPv4.Mask), Mask: brIPv4.Mask} - maskedBrIPv6 := &net.IPNet{IP: brIPv6.IP.Mask(brIPv6.Mask), Mask: brIPv6.Mask} - hostIPv4 := net.ParseIP("192.0.2.2") - hostIPv6 := net.ParseIP("2001:db8:1::1") - for _, tc := range []struct { - desc string - enableIPTables bool - enableIP6Tables bool - enableIPv4 bool - enableIPv6 bool - enableIPMasquerade bool - hostIPv4 net.IP - hostIPv6 net.IP - // Hairpin NAT rules are not tested here because they are orthogonal to outgoing NAT. They - // exist to support the port forwarding DNAT rules: without any port forwarding there would be - // no need for any hairpin NAT rules, and when there is port forwarding then hairpin NAT rules - // are needed even if outgoing NAT is disabled. Hairpin NAT tests belong with the port - // forwarding DNAT tests. - wantIPv4Masq bool - wantIPv4Snat bool - wantIPv6Masq bool - wantIPv6Snat bool - }{ - { - desc: "everything disabled except ipv4", - enableIPv4: true, // one of IPv4 or IPv6 must be enabled - }, - { - desc: "everything disabled except ipv6", - enableIPv6: true, - }, - { - desc: "iptables/ip6tables disabled", - enableIPv4: true, - enableIPv6: true, - enableIPMasquerade: true, - }, - { - desc: "host IP with iptables/ip6tables disabled", - enableIPv4: true, - enableIPv6: true, - enableIPMasquerade: true, - hostIPv4: hostIPv4, - hostIPv6: hostIPv6, - }, - { - desc: "masquerade disabled, no host IP", - enableIPTables: true, - enableIP6Tables: true, - enableIPv4: true, - enableIPv6: true, - }, - { - desc: "masquerade disabled, with host IP", - enableIPTables: true, - enableIP6Tables: true, - enableIPv4: true, - enableIPv6: true, - hostIPv4: hostIPv4, - hostIPv6: hostIPv6, - }, - { - desc: "IPv4 masquerade, IPv6 disabled", - enableIPv4: true, - enableIPTables: true, - enableIPMasquerade: true, - wantIPv4Masq: true, - }, - { - desc: "IPv6 masquerade, IPv4 disabled", - enableIPv6: true, - enableIP6Tables: true, - enableIPMasquerade: true, - wantIPv6Masq: true, - }, - { - desc: "IPv4 SNAT, IPv6 disabled", - enableIPv4: true, - enableIPTables: true, - enableIPMasquerade: true, - hostIPv4: hostIPv4, - wantIPv4Snat: true, - }, - { - desc: "IPv6 SNAT, IPv4 disabled", - enableIPv6: true, - enableIP6Tables: true, - enableIPMasquerade: true, - hostIPv6: hostIPv6, - wantIPv6Snat: true, - }, - { - desc: "IPv4 masquerade, IPv6 masquerade", - enableIPTables: true, - enableIP6Tables: true, - enableIPv4: true, - enableIPv6: true, - enableIPMasquerade: true, - wantIPv4Masq: true, - wantIPv6Masq: true, - }, - { - desc: "IPv4 masquerade, IPv6 SNAT", - enableIPTables: true, - enableIP6Tables: true, - enableIPv4: true, - enableIPv6: true, - enableIPMasquerade: true, - hostIPv6: hostIPv6, - wantIPv4Masq: true, - wantIPv6Snat: true, - }, - { - desc: "IPv4 SNAT, IPv6 masquerade", - enableIPTables: true, - enableIP6Tables: true, - enableIPv4: true, - enableIPv6: true, - enableIPMasquerade: true, - hostIPv4: hostIPv4, - wantIPv4Snat: true, - wantIPv6Masq: true, - }, - { - desc: "IPv4 SNAT, IPv6 SNAT", - enableIPTables: true, - enableIP6Tables: true, - enableIPv4: true, - enableIPv6: true, - enableIPMasquerade: true, - hostIPv4: hostIPv4, - hostIPv6: hostIPv6, - wantIPv4Snat: true, - wantIPv6Snat: true, - }, - } { - t.Run(tc.desc, func(t *testing.T) { - defer netnsutils.SetupTestOSContext(t)() - dc := &configuration{ - EnableIPTables: tc.enableIPTables, - EnableIP6Tables: tc.enableIP6Tables, - } - r := &testRegisterer{t: t} - if err := Register(r, storeutils.NewTempStore(t), map[string]interface{}{netlabel.GenericData: dc}); err != nil { - t.Fatal(err) - } - if r.d == nil { - t.Fatal("testRegisterer.RegisterDriver never called") - } - nc := &networkConfiguration{ - BridgeName: br, - AddressIPv4: brIPv4, - AddressIPv6: brIPv6, - EnableIPv4: tc.enableIPv4, - EnableIPv6: tc.enableIPv6, - EnableIPMasquerade: tc.enableIPMasquerade, - HostIPv4: tc.hostIPv4, - HostIPv6: tc.hostIPv6, - } - ipv4Data := []driverapi.IPAMData{{Pool: maskedBrIPv4, Gateway: brIPv4}} - ipv6Data := []driverapi.IPAMData{{Pool: maskedBrIPv6, Gateway: brIPv6}} - if !nc.EnableIPv4 { - nc.AddressIPv4 = nil - ipv4Data = nil - } - if !nc.EnableIPv6 { - nc.AddressIPv6 = nil - ipv6Data = nil - } - if err := r.d.CreateNetwork(context.Background(), "nattest", map[string]interface{}{netlabel.GenericData: nc}, nil, ipv4Data, ipv6Data); err != nil { - t.Fatal(err) - } - defer func() { - if err := r.d.DeleteNetwork("nattest"); err != nil { - t.Fatal(err) - } - }() - // Log the contents of all chains to aid troubleshooting. - for _, ipv := range []iptables.IPVersion{iptables.IPv4, iptables.IPv6} { - ipt := iptables.GetIptable(ipv) - for _, table := range []iptables.Table{iptables.Nat, iptables.Filter, iptables.Mangle} { - out, err := ipt.Raw("-t", string(table), "-S") - if err != nil { - t.Error(err) - } - t.Logf("%s: %s %s table rules:\n%s", tc.desc, ipv, table, string(out)) - } - } - for _, rc := range []struct { - want bool - rule iptables.Rule - }{ - // Rule order doesn't matter: At most one of the following IPv4 rules will exist, and the - // same goes for the IPv6 rules. - {tc.wantIPv4Masq, iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", maskedBrIPv4.String(), "!", "-o", br, "-j", "MASQUERADE"}}}, - {tc.wantIPv4Snat, iptables.Rule{IPVer: iptables.IPv4, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", maskedBrIPv4.String(), "!", "-o", br, "-j", "SNAT", "--to-source", hostIPv4.String()}}}, - {tc.wantIPv6Masq, iptables.Rule{IPVer: iptables.IPv6, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", maskedBrIPv6.String(), "!", "-o", br, "-j", "MASQUERADE"}}}, - {tc.wantIPv6Snat, iptables.Rule{IPVer: iptables.IPv6, Table: iptables.Nat, Chain: "POSTROUTING", Args: []string{"-s", maskedBrIPv6.String(), "!", "-o", br, "-j", "SNAT", "--to-source", hostIPv6.String()}}}, - } { - assert.Equal(t, rc.rule.Exists(), rc.want) - } - }) - } -} - -func TestMirroredWSL2Workaround(t *testing.T) { - for _, tc := range []struct { - desc string - loopback0 bool - userlandProxy bool - wslinfoPerm os.FileMode // 0 for no-file - expLoopback0Rule bool - }{ - { - desc: "No loopback0", - }, - { - desc: "WSL2 mirrored", - loopback0: true, - userlandProxy: true, - wslinfoPerm: 0o777, - expLoopback0Rule: true, - }, - { - desc: "loopback0 but wslinfo not executable", - loopback0: true, - userlandProxy: true, - wslinfoPerm: 0o666, - }, - { - desc: "loopback0 but no wslinfo", - loopback0: true, - userlandProxy: true, - }, - { - desc: "loopback0 but no userland proxy", - loopback0: true, - wslinfoPerm: 0o777, - }, - } { - t.Run(tc.desc, func(t *testing.T) { - defer netnsutils.SetupTestOSContext(t)() - restoreWslinfoPath := simulateWSL2MirroredMode(t, tc.loopback0, tc.wslinfoPerm) - defer restoreWslinfoPath() - - config := configuration{EnableIPTables: true} - if tc.userlandProxy { - config.UserlandProxyPath = "some-proxy" - config.EnableUserlandProxy = true - } - err := setupIPChains(iptables.IPv4, !tc.userlandProxy) - assert.NilError(t, err) - assert.Check(t, is.Equal(mirroredWSL2Rule().Exists(), tc.expLoopback0Rule)) - }) - } -} - -// simulateWSL2MirroredMode simulates the WSL2 mirrored mode by creating a -// loopback0 interface and optionally creating a wslinfo file with the given -// permissions. -// A clean up function is returned and will restore the original wslinfoPath -// used within the 'bridge' package. The loopback0 interface isn't cleaned up. -// Instead this function should be called from a disposable network namespace. -func simulateWSL2MirroredMode(t *testing.T, loopback0 bool, wslinfoPerm os.FileMode) func() { - if loopback0 { - iface := &netlink.Dummy{ - LinkAttrs: netlink.LinkAttrs{ - Name: "loopback0", - }, - } - err := netlink.LinkAdd(iface) - assert.NilError(t, err) - } - - wslinfoPathOrig := wslinfoPath - if wslinfoPerm != 0 { - tmpdir := t.TempDir() - p := filepath.Join(tmpdir, "wslinfo") - err := os.WriteFile(p, []byte("#!/bin/sh\necho dummy file\n"), wslinfoPerm) - assert.NilError(t, err) - wslinfoPath = p - } - - return func() { - wslinfoPath = wslinfoPathOrig - } -} - -func TestMirroredWSL2LoopbackFiltering(t *testing.T) { - for _, tc := range []struct { - desc string - loopback0 bool - wslinfoPerm os.FileMode // 0 for no-file - expLoopback0Rule bool - }{ - { - desc: "No loopback0", - }, - { - desc: "WSL2 mirrored", - loopback0: true, - wslinfoPerm: 0o777, - expLoopback0Rule: true, - }, - { - desc: "loopback0 but wslinfo not executable", - loopback0: true, - wslinfoPerm: 0o666, - }, - { - desc: "loopback0 but no wslinfo", - loopback0: true, - }, - } { - t.Run(tc.desc, func(t *testing.T) { - defer netnsutils.SetupTestOSContext(t)() - restoreWslinfoPath := simulateWSL2MirroredMode(t, tc.loopback0, tc.wslinfoPerm) - defer restoreWslinfoPath() - - hostIP := net.ParseIP("127.0.0.1") - err := filterPortMappedOnLoopback(context.TODO(), types.PortBinding{ - Proto: types.TCP, - IP: hostIP, - HostPort: 8000, - }, hostIP, true) - assert.NilError(t, err) - - // Checking this after trying to create rules, to make sure the init code in iptables/firewalld.go has run. - if fw, _ := iptables.UsingFirewalld(); fw { - t.Skip("firewalld is running in the host netns, it can't modify rules in the test's netns") - } - - out, err := exec.Command("iptables-save", "-t", "raw").CombinedOutput() - assert.NilError(t, err) - - if tc.expLoopback0Rule { - assert.Check(t, is.Equal(strings.Count(string(out), "-A PREROUTING"), 2)) - assert.Check(t, is.Contains(string(out), "-A PREROUTING -d 127.0.0.1/32 -i loopback0 -p tcp -m tcp --dport 8000 -j ACCEPT")) - } else { - assert.Check(t, is.Equal(strings.Count(string(out), "-A PREROUTING"), 1)) - assert.Check(t, !strings.Contains(string(out), "loopback0"), "There should be no rule in the raw-PREROUTING chain") - } - }) - } -}