From a94643a1b3ed12e3c308667952b3acfb20a8b9b4 Mon Sep 17 00:00:00 2001 From: Rob Murray Date: Mon, 14 Apr 2025 17:35:30 +0100 Subject: [PATCH 1/3] bridge: add option com.docker.network.bridge.trusted_host_interfaces trusted_host_interface have access to published ports on container addresses - enabling direct routing to the container via those interfaces. Signed-off-by: Rob Murray --- .../networking/port_mapping_linux_test.go | 24 ++++++++-- libnetwork/drivers/bridge/bridge_linux.go | 47 ++++++++++--------- libnetwork/drivers/bridge/bridge_store.go | 6 +++ .../bridge/internal/iptabler/endpoint.go | 12 +++++ .../bridge/internal/iptabler/network.go | 13 ++--- libnetwork/drivers/bridge/labels.go | 4 ++ 6 files changed, 75 insertions(+), 31 deletions(-) diff --git a/integration/networking/port_mapping_linux_test.go b/integration/networking/port_mapping_linux_test.go index 78e6db417a..3025572ea3 100644 --- a/integration/networking/port_mapping_linux_test.go +++ b/integration/networking/port_mapping_linux_test.go @@ -928,7 +928,8 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) { netip.MustParsePrefix("fdbc:277b:d40b::1/64")) defer l3.Destroy(t) // "docker" is the host where dockerd is running. - l3.AddHost(t, "docker", networking.CurrentNetns, "test-eth", + const hostIfName = "test-eth" + l3.AddHost(t, "docker", networking.CurrentNetns, hostIfName, netip.MustParsePrefix(hostIPv4+"/24"), netip.MustParsePrefix(hostIPv6+"/64")) l3.AddHost(t, "attacker", "test-direct-remote-access-attacker", "eth0", @@ -947,6 +948,7 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) { gwAddr netip.Prefix ipv4Disabled bool ipv6Disabled bool + trusted bool }{ { name: "NAT/IPv4", @@ -958,6 +960,18 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) { gwMode: "nat", gwAddr: netip.MustParsePrefix("fda9:a651:db6d::1/64"), }, + { + name: "NAT/IPv4/trusted", + gwMode: "nat", + gwAddr: netip.MustParsePrefix("172.24.10.1/24"), + trusted: true, + }, + { + name: "NAT/IPv6/trusted", + gwMode: "nat", + gwAddr: netip.MustParsePrefix("fda9:a651:db6d::1/64"), + trusted: true, + }, { name: "NAT unprotected/IPv4", gwMode: "nat-unprotected", @@ -992,7 +1006,8 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - skip.If(t, tc.gwMode == "routed" && testEnv.IsRootless(), "rootlesskit doesn't support routed mode as it's running in a separate netns") + expDirectAccess := tc.gwMode == "routed" || tc.gwMode == "nat-unprotected" || tc.trusted + skip.If(t, expDirectAccess && testEnv.IsRootless(), "rootlesskit doesn't support routed mode as it's running in a separate netns") testutil.StartSpan(ctx, t) @@ -1011,6 +1026,9 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) { if tc.gwAddr.Addr().Is6() { nwOpts = append(nwOpts, network.WithIPv6()) } + if tc.trusted { + nwOpts = append(nwOpts, network.WithOption(bridge.TrustedHostInterfaces, hostIfName)) + } const bridgeName = "brattacked" network.CreateNoError(ctx, t, c, bridgeName, append(nwOpts, @@ -1054,8 +1072,6 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) { // Now send a payload directly to the container. With gw_mode=routed, // this should work. With gw_mode=nat, this should fail. - expDirectAccess := tc.gwMode == "routed" || (tc.gwMode == "nat-unprotected" && !testEnv.IsRootless()) - l3.Hosts["attacker"].Run(t, "ip", "route", "add", tc.gwAddr.Masked().String(), "via", hostIP, "dev", "eth0") defer l3.Hosts["attacker"].Run(t, "ip", "route", "delete", tc.gwAddr.Masked().String(), "via", hostIP, "dev", "eth0") diff --git a/libnetwork/drivers/bridge/bridge_linux.go b/libnetwork/drivers/bridge/bridge_linux.go index 4c37cb2455..c2a745c52f 100644 --- a/libnetwork/drivers/bridge/bridge_linux.go +++ b/libnetwork/drivers/bridge/bridge_linux.go @@ -7,6 +7,7 @@ import ( "net/netip" "os" "strconv" + "strings" "sync" "syscall" @@ -69,21 +70,22 @@ type configuration struct { // networkConfiguration for network specific configuration type networkConfiguration struct { - ID string - BridgeName string - EnableIPv4 bool - EnableIPv6 bool - EnableIPMasquerade bool - GwModeIPv4 gwMode - GwModeIPv6 gwMode - EnableICC bool - InhibitIPv4 bool - Mtu int - DefaultBindingIP net.IP - DefaultBridge bool - HostIPv4 net.IP - HostIPv6 net.IP - ContainerIfacePrefix string + ID string + BridgeName string + EnableIPv4 bool + EnableIPv6 bool + EnableIPMasquerade bool + GwModeIPv4 gwMode + GwModeIPv6 gwMode + EnableICC bool + TrustedHostInterfaces []string // Interface names must not contain ':' characters + InhibitIPv4 bool + Mtu int + DefaultBindingIP net.IP + DefaultBridge bool + HostIPv4 net.IP + HostIPv6 net.IP + ContainerIfacePrefix string // Internal fields set after ipam data parsing AddressIPv4 *net.IPNet AddressIPv6 *net.IPNet @@ -337,6 +339,8 @@ func (c *networkConfiguration) fromLabels(labels map[string]string) error { if c.EnableICC, err = strconv.ParseBool(value); err != nil { return parseErr(label, value, err.Error()) } + case TrustedHostInterfaces: + c.TrustedHostInterfaces = strings.FieldsFunc(value, func(r rune) bool { return r == ':' }) case InhibitIPv4: if c.InhibitIPv4, err = strconv.ParseBool(value); err != nil { return parseErr(label, value, err.Error()) @@ -405,12 +409,13 @@ func (n *bridgeNetwork) newIptablesNetwork() (*iptabler.Network, error) { return nil, err } return n.driver.firewaller.NewNetwork(iptabler.NetworkConfig{ - IfName: n.config.BridgeName, - Internal: n.config.Internal, - ICC: n.config.EnableICC, - Masquerade: n.config.EnableIPMasquerade, - Config4: config4, - Config6: config6, + IfName: n.config.BridgeName, + Internal: n.config.Internal, + ICC: n.config.EnableICC, + Masquerade: n.config.EnableIPMasquerade, + TrustedHostInterfaces: n.config.TrustedHostInterfaces, + Config4: config4, + Config6: config6, }) } diff --git a/libnetwork/drivers/bridge/bridge_store.go b/libnetwork/drivers/bridge/bridge_store.go index 78340d2250..d976488e35 100644 --- a/libnetwork/drivers/bridge/bridge_store.go +++ b/libnetwork/drivers/bridge/bridge_store.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "net" + "strings" "github.com/containerd/log" "github.com/docker/docker/internal/otelutil" @@ -137,6 +138,7 @@ func (ncfg *networkConfiguration) MarshalJSON() ([]byte, error) { nMap["GwModeIPv4"] = ncfg.GwModeIPv4 nMap["GwModeIPv6"] = ncfg.GwModeIPv6 nMap["EnableICC"] = ncfg.EnableICC + nMap["TrustedHostInterfaces"] = strings.Join(ncfg.TrustedHostInterfaces, ":") nMap["InhibitIPv4"] = ncfg.InhibitIPv4 nMap["Mtu"] = ncfg.Mtu nMap["Internal"] = ncfg.Internal @@ -215,6 +217,10 @@ func (ncfg *networkConfiguration) UnmarshalJSON(b []byte) error { ncfg.GwModeIPv6, _ = newGwMode(v.(string)) } ncfg.EnableICC = nMap["EnableICC"].(bool) + if v, ok := nMap["TrustedHostInterfaces"]; ok { + s, _ := v.(string) + ncfg.TrustedHostInterfaces = strings.FieldsFunc(s, func(r rune) bool { return r == ':' }) + } if v, ok := nMap["InhibitIPv4"]; ok { ncfg.InhibitIPv4 = v.(bool) } diff --git a/libnetwork/drivers/bridge/internal/iptabler/endpoint.go b/libnetwork/drivers/bridge/internal/iptabler/endpoint.go index 0716948774..1feaa74e16 100644 --- a/libnetwork/drivers/bridge/internal/iptabler/endpoint.go +++ b/libnetwork/drivers/bridge/internal/iptabler/endpoint.go @@ -43,6 +43,8 @@ func (n *Network) modEndpoint(ctx context.Context, epIPv4, epIPv6 netip.Addr, en // Packets originating on the bridge's own interface and addressed directly to the // container are allowed - the host always has direct access to its own containers // (it doesn't need to use the port mapped to its own addresses, although it can). +// +// "Trusted interfaces" are treated in the same way as the bridge itself. func (n *Network) filterDirectAccess(ctx context.Context, ipv iptables.IPVersion, config NetworkConfigFam, epIP netip.Addr, enable bool) error { if n.Internal || config.Unprotected || config.Routed { return nil @@ -55,6 +57,16 @@ func (n *Network) filterDirectAccess(ctx context.Context, ipv iptables.IPVersion if rawRulesDisabled(ctx) { enable = false } + for _, ifName := range n.TrustedHostInterfaces { + accept := iptables.Rule{IPVer: ipv, Table: iptables.Raw, Chain: "PREROUTING", Args: []string{ + "-d", epIP.String(), + "-i", ifName, + "-j", "ACCEPT", + }} + if err := appendOrDelChainRule(accept, "DIRECT ACCESS FILTERING - ACCEPT", enable); err != nil { + return err + } + } accept := iptables.Rule{IPVer: ipv, Table: iptables.Raw, Chain: "PREROUTING", Args: []string{ "-d", epIP.String(), "!", "-i", n.IfName, diff --git a/libnetwork/drivers/bridge/internal/iptabler/network.go b/libnetwork/drivers/bridge/internal/iptabler/network.go index 4bc9c114e5..3c5a28e654 100644 --- a/libnetwork/drivers/bridge/internal/iptabler/network.go +++ b/libnetwork/drivers/bridge/internal/iptabler/network.go @@ -26,12 +26,13 @@ type NetworkConfigFam struct { } type NetworkConfig struct { - IfName string - Internal bool - ICC bool - Masquerade bool - Config4 NetworkConfigFam - Config6 NetworkConfigFam + IfName string + Internal bool + ICC bool + Masquerade bool + TrustedHostInterfaces []string + Config4 NetworkConfigFam + Config6 NetworkConfigFam } type Network struct { diff --git a/libnetwork/drivers/bridge/labels.go b/libnetwork/drivers/bridge/labels.go index f99ae79b8f..364ec6c5e6 100644 --- a/libnetwork/drivers/bridge/labels.go +++ b/libnetwork/drivers/bridge/labels.go @@ -23,4 +23,8 @@ const ( // DefaultBridge label DefaultBridge = "com.docker.network.bridge.default_bridge" + + // TrustedHostInterfaces can be used to supply a list of host interfaces that are + // allowed direct access to published ports on a container's address. + TrustedHostInterfaces = "com.docker.network.bridge.trusted_host_interfaces" ) From c16caabe36dc688607f8ed79913db44366bc0da7 Mon Sep 17 00:00:00 2001 From: Rob Murray Date: Wed, 30 Apr 2025 12:25:08 +0100 Subject: [PATCH 2/3] Add TestNetworkConfigurationMarshalling Signed-off-by: Rob Murray --- .../drivers/bridge/bridge_linux_test.go | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/libnetwork/drivers/bridge/bridge_linux_test.go b/libnetwork/drivers/bridge/bridge_linux_test.go index 4d13b9e501..2c9168a18d 100644 --- a/libnetwork/drivers/bridge/bridge_linux_test.go +++ b/libnetwork/drivers/bridge/bridge_linux_test.go @@ -25,6 +25,7 @@ import ( "github.com/docker/docker/libnetwork/options" "github.com/docker/docker/libnetwork/portallocator" "github.com/docker/docker/libnetwork/types" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" "gotest.tools/v3/assert" @@ -225,6 +226,35 @@ func compareBindings(a, b []portBinding) bool { return true } +func TestNetworkConfigurationMarshalling(t *testing.T) { + nc := &networkConfiguration{ + ID: "nid", + BridgeName: "bridgename", + EnableIPv4: true, + EnableIPv6: true, + EnableIPMasquerade: true, + GwModeIPv4: gwModeRouted, + GwModeIPv6: gwModeIsolated, + EnableICC: true, + TrustedHostInterfaces: []string{"foo0", "bar1"}, + InhibitIPv4: true, + Mtu: 1234, + DefaultBindingIP: net.ParseIP("192.0.2.1"), + DefaultBridge: true, + HostIPv4: net.ParseIP("192.0.2.2"), + HostIPv6: net.ParseIP("2001:db8::1"), + ContainerIfacePrefix: "baz", + } + + b, err := json.Marshal(nc) + assert.Assert(t, err) + + nnc := &networkConfiguration{} + err = json.Unmarshal(b, nnc) + assert.Assert(t, err) + assert.Check(t, is.DeepEqual(nnc, nc, cmpopts.IgnoreUnexported(networkConfiguration{}))) +} + func getIPv4Data(t *testing.T) []driverapi.IPAMData { t.Helper() From 44a3453d73f5366fc368ed5ef6957ec03495df1c Mon Sep 17 00:00:00 2001 From: Rob Murray Date: Mon, 14 Apr 2025 17:59:25 +0100 Subject: [PATCH 3/3] Add daemon option --allow-direct-routing Per-network option com.docker.network.bridge.trusted-host-interfaces accepts a list of interfaces that are allowed to route directly to a container's published ports in a bridge network with nat enabled. This daemon level option disables direct access filtering, enabling direct access to published ports on container addresses in all bridge networks, via all host interfaces. It overlaps with short-term env-var workaround: DOCKER_INSECURE_NO_IPTABLES_RAW=1 - it does not allow packets sent from outside the host to reach ports published only to 127.0.0.1 - it will outlive iptables (the workaround was initially intended for hosts that do not have kernel support for the "raw" iptables table). Signed-off-by: Rob Murray --- cmd/dockerd/config_unix.go | 1 + daemon/config/config_linux.go | 1 + daemon/daemon_unix.go | 1 + .../networking/port_mapping_linux_test.go | 29 +++++++++++++++---- libnetwork/drivers/bridge/bridge_linux.go | 8 +++-- .../bridge/internal/iptabler/endpoint.go | 3 +- .../bridge/internal/iptabler/iptabler.go | 7 +++-- man/dockerd.8.md | 4 +++ 8 files changed, 42 insertions(+), 12 deletions(-) diff --git a/cmd/dockerd/config_unix.go b/cmd/dockerd/config_unix.go index 2706b5f1ed..de3fa443b3 100644 --- a/cmd/dockerd/config_unix.go +++ b/cmd/dockerd/config_unix.go @@ -38,6 +38,7 @@ func installConfigFlags(conf *config.Config, flags *pflag.FlagSet) { flags.IPVar(&conf.BridgeConfig.DefaultIP, "ip", net.IPv4zero, "Host IP for port publishing from the default bridge network") flags.BoolVar(&conf.BridgeConfig.EnableUserlandProxy, "userland-proxy", true, "Use userland proxy for loopback traffic") flags.StringVar(&conf.BridgeConfig.UserlandProxyPath, "userland-proxy-path", conf.BridgeConfig.UserlandProxyPath, "Path to the userland proxy binary") + flags.BoolVar(&conf.BridgeConfig.AllowDirectRouting, "allow-direct-routing", false, "Allow remote access to published ports on container IP addresses") flags.StringVar(&conf.CgroupParent, "cgroup-parent", "", "Set parent cgroup for all containers") flags.StringVar(&conf.RemappedRoot, "userns-remap", "", "User/Group setting for user namespaces") flags.BoolVar(&conf.LiveRestoreEnabled, "live-restore", false, "Enable live restore of docker when containers are still running") diff --git a/daemon/config/config_linux.go b/daemon/config/config_linux.go index 99c2e3d910..fb76204609 100644 --- a/daemon/config/config_linux.go +++ b/daemon/config/config_linux.go @@ -48,6 +48,7 @@ type BridgeConfig struct { EnableIPMasq bool `json:"ip-masq,omitempty"` EnableUserlandProxy bool `json:"userland-proxy,omitempty"` UserlandProxyPath string `json:"userland-proxy-path,omitempty"` + AllowDirectRouting bool `json:"allow-direct-routing,omitempty"` } // DefaultBridgeConfig stores all the parameters for the default bridge network. diff --git a/daemon/daemon_unix.go b/daemon/daemon_unix.go index 6cf97e1159..fd62b6f2a1 100644 --- a/daemon/daemon_unix.go +++ b/daemon/daemon_unix.go @@ -933,6 +933,7 @@ func driverOptions(config *config.Config) nwconfig.Option { "EnableIP6Tables": config.BridgeConfig.EnableIP6Tables, "EnableUserlandProxy": config.BridgeConfig.EnableUserlandProxy, "UserlandProxyPath": config.BridgeConfig.UserlandProxyPath, + "AllowDirectRouting": config.BridgeConfig.AllowDirectRouting, "Rootless": config.Rootless, }, }) diff --git a/integration/networking/port_mapping_linux_test.go b/integration/networking/port_mapping_linux_test.go index 3025572ea3..f17955d37e 100644 --- a/integration/networking/port_mapping_linux_test.go +++ b/integration/networking/port_mapping_linux_test.go @@ -917,7 +917,30 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) { // skip.If(t, testEnv.IsRootless, "rootlesskit has its own netns") ctx := setupTest(t) + d := daemon.New(t) + d.StartWithBusybox(ctx, t) + defer d.Stop(t) + testDirectRemoteAccessOnExposedPort(t, ctx, d, false) +} +// TestAllowDirectRemoteAccessOnExposedPort checks that remote hosts can directly +// reach a container on one of its exposed ports - if the daemon is running with +// option --allow-direct-routing. +func TestAllowDirectRemoteAccessOnExposedPort(t *testing.T) { + // This test checks iptables rules that live in dockerd's netns. In the case + // of rootlesskit, this is not the same netns as the host, so they don't + // have any effect. + // TODO(aker): we need to figure out what we want to do for rootlesskit. + // skip.If(t, testEnv.IsRootless, "rootlesskit has its own netns") + + ctx := setupTest(t) + d := daemon.New(t) + d.StartWithBusybox(ctx, t, "--allow-direct-routing") + defer d.Stop(t) + testDirectRemoteAccessOnExposedPort(t, ctx, d, true) +} + +func testDirectRemoteAccessOnExposedPort(t *testing.T, ctx context.Context, d *daemon.Daemon, allowDirectRouting bool) { const ( hostIPv4 = "192.168.120.2" hostIPv6 = "fdbc:277b:d40b::2" @@ -936,10 +959,6 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) { netip.MustParsePrefix("192.168.120.3/24"), netip.MustParsePrefix("fdbc:277b:d40b::3/64")) - d := daemon.New(t) - d.StartWithBusybox(ctx, t) - defer d.Stop(t) - c := d.NewClientT(t) defer c.Close() for _, tc := range []struct { @@ -1006,7 +1025,7 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - expDirectAccess := tc.gwMode == "routed" || tc.gwMode == "nat-unprotected" || tc.trusted + expDirectAccess := tc.gwMode == "routed" || tc.gwMode == "nat-unprotected" || tc.trusted || allowDirectRouting skip.If(t, expDirectAccess && testEnv.IsRootless(), "rootlesskit doesn't support routed mode as it's running in a separate netns") testutil.StartSpan(ctx, t) diff --git a/libnetwork/drivers/bridge/bridge_linux.go b/libnetwork/drivers/bridge/bridge_linux.go index c2a745c52f..83879fdef3 100644 --- a/libnetwork/drivers/bridge/bridge_linux.go +++ b/libnetwork/drivers/bridge/bridge_linux.go @@ -66,6 +66,7 @@ type configuration struct { EnableUserlandProxy bool UserlandProxyPath string Rootless bool + AllowDirectRouting bool } // networkConfiguration for network specific configuration @@ -508,9 +509,10 @@ func (d *driver) configure(option map[string]interface{}) error { var err error d.firewaller, err = iptabler.NewIptabler(iptabler.FirewallConfig{ - IPv4: config.EnableIPTables, - IPv6: config.EnableIP6Tables, - Hairpin: !config.EnableUserlandProxy || config.UserlandProxyPath == "", + IPv4: config.EnableIPTables, + IPv6: config.EnableIP6Tables, + Hairpin: !config.EnableUserlandProxy || config.UserlandProxyPath == "", + AllowDirectRouting: config.AllowDirectRouting, }) if err != nil { return err diff --git a/libnetwork/drivers/bridge/internal/iptabler/endpoint.go b/libnetwork/drivers/bridge/internal/iptabler/endpoint.go index 1feaa74e16..aeea4504ca 100644 --- a/libnetwork/drivers/bridge/internal/iptabler/endpoint.go +++ b/libnetwork/drivers/bridge/internal/iptabler/endpoint.go @@ -37,6 +37,7 @@ func (n *Network) modEndpoint(ctx context.Context, epIPv4, epIPv6 netip.Addr, en // It is a no-op if: // - the network is internal // - gateway mode is "nat-unprotected" or "routed". +// - direct routing is enabled at the daemon level. // - "raw" rules are disabled (possibly because the host doesn't have the necessary // kernel support). // @@ -54,7 +55,7 @@ func (n *Network) filterDirectAccess(ctx context.Context, ipv iptables.IPVersion // direct routing has since been disabled, the rules need to be deleted when // cleanup happens on restart. This also means a change in config over a // live-restore restart will take effect. - if rawRulesDisabled(ctx) { + if n.ipt.AllowDirectRouting || rawRulesDisabled(ctx) { enable = false } for _, ifName := range n.TrustedHostInterfaces { diff --git a/libnetwork/drivers/bridge/internal/iptabler/iptabler.go b/libnetwork/drivers/bridge/internal/iptabler/iptabler.go index b88e2739a6..77200037be 100644 --- a/libnetwork/drivers/bridge/internal/iptabler/iptabler.go +++ b/libnetwork/drivers/bridge/internal/iptabler/iptabler.go @@ -35,9 +35,10 @@ const ( ) type FirewallConfig struct { - IPv4 bool - IPv6 bool - Hairpin bool + IPv4 bool + IPv6 bool + Hairpin bool + AllowDirectRouting bool } type Iptabler struct { diff --git a/man/dockerd.8.md b/man/dockerd.8.md index 276ee90f5f..042f6ed0b7 100644 --- a/man/dockerd.8.md +++ b/man/dockerd.8.md @@ -415,6 +415,10 @@ unix://[/path/to/socket] to use. Use TLS and verify the remote (daemon: verify client, client: verify daemon). Default is **false**. +**--allow-direct-routing**=**true**|**false** + Allow remote access to published ports on container IP addresses. + Default is **false**. + **--userland-proxy**=**true**|**false** Rely on a userland proxy implementation for inter-container and outside-to-container loopback communications. Default is **true**.