diff --git a/cmd/dockerd/config_unix.go b/cmd/dockerd/config_unix.go index 2706b5f1ed..de3fa443b3 100644 --- a/cmd/dockerd/config_unix.go +++ b/cmd/dockerd/config_unix.go @@ -38,6 +38,7 @@ func installConfigFlags(conf *config.Config, flags *pflag.FlagSet) { flags.IPVar(&conf.BridgeConfig.DefaultIP, "ip", net.IPv4zero, "Host IP for port publishing from the default bridge network") flags.BoolVar(&conf.BridgeConfig.EnableUserlandProxy, "userland-proxy", true, "Use userland proxy for loopback traffic") flags.StringVar(&conf.BridgeConfig.UserlandProxyPath, "userland-proxy-path", conf.BridgeConfig.UserlandProxyPath, "Path to the userland proxy binary") + flags.BoolVar(&conf.BridgeConfig.AllowDirectRouting, "allow-direct-routing", false, "Allow remote access to published ports on container IP addresses") flags.StringVar(&conf.CgroupParent, "cgroup-parent", "", "Set parent cgroup for all containers") flags.StringVar(&conf.RemappedRoot, "userns-remap", "", "User/Group setting for user namespaces") flags.BoolVar(&conf.LiveRestoreEnabled, "live-restore", false, "Enable live restore of docker when containers are still running") diff --git a/daemon/config/config_linux.go b/daemon/config/config_linux.go index 99c2e3d910..fb76204609 100644 --- a/daemon/config/config_linux.go +++ b/daemon/config/config_linux.go @@ -48,6 +48,7 @@ type BridgeConfig struct { EnableIPMasq bool `json:"ip-masq,omitempty"` EnableUserlandProxy bool `json:"userland-proxy,omitempty"` UserlandProxyPath string `json:"userland-proxy-path,omitempty"` + AllowDirectRouting bool `json:"allow-direct-routing,omitempty"` } // DefaultBridgeConfig stores all the parameters for the default bridge network. diff --git a/daemon/daemon_unix.go b/daemon/daemon_unix.go index 6cf97e1159..fd62b6f2a1 100644 --- a/daemon/daemon_unix.go +++ b/daemon/daemon_unix.go @@ -933,6 +933,7 @@ func driverOptions(config *config.Config) nwconfig.Option { "EnableIP6Tables": config.BridgeConfig.EnableIP6Tables, "EnableUserlandProxy": config.BridgeConfig.EnableUserlandProxy, "UserlandProxyPath": config.BridgeConfig.UserlandProxyPath, + "AllowDirectRouting": config.BridgeConfig.AllowDirectRouting, "Rootless": config.Rootless, }, }) diff --git a/integration/networking/port_mapping_linux_test.go b/integration/networking/port_mapping_linux_test.go index 78e6db417a..f17955d37e 100644 --- a/integration/networking/port_mapping_linux_test.go +++ b/integration/networking/port_mapping_linux_test.go @@ -917,7 +917,30 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) { // skip.If(t, testEnv.IsRootless, "rootlesskit has its own netns") ctx := setupTest(t) + d := daemon.New(t) + d.StartWithBusybox(ctx, t) + defer d.Stop(t) + testDirectRemoteAccessOnExposedPort(t, ctx, d, false) +} +// TestAllowDirectRemoteAccessOnExposedPort checks that remote hosts can directly +// reach a container on one of its exposed ports - if the daemon is running with +// option --allow-direct-routing. +func TestAllowDirectRemoteAccessOnExposedPort(t *testing.T) { + // This test checks iptables rules that live in dockerd's netns. In the case + // of rootlesskit, this is not the same netns as the host, so they don't + // have any effect. + // TODO(aker): we need to figure out what we want to do for rootlesskit. + // skip.If(t, testEnv.IsRootless, "rootlesskit has its own netns") + + ctx := setupTest(t) + d := daemon.New(t) + d.StartWithBusybox(ctx, t, "--allow-direct-routing") + defer d.Stop(t) + testDirectRemoteAccessOnExposedPort(t, ctx, d, true) +} + +func testDirectRemoteAccessOnExposedPort(t *testing.T, ctx context.Context, d *daemon.Daemon, allowDirectRouting bool) { const ( hostIPv4 = "192.168.120.2" hostIPv6 = "fdbc:277b:d40b::2" @@ -928,17 +951,14 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) { netip.MustParsePrefix("fdbc:277b:d40b::1/64")) defer l3.Destroy(t) // "docker" is the host where dockerd is running. - l3.AddHost(t, "docker", networking.CurrentNetns, "test-eth", + const hostIfName = "test-eth" + l3.AddHost(t, "docker", networking.CurrentNetns, hostIfName, netip.MustParsePrefix(hostIPv4+"/24"), netip.MustParsePrefix(hostIPv6+"/64")) l3.AddHost(t, "attacker", "test-direct-remote-access-attacker", "eth0", netip.MustParsePrefix("192.168.120.3/24"), netip.MustParsePrefix("fdbc:277b:d40b::3/64")) - d := daemon.New(t) - d.StartWithBusybox(ctx, t) - defer d.Stop(t) - c := d.NewClientT(t) defer c.Close() for _, tc := range []struct { @@ -947,6 +967,7 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) { gwAddr netip.Prefix ipv4Disabled bool ipv6Disabled bool + trusted bool }{ { name: "NAT/IPv4", @@ -958,6 +979,18 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) { gwMode: "nat", gwAddr: netip.MustParsePrefix("fda9:a651:db6d::1/64"), }, + { + name: "NAT/IPv4/trusted", + gwMode: "nat", + gwAddr: netip.MustParsePrefix("172.24.10.1/24"), + trusted: true, + }, + { + name: "NAT/IPv6/trusted", + gwMode: "nat", + gwAddr: netip.MustParsePrefix("fda9:a651:db6d::1/64"), + trusted: true, + }, { name: "NAT unprotected/IPv4", gwMode: "nat-unprotected", @@ -992,7 +1025,8 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - skip.If(t, tc.gwMode == "routed" && testEnv.IsRootless(), "rootlesskit doesn't support routed mode as it's running in a separate netns") + expDirectAccess := tc.gwMode == "routed" || tc.gwMode == "nat-unprotected" || tc.trusted || allowDirectRouting + skip.If(t, expDirectAccess && testEnv.IsRootless(), "rootlesskit doesn't support routed mode as it's running in a separate netns") testutil.StartSpan(ctx, t) @@ -1011,6 +1045,9 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) { if tc.gwAddr.Addr().Is6() { nwOpts = append(nwOpts, network.WithIPv6()) } + if tc.trusted { + nwOpts = append(nwOpts, network.WithOption(bridge.TrustedHostInterfaces, hostIfName)) + } const bridgeName = "brattacked" network.CreateNoError(ctx, t, c, bridgeName, append(nwOpts, @@ -1054,8 +1091,6 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) { // Now send a payload directly to the container. With gw_mode=routed, // this should work. With gw_mode=nat, this should fail. - expDirectAccess := tc.gwMode == "routed" || (tc.gwMode == "nat-unprotected" && !testEnv.IsRootless()) - l3.Hosts["attacker"].Run(t, "ip", "route", "add", tc.gwAddr.Masked().String(), "via", hostIP, "dev", "eth0") defer l3.Hosts["attacker"].Run(t, "ip", "route", "delete", tc.gwAddr.Masked().String(), "via", hostIP, "dev", "eth0") diff --git a/libnetwork/drivers/bridge/bridge_linux.go b/libnetwork/drivers/bridge/bridge_linux.go index 4c37cb2455..83879fdef3 100644 --- a/libnetwork/drivers/bridge/bridge_linux.go +++ b/libnetwork/drivers/bridge/bridge_linux.go @@ -7,6 +7,7 @@ import ( "net/netip" "os" "strconv" + "strings" "sync" "syscall" @@ -65,25 +66,27 @@ type configuration struct { EnableUserlandProxy bool UserlandProxyPath string Rootless bool + AllowDirectRouting bool } // networkConfiguration for network specific configuration type networkConfiguration struct { - ID string - BridgeName string - EnableIPv4 bool - EnableIPv6 bool - EnableIPMasquerade bool - GwModeIPv4 gwMode - GwModeIPv6 gwMode - EnableICC bool - InhibitIPv4 bool - Mtu int - DefaultBindingIP net.IP - DefaultBridge bool - HostIPv4 net.IP - HostIPv6 net.IP - ContainerIfacePrefix string + ID string + BridgeName string + EnableIPv4 bool + EnableIPv6 bool + EnableIPMasquerade bool + GwModeIPv4 gwMode + GwModeIPv6 gwMode + EnableICC bool + TrustedHostInterfaces []string // Interface names must not contain ':' characters + InhibitIPv4 bool + Mtu int + DefaultBindingIP net.IP + DefaultBridge bool + HostIPv4 net.IP + HostIPv6 net.IP + ContainerIfacePrefix string // Internal fields set after ipam data parsing AddressIPv4 *net.IPNet AddressIPv6 *net.IPNet @@ -337,6 +340,8 @@ func (c *networkConfiguration) fromLabels(labels map[string]string) error { if c.EnableICC, err = strconv.ParseBool(value); err != nil { return parseErr(label, value, err.Error()) } + case TrustedHostInterfaces: + c.TrustedHostInterfaces = strings.FieldsFunc(value, func(r rune) bool { return r == ':' }) case InhibitIPv4: if c.InhibitIPv4, err = strconv.ParseBool(value); err != nil { return parseErr(label, value, err.Error()) @@ -405,12 +410,13 @@ func (n *bridgeNetwork) newIptablesNetwork() (*iptabler.Network, error) { return nil, err } return n.driver.firewaller.NewNetwork(iptabler.NetworkConfig{ - IfName: n.config.BridgeName, - Internal: n.config.Internal, - ICC: n.config.EnableICC, - Masquerade: n.config.EnableIPMasquerade, - Config4: config4, - Config6: config6, + IfName: n.config.BridgeName, + Internal: n.config.Internal, + ICC: n.config.EnableICC, + Masquerade: n.config.EnableIPMasquerade, + TrustedHostInterfaces: n.config.TrustedHostInterfaces, + Config4: config4, + Config6: config6, }) } @@ -503,9 +509,10 @@ func (d *driver) configure(option map[string]interface{}) error { var err error d.firewaller, err = iptabler.NewIptabler(iptabler.FirewallConfig{ - IPv4: config.EnableIPTables, - IPv6: config.EnableIP6Tables, - Hairpin: !config.EnableUserlandProxy || config.UserlandProxyPath == "", + IPv4: config.EnableIPTables, + IPv6: config.EnableIP6Tables, + Hairpin: !config.EnableUserlandProxy || config.UserlandProxyPath == "", + AllowDirectRouting: config.AllowDirectRouting, }) if err != nil { return err diff --git a/libnetwork/drivers/bridge/bridge_linux_test.go b/libnetwork/drivers/bridge/bridge_linux_test.go index 4d13b9e501..2c9168a18d 100644 --- a/libnetwork/drivers/bridge/bridge_linux_test.go +++ b/libnetwork/drivers/bridge/bridge_linux_test.go @@ -25,6 +25,7 @@ import ( "github.com/docker/docker/libnetwork/options" "github.com/docker/docker/libnetwork/portallocator" "github.com/docker/docker/libnetwork/types" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" "gotest.tools/v3/assert" @@ -225,6 +226,35 @@ func compareBindings(a, b []portBinding) bool { return true } +func TestNetworkConfigurationMarshalling(t *testing.T) { + nc := &networkConfiguration{ + ID: "nid", + BridgeName: "bridgename", + EnableIPv4: true, + EnableIPv6: true, + EnableIPMasquerade: true, + GwModeIPv4: gwModeRouted, + GwModeIPv6: gwModeIsolated, + EnableICC: true, + TrustedHostInterfaces: []string{"foo0", "bar1"}, + InhibitIPv4: true, + Mtu: 1234, + DefaultBindingIP: net.ParseIP("192.0.2.1"), + DefaultBridge: true, + HostIPv4: net.ParseIP("192.0.2.2"), + HostIPv6: net.ParseIP("2001:db8::1"), + ContainerIfacePrefix: "baz", + } + + b, err := json.Marshal(nc) + assert.Assert(t, err) + + nnc := &networkConfiguration{} + err = json.Unmarshal(b, nnc) + assert.Assert(t, err) + assert.Check(t, is.DeepEqual(nnc, nc, cmpopts.IgnoreUnexported(networkConfiguration{}))) +} + func getIPv4Data(t *testing.T) []driverapi.IPAMData { t.Helper() diff --git a/libnetwork/drivers/bridge/bridge_store.go b/libnetwork/drivers/bridge/bridge_store.go index 78340d2250..d976488e35 100644 --- a/libnetwork/drivers/bridge/bridge_store.go +++ b/libnetwork/drivers/bridge/bridge_store.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "net" + "strings" "github.com/containerd/log" "github.com/docker/docker/internal/otelutil" @@ -137,6 +138,7 @@ func (ncfg *networkConfiguration) MarshalJSON() ([]byte, error) { nMap["GwModeIPv4"] = ncfg.GwModeIPv4 nMap["GwModeIPv6"] = ncfg.GwModeIPv6 nMap["EnableICC"] = ncfg.EnableICC + nMap["TrustedHostInterfaces"] = strings.Join(ncfg.TrustedHostInterfaces, ":") nMap["InhibitIPv4"] = ncfg.InhibitIPv4 nMap["Mtu"] = ncfg.Mtu nMap["Internal"] = ncfg.Internal @@ -215,6 +217,10 @@ func (ncfg *networkConfiguration) UnmarshalJSON(b []byte) error { ncfg.GwModeIPv6, _ = newGwMode(v.(string)) } ncfg.EnableICC = nMap["EnableICC"].(bool) + if v, ok := nMap["TrustedHostInterfaces"]; ok { + s, _ := v.(string) + ncfg.TrustedHostInterfaces = strings.FieldsFunc(s, func(r rune) bool { return r == ':' }) + } if v, ok := nMap["InhibitIPv4"]; ok { ncfg.InhibitIPv4 = v.(bool) } diff --git a/libnetwork/drivers/bridge/internal/iptabler/endpoint.go b/libnetwork/drivers/bridge/internal/iptabler/endpoint.go index 0716948774..aeea4504ca 100644 --- a/libnetwork/drivers/bridge/internal/iptabler/endpoint.go +++ b/libnetwork/drivers/bridge/internal/iptabler/endpoint.go @@ -37,12 +37,15 @@ func (n *Network) modEndpoint(ctx context.Context, epIPv4, epIPv6 netip.Addr, en // It is a no-op if: // - the network is internal // - gateway mode is "nat-unprotected" or "routed". +// - direct routing is enabled at the daemon level. // - "raw" rules are disabled (possibly because the host doesn't have the necessary // kernel support). // // Packets originating on the bridge's own interface and addressed directly to the // container are allowed - the host always has direct access to its own containers // (it doesn't need to use the port mapped to its own addresses, although it can). +// +// "Trusted interfaces" are treated in the same way as the bridge itself. func (n *Network) filterDirectAccess(ctx context.Context, ipv iptables.IPVersion, config NetworkConfigFam, epIP netip.Addr, enable bool) error { if n.Internal || config.Unprotected || config.Routed { return nil @@ -52,9 +55,19 @@ func (n *Network) filterDirectAccess(ctx context.Context, ipv iptables.IPVersion // direct routing has since been disabled, the rules need to be deleted when // cleanup happens on restart. This also means a change in config over a // live-restore restart will take effect. - if rawRulesDisabled(ctx) { + if n.ipt.AllowDirectRouting || rawRulesDisabled(ctx) { enable = false } + for _, ifName := range n.TrustedHostInterfaces { + accept := iptables.Rule{IPVer: ipv, Table: iptables.Raw, Chain: "PREROUTING", Args: []string{ + "-d", epIP.String(), + "-i", ifName, + "-j", "ACCEPT", + }} + if err := appendOrDelChainRule(accept, "DIRECT ACCESS FILTERING - ACCEPT", enable); err != nil { + return err + } + } accept := iptables.Rule{IPVer: ipv, Table: iptables.Raw, Chain: "PREROUTING", Args: []string{ "-d", epIP.String(), "!", "-i", n.IfName, diff --git a/libnetwork/drivers/bridge/internal/iptabler/iptabler.go b/libnetwork/drivers/bridge/internal/iptabler/iptabler.go index b88e2739a6..77200037be 100644 --- a/libnetwork/drivers/bridge/internal/iptabler/iptabler.go +++ b/libnetwork/drivers/bridge/internal/iptabler/iptabler.go @@ -35,9 +35,10 @@ const ( ) type FirewallConfig struct { - IPv4 bool - IPv6 bool - Hairpin bool + IPv4 bool + IPv6 bool + Hairpin bool + AllowDirectRouting bool } type Iptabler struct { diff --git a/libnetwork/drivers/bridge/internal/iptabler/network.go b/libnetwork/drivers/bridge/internal/iptabler/network.go index 4bc9c114e5..3c5a28e654 100644 --- a/libnetwork/drivers/bridge/internal/iptabler/network.go +++ b/libnetwork/drivers/bridge/internal/iptabler/network.go @@ -26,12 +26,13 @@ type NetworkConfigFam struct { } type NetworkConfig struct { - IfName string - Internal bool - ICC bool - Masquerade bool - Config4 NetworkConfigFam - Config6 NetworkConfigFam + IfName string + Internal bool + ICC bool + Masquerade bool + TrustedHostInterfaces []string + Config4 NetworkConfigFam + Config6 NetworkConfigFam } type Network struct { diff --git a/libnetwork/drivers/bridge/labels.go b/libnetwork/drivers/bridge/labels.go index f99ae79b8f..364ec6c5e6 100644 --- a/libnetwork/drivers/bridge/labels.go +++ b/libnetwork/drivers/bridge/labels.go @@ -23,4 +23,8 @@ const ( // DefaultBridge label DefaultBridge = "com.docker.network.bridge.default_bridge" + + // TrustedHostInterfaces can be used to supply a list of host interfaces that are + // allowed direct access to published ports on a container's address. + TrustedHostInterfaces = "com.docker.network.bridge.trusted_host_interfaces" ) diff --git a/man/dockerd.8.md b/man/dockerd.8.md index 276ee90f5f..042f6ed0b7 100644 --- a/man/dockerd.8.md +++ b/man/dockerd.8.md @@ -415,6 +415,10 @@ unix://[/path/to/socket] to use. Use TLS and verify the remote (daemon: verify client, client: verify daemon). Default is **false**. +**--allow-direct-routing**=**true**|**false** + Allow remote access to published ports on container IP addresses. + Default is **false**. + **--userland-proxy**=**true**|**false** Rely on a userland proxy implementation for inter-container and outside-to-container loopback communications. Default is **true**.