Merge pull request #49832 from robmry/trusted_interfaces

Allow direct routing to container ports from trusted interfaces

cmd/dockerd/config_unix.go

@@ -38,6 +38,7 @@ func installConfigFlags(conf *config.Config, flags *pflag.FlagSet) {
 	flags.IPVar(&conf.BridgeConfig.DefaultIP, "ip", net.IPv4zero, "Host IP for port publishing from the default bridge network")
 	flags.BoolVar(&conf.BridgeConfig.EnableUserlandProxy, "userland-proxy", true, "Use userland proxy for loopback traffic")
 	flags.StringVar(&conf.BridgeConfig.UserlandProxyPath, "userland-proxy-path", conf.BridgeConfig.UserlandProxyPath, "Path to the userland proxy binary")
+	flags.BoolVar(&conf.BridgeConfig.AllowDirectRouting, "allow-direct-routing", false, "Allow remote access to published ports on container IP addresses")
 	flags.StringVar(&conf.CgroupParent, "cgroup-parent", "", "Set parent cgroup for all containers")
 	flags.StringVar(&conf.RemappedRoot, "userns-remap", "", "User/Group setting for user namespaces")
 	flags.BoolVar(&conf.LiveRestoreEnabled, "live-restore", false, "Enable live restore of docker when containers are still running")

daemon/config/config_linux.go

@@ -48,6 +48,7 @@ type BridgeConfig struct {
 	EnableIPMasq             bool   `json:"ip-masq,omitempty"`
 	EnableUserlandProxy      bool   `json:"userland-proxy,omitempty"`
 	UserlandProxyPath        string `json:"userland-proxy-path,omitempty"`
+	AllowDirectRouting       bool   `json:"allow-direct-routing,omitempty"`
 }
 
 // DefaultBridgeConfig stores all the parameters for the default bridge network.

daemon/daemon_unix.go

@@ -933,6 +933,7 @@ func driverOptions(config *config.Config) nwconfig.Option {
 			"EnableIP6Tables":          config.BridgeConfig.EnableIP6Tables,
 			"EnableUserlandProxy":      config.BridgeConfig.EnableUserlandProxy,
 			"UserlandProxyPath":        config.BridgeConfig.UserlandProxyPath,
+			"AllowDirectRouting":       config.BridgeConfig.AllowDirectRouting,
 			"Rootless":                 config.Rootless,
 		},
 	})

integration/networking/port_mapping_linux_test.go

@@ -917,7 +917,30 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) {
 	// skip.If(t, testEnv.IsRootless, "rootlesskit has its own netns")
 
 	ctx := setupTest(t)
+	d := daemon.New(t)
+	d.StartWithBusybox(ctx, t)
+	defer d.Stop(t)
+	testDirectRemoteAccessOnExposedPort(t, ctx, d, false)
+}
 
+// TestAllowDirectRemoteAccessOnExposedPort checks that remote hosts can directly
+// reach a container on one of its exposed ports - if the daemon is running with
+// option --allow-direct-routing.
+func TestAllowDirectRemoteAccessOnExposedPort(t *testing.T) {
+	// This test checks iptables rules that live in dockerd's netns. In the case
+	// of rootlesskit, this is not the same netns as the host, so they don't
+	// have any effect.
+	// TODO(aker): we need to figure out what we want to do for rootlesskit.
+	// skip.If(t, testEnv.IsRootless, "rootlesskit has its own netns")
+
+	ctx := setupTest(t)
+	d := daemon.New(t)
+	d.StartWithBusybox(ctx, t, "--allow-direct-routing")
+	defer d.Stop(t)
+	testDirectRemoteAccessOnExposedPort(t, ctx, d, true)
+}
+
+func testDirectRemoteAccessOnExposedPort(t *testing.T, ctx context.Context, d *daemon.Daemon, allowDirectRouting bool) {
 	const (
 		hostIPv4 = "192.168.120.2"
 		hostIPv6 = "fdbc:277b:d40b::2"
@@ -928,17 +951,14 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) {
 		netip.MustParsePrefix("fdbc:277b:d40b::1/64"))
 	defer l3.Destroy(t)
 	// "docker" is the host where dockerd is running.
-	l3.AddHost(t, "docker", networking.CurrentNetns, "test-eth",
+	const hostIfName = "test-eth"
+	l3.AddHost(t, "docker", networking.CurrentNetns, hostIfName,
 		netip.MustParsePrefix(hostIPv4+"/24"),
 		netip.MustParsePrefix(hostIPv6+"/64"))
 	l3.AddHost(t, "attacker", "test-direct-remote-access-attacker", "eth0",
 		netip.MustParsePrefix("192.168.120.3/24"),
 		netip.MustParsePrefix("fdbc:277b:d40b::3/64"))
 
-	d := daemon.New(t)
-	d.StartWithBusybox(ctx, t)
-	defer d.Stop(t)
-
 	c := d.NewClientT(t)
 	defer c.Close()
 	for _, tc := range []struct {
@@ -947,6 +967,7 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) {
 		gwAddr       netip.Prefix
 		ipv4Disabled bool
 		ipv6Disabled bool
+		trusted      bool
 	}{
 		{
 			name:   "NAT/IPv4",
@@ -958,6 +979,18 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) {
 			gwMode: "nat",
 			gwAddr: netip.MustParsePrefix("fda9:a651:db6d::1/64"),
 		},
+		{
+			name:    "NAT/IPv4/trusted",
+			gwMode:  "nat",
+			gwAddr:  netip.MustParsePrefix("172.24.10.1/24"),
+			trusted: true,
+		},
+		{
+			name:    "NAT/IPv6/trusted",
+			gwMode:  "nat",
+			gwAddr:  netip.MustParsePrefix("fda9:a651:db6d::1/64"),
+			trusted: true,
+		},
 		{
 			name:   "NAT unprotected/IPv4",
 			gwMode: "nat-unprotected",
@@ -992,7 +1025,8 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) {
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
-			skip.If(t, tc.gwMode == "routed" && testEnv.IsRootless(), "rootlesskit doesn't support routed mode as it's running in a separate netns")
+			expDirectAccess := tc.gwMode == "routed" || tc.gwMode == "nat-unprotected" || tc.trusted || allowDirectRouting
+			skip.If(t, expDirectAccess && testEnv.IsRootless(), "rootlesskit doesn't support routed mode as it's running in a separate netns")
 
 			testutil.StartSpan(ctx, t)
 
@@ -1011,6 +1045,9 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) {
 			if tc.gwAddr.Addr().Is6() {
 				nwOpts = append(nwOpts, network.WithIPv6())
 			}
+			if tc.trusted {
+				nwOpts = append(nwOpts, network.WithOption(bridge.TrustedHostInterfaces, hostIfName))
+			}
 
 			const bridgeName = "brattacked"
 			network.CreateNoError(ctx, t, c, bridgeName, append(nwOpts,
@@ -1054,8 +1091,6 @@ func TestDirectRemoteAccessOnExposedPort(t *testing.T) {
 
 			// Now send a payload directly to the container. With gw_mode=routed,
 			// this should work. With gw_mode=nat, this should fail.
-			expDirectAccess := tc.gwMode == "routed" || (tc.gwMode == "nat-unprotected" && !testEnv.IsRootless())
-
 			l3.Hosts["attacker"].Run(t, "ip", "route", "add", tc.gwAddr.Masked().String(), "via", hostIP, "dev", "eth0")
 			defer l3.Hosts["attacker"].Run(t, "ip", "route", "delete", tc.gwAddr.Masked().String(), "via", hostIP, "dev", "eth0")
 

libnetwork/drivers/bridge/bridge_linux.go

@@ -7,6 +7,7 @@ import (
 	"net/netip"
 	"os"
 	"strconv"
+	"strings"
 	"sync"
 	"syscall"
 
@@ -65,25 +66,27 @@ type configuration struct {
 	EnableUserlandProxy      bool
 	UserlandProxyPath        string
 	Rootless                 bool
+	AllowDirectRouting       bool
 }
 
 // networkConfiguration for network specific configuration
 type networkConfiguration struct {
-	ID                   string
-	BridgeName           string
-	EnableIPv4           bool
-	EnableIPv6           bool
-	EnableIPMasquerade   bool
-	GwModeIPv4           gwMode
-	GwModeIPv6           gwMode
-	EnableICC            bool
-	InhibitIPv4          bool
-	Mtu                  int
-	DefaultBindingIP     net.IP
-	DefaultBridge        bool
-	HostIPv4             net.IP
-	HostIPv6             net.IP
-	ContainerIfacePrefix string
+	ID                    string
+	BridgeName            string
+	EnableIPv4            bool
+	EnableIPv6            bool
+	EnableIPMasquerade    bool
+	GwModeIPv4            gwMode
+	GwModeIPv6            gwMode
+	EnableICC             bool
+	TrustedHostInterfaces []string // Interface names must not contain ':' characters
+	InhibitIPv4           bool
+	Mtu                   int
+	DefaultBindingIP      net.IP
+	DefaultBridge         bool
+	HostIPv4              net.IP
+	HostIPv6              net.IP
+	ContainerIfacePrefix  string
 	// Internal fields set after ipam data parsing
 	AddressIPv4        *net.IPNet
 	AddressIPv6        *net.IPNet
@@ -337,6 +340,8 @@ func (c *networkConfiguration) fromLabels(labels map[string]string) error {
 			if c.EnableICC, err = strconv.ParseBool(value); err != nil {
 				return parseErr(label, value, err.Error())
 			}
+		case TrustedHostInterfaces:
+			c.TrustedHostInterfaces = strings.FieldsFunc(value, func(r rune) bool { return r == ':' })
 		case InhibitIPv4:
 			if c.InhibitIPv4, err = strconv.ParseBool(value); err != nil {
 				return parseErr(label, value, err.Error())
@@ -405,12 +410,13 @@ func (n *bridgeNetwork) newIptablesNetwork() (*iptabler.Network, error) {
 		return nil, err
 	}
 	return n.driver.firewaller.NewNetwork(iptabler.NetworkConfig{
-		IfName:     n.config.BridgeName,
-		Internal:   n.config.Internal,
-		ICC:        n.config.EnableICC,
-		Masquerade: n.config.EnableIPMasquerade,
-		Config4:    config4,
-		Config6:    config6,
+		IfName:                n.config.BridgeName,
+		Internal:              n.config.Internal,
+		ICC:                   n.config.EnableICC,
+		Masquerade:            n.config.EnableIPMasquerade,
+		TrustedHostInterfaces: n.config.TrustedHostInterfaces,
+		Config4:               config4,
+		Config6:               config6,
 	})
 }
 
@@ -503,9 +509,10 @@ func (d *driver) configure(option map[string]interface{}) error {
 
 	var err error
 	d.firewaller, err = iptabler.NewIptabler(iptabler.FirewallConfig{
-		IPv4:    config.EnableIPTables,
-		IPv6:    config.EnableIP6Tables,
-		Hairpin: !config.EnableUserlandProxy || config.UserlandProxyPath == "",
+		IPv4:               config.EnableIPTables,
+		IPv6:               config.EnableIP6Tables,
+		Hairpin:            !config.EnableUserlandProxy || config.UserlandProxyPath == "",
+		AllowDirectRouting: config.AllowDirectRouting,
 	})
 	if err != nil {
 		return err

libnetwork/drivers/bridge/bridge_linux_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/docker/docker/libnetwork/options"
 	"github.com/docker/docker/libnetwork/portallocator"
 	"github.com/docker/docker/libnetwork/types"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/vishvananda/netlink"
 	"github.com/vishvananda/netns"
 	"gotest.tools/v3/assert"
@@ -225,6 +226,35 @@ func compareBindings(a, b []portBinding) bool {
 	return true
 }
 
+func TestNetworkConfigurationMarshalling(t *testing.T) {
+	nc := &networkConfiguration{
+		ID:                    "nid",
+		BridgeName:            "bridgename",
+		EnableIPv4:            true,
+		EnableIPv6:            true,
+		EnableIPMasquerade:    true,
+		GwModeIPv4:            gwModeRouted,
+		GwModeIPv6:            gwModeIsolated,
+		EnableICC:             true,
+		TrustedHostInterfaces: []string{"foo0", "bar1"},
+		InhibitIPv4:           true,
+		Mtu:                   1234,
+		DefaultBindingIP:      net.ParseIP("192.0.2.1"),
+		DefaultBridge:         true,
+		HostIPv4:              net.ParseIP("192.0.2.2"),
+		HostIPv6:              net.ParseIP("2001:db8::1"),
+		ContainerIfacePrefix:  "baz",
+	}
+
+	b, err := json.Marshal(nc)
+	assert.Assert(t, err)
+
+	nnc := &networkConfiguration{}
+	err = json.Unmarshal(b, nnc)
+	assert.Assert(t, err)
+	assert.Check(t, is.DeepEqual(nnc, nc, cmpopts.IgnoreUnexported(networkConfiguration{})))
+}
+
 func getIPv4Data(t *testing.T) []driverapi.IPAMData {
 	t.Helper()
 

libnetwork/drivers/bridge/bridge_store.go

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"strings"
 
 	"github.com/containerd/log"
 	"github.com/docker/docker/internal/otelutil"
@@ -137,6 +138,7 @@ func (ncfg *networkConfiguration) MarshalJSON() ([]byte, error) {
 	nMap["GwModeIPv4"] = ncfg.GwModeIPv4
 	nMap["GwModeIPv6"] = ncfg.GwModeIPv6
 	nMap["EnableICC"] = ncfg.EnableICC
+	nMap["TrustedHostInterfaces"] = strings.Join(ncfg.TrustedHostInterfaces, ":")
 	nMap["InhibitIPv4"] = ncfg.InhibitIPv4
 	nMap["Mtu"] = ncfg.Mtu
 	nMap["Internal"] = ncfg.Internal
@@ -215,6 +217,10 @@ func (ncfg *networkConfiguration) UnmarshalJSON(b []byte) error {
 		ncfg.GwModeIPv6, _ = newGwMode(v.(string))
 	}
 	ncfg.EnableICC = nMap["EnableICC"].(bool)
+	if v, ok := nMap["TrustedHostInterfaces"]; ok {
+		s, _ := v.(string)
+		ncfg.TrustedHostInterfaces = strings.FieldsFunc(s, func(r rune) bool { return r == ':' })
+	}
 	if v, ok := nMap["InhibitIPv4"]; ok {
 		ncfg.InhibitIPv4 = v.(bool)
 	}

libnetwork/drivers/bridge/internal/iptabler/endpoint.go

@@ -37,12 +37,15 @@ func (n *Network) modEndpoint(ctx context.Context, epIPv4, epIPv6 netip.Addr, en
 // It is a no-op if:
 //   - the network is internal
 //   - gateway mode is "nat-unprotected" or "routed".
+//   - direct routing is enabled at the daemon level.
 //   - "raw" rules are disabled (possibly because the host doesn't have the necessary
 //     kernel support).
 //
 // Packets originating on the bridge's own interface and addressed directly to the
 // container are allowed - the host always has direct access to its own containers
 // (it doesn't need to use the port mapped to its own addresses, although it can).
+//
+// "Trusted interfaces" are treated in the same way as the bridge itself.
 func (n *Network) filterDirectAccess(ctx context.Context, ipv iptables.IPVersion, config NetworkConfigFam, epIP netip.Addr, enable bool) error {
 	if n.Internal || config.Unprotected || config.Routed {
 		return nil
@@ -52,9 +55,19 @@ func (n *Network) filterDirectAccess(ctx context.Context, ipv iptables.IPVersion
 	// direct routing has since been disabled, the rules need to be deleted when
 	// cleanup happens on restart. This also means a change in config over a
 	// live-restore restart will take effect.
-	if rawRulesDisabled(ctx) {
+	if n.ipt.AllowDirectRouting || rawRulesDisabled(ctx) {
 		enable = false
 	}
+	for _, ifName := range n.TrustedHostInterfaces {
+		accept := iptables.Rule{IPVer: ipv, Table: iptables.Raw, Chain: "PREROUTING", Args: []string{
+			"-d", epIP.String(),
+			"-i", ifName,
+			"-j", "ACCEPT",
+		}}
+		if err := appendOrDelChainRule(accept, "DIRECT ACCESS FILTERING - ACCEPT", enable); err != nil {
+			return err
+		}
+	}
 	accept := iptables.Rule{IPVer: ipv, Table: iptables.Raw, Chain: "PREROUTING", Args: []string{
 		"-d", epIP.String(),
 		"!", "-i", n.IfName,

libnetwork/drivers/bridge/internal/iptabler/iptabler.go

@@ -35,9 +35,10 @@ const (
 )
 
 type FirewallConfig struct {
-	IPv4    bool
-	IPv6    bool
-	Hairpin bool
+	IPv4               bool
+	IPv6               bool
+	Hairpin            bool
+	AllowDirectRouting bool
 }
 
 type Iptabler struct {

libnetwork/drivers/bridge/internal/iptabler/network.go

@@ -26,12 +26,13 @@ type NetworkConfigFam struct {
 }
 
 type NetworkConfig struct {
-	IfName     string
-	Internal   bool
-	ICC        bool
-	Masquerade bool
-	Config4    NetworkConfigFam
-	Config6    NetworkConfigFam
+	IfName                string
+	Internal              bool
+	ICC                   bool
+	Masquerade            bool
+	TrustedHostInterfaces []string
+	Config4               NetworkConfigFam
+	Config6               NetworkConfigFam
 }
 
 type Network struct {

libnetwork/drivers/bridge/labels.go

@@ -23,4 +23,8 @@ const (
 
 	// DefaultBridge label
 	DefaultBridge = "com.docker.network.bridge.default_bridge"
+
+	// TrustedHostInterfaces can be used to supply a list of host interfaces that are
+	// allowed direct access to published ports on a container's address.
+	TrustedHostInterfaces = "com.docker.network.bridge.trusted_host_interfaces"
 )

man/dockerd.8.md

@@ -415,6 +415,10 @@ unix://[/path/to/socket] to use.
   Use TLS and verify the remote (daemon: verify client, client: verify daemon).
   Default is **false**.
 
+**--allow-direct-routing**=**true**|**false**
+  Allow remote access to published ports on container IP addresses.
+  Default is **false**.
+
 **--userland-proxy**=**true**|**false**
   Rely on a userland proxy implementation for inter-container and
   outside-to-container loopback communications. Default is **true**.