Always allow access to routed endpoints

When an endpoint in a gateway mode "nat" network is selected
as a container's default gateway, the bridge driver sets up
bindings between host and container ports (NAT, userland proxy
etc).

When gateway mode "routed" was added as an alternative to
the default "nat" mode - port bindings followed the same rules.

But, unlike "nat" mode, there's no host port binding to set
up - there's routing between remote client and the container,
so it doesn't matter what the default gateway is.

So, in "routed" mode, set up the rules to make a container's
published ports accessible when the endpoint is added, and
remove those rules when the endpoint is removed (when the
container is disconnected from the endpoint's network).

Port mappings are only provided by ProgramExternalConnectivity,
they can't be set up during the Join. So, include routed
bindings in the port bindings mode that's stored as part of
endpoint state - and use that to work out whether to add or
remove bindings.

Signed-off-by: Rob Murray <rob.murray@docker.com>

daemon/libnetwork/drivers/bridge/bridge_linux.go

@@ -1488,6 +1488,15 @@ func (d *driver) Leave(nid, eid string) error {
 		return endpointNotFoundError(eid)
 	}
 
+	if endpoint.portMapping != nil {
+		if err := network.releasePorts(endpoint); err != nil {
+			return err
+		}
+		if err = d.storeUpdate(context.TODO(), endpoint); err != nil {
+			return fmt.Errorf("during leave, failed to store bridge endpoint %.7s: %v", endpoint.id, err)
+		}
+	}
+
 	if !network.config.EnableICC {
 		if err = d.link(network, endpoint, false); err != nil {
 			return err
@@ -1498,8 +1507,9 @@ func (d *driver) Leave(nid, eid string) error {
 }
 
 type portBindingMode struct {
-	ipv4 bool
-	ipv6 bool
+	routed bool
+	ipv4   bool
+	ipv6   bool
 }
 
 func (d *driver) ProgramExternalConnectivity(ctx context.Context, nid, eid string, gw4Id, gw6Id string) (retErr error) {
@@ -1524,12 +1534,12 @@ func (d *driver) ProgramExternalConnectivity(ctx context.Context, nid, eid strin
 	if err != nil {
 		return err
 	}
-
 	if endpoint == nil {
 		return endpointNotFoundError(eid)
 	}
 
-	var pbmReq portBindingMode
+	// Always include rules for routed-mode port mappings - they'll be removed on Leave.
+	pbmReq := portBindingMode{routed: true}
 	// Act as the IPv4 gateway if explicitly selected.
 	if gw4Id == eid {
 		pbmReq.ipv4 = true
@@ -1558,7 +1568,7 @@ func (d *driver) ProgramExternalConnectivity(ctx context.Context, nid, eid strin
 	}()
 
 	// Set up new port bindings, and store them in the endpoint.
-	if (pbmReq.ipv4 || pbmReq.ipv6) && endpoint.extConnConfig != nil && endpoint.extConnConfig.PortBindings != nil {
+	if endpoint.extConnConfig != nil && endpoint.extConnConfig.PortBindings != nil {
 		newPMs, err := network.addPortMappings(ctx, endpoint, endpoint.extConnConfig.PortBindings, network.config.DefaultBindingIP, pbmReq)
 		if err != nil {
 			return err
@@ -1585,15 +1595,23 @@ func (d *driver) ProgramExternalConnectivity(ctx context.Context, nid, eid strin
 //
 // ep.portMapping is updated when bindings are removed.
 func (ep *bridgeEndpoint) trimPortBindings(ctx context.Context, n *bridgeNetwork, pbmReq portBindingMode) (func() []portmapperapi.PortBinding, error) {
-	// If the endpoint is the gateway for IPv4 and IPv6, there's nothing to drop.
-	if pbmReq.ipv4 && pbmReq.ipv6 {
+	// Drop IPv4 bindings if this endpoint is not the IPv4 gateway, unless the
+	// network is "routed" (routed bindings get dropped unconditionally by Leave).
+	drop4 := !pbmReq.ipv4 && !n.gwMode(firewaller.IPv4).routed()
+
+	// Drop IPv6 bindings if this endpoint is not the IPv6 gateway, and not proxying
+	// from host IPv6 to container IPv6 because there is no IPv6 gateway - unless the
+	// IPv6 network is "routed" (routed bindings get dropped unconditionally by Leave).
+	drop6 := !pbmReq.ipv6 && !n.gwMode(firewaller.IPv6).routed()
+
+	if !drop4 && !drop6 {
 		return nil, nil
 	}
 
 	toDrop := make([]portmapperapi.PortBinding, 0, len(ep.portMapping))
 	toKeep := slices.DeleteFunc(ep.portMapping, func(pb portmapperapi.PortBinding) bool {
 		is4 := pb.HostIP.To4() != nil
-		if (is4 && !pbmReq.ipv4) || (!is4 && !pbmReq.ipv6) {
+		if (is4 && drop4) || (!is4 && drop6) {
 			toDrop = append(toDrop, pb)
 			return true
 		}

daemon/libnetwork/drivers/bridge/bridge_store.go

@@ -471,7 +471,7 @@ func (n *bridgeNetwork) restorePortAllocations(ep *bridgeEndpoint) {
 	// there are no IPv6 bindings, it doesn't matter whether that was because this
 	// endpoint is not an IPv6 gateway and "pbmIPv6" was not set in the port
 	// binding state, or there were just no IPv6 port bindings configured.)
-	var pbm portBindingMode
+	pbm := portBindingMode{routed: true}
 	for _, b := range ep.portMapping {
 		if b.HostIP.To4() == nil {
 			pbm.ipv6 = true

daemon/libnetwork/drivers/bridge/port_mapping_linux.go

@@ -129,8 +129,8 @@ func (n *bridgeNetwork) sortAndNormPBs(
 	hairpin := n.hairpin()
 	disableNAT4, disableNAT6 := n.getNATDisabled()
 
-	add4 := !ep.portBindingState.ipv4 && pbmReq.ipv4
-	add6 := !ep.portBindingState.ipv6 && pbmReq.ipv6
+	add4 := !ep.portBindingState.ipv4 && pbmReq.ipv4 || (disableNAT4 && !ep.portBindingState.routed && pbmReq.routed)
+	add6 := !ep.portBindingState.ipv6 && pbmReq.ipv6 || (disableNAT6 && !ep.portBindingState.routed && pbmReq.routed)
 
 	reqs := make([]portmapperapi.PortBindingReq, 0, len(cfg))
 	for _, c := range cfg {

daemon/libnetwork/drivers/bridge/port_mapping_linux_test.go

@@ -833,7 +833,7 @@ func TestAddPortMappings(t *testing.T) {
 				addr:   tc.epAddrV4,
 				addrv6: tc.epAddrV6,
 			}
-			var pbm portBindingMode
+			pbm := portBindingMode{routed: true}
 			if ep.addr != nil {
 				pbm.ipv4 = true
 			}

integration/networking/port_mapping_linux_test.go

@@ -912,13 +912,13 @@ func TestRoutedNonGateway(t *testing.T) {
 			name:    "routed/direct/v4",
 			addr:    insp.NetworkSettings.Networks[routedNetName].IPAddress,
 			port:    "80",
-			expHttp: httpFail,
+			expHttp: httpSuccess,
 		},
 		{
 			name:    "routed/direct/v6",
 			addr:    insp.NetworkSettings.Networks[routedNetName].GlobalIPv6Address,
 			port:    "80",
-			expHttp: httpFail,
+			expHttp: httpSuccess,
 		},
 	}