mirror of
https://github.com/moby/moby.git
synced 2026-07-22 15:32:50 +00:00
This patch allows endpoints to complete servicing connections while being removed from a service. The fix is entirely within libnetwork and requires no changes to the moby codebase proper. It operates by initially down-weighting a container endpoint in the load balancer to 0 while keeping the endpoint present in the load balancer. This allows traffic to continue to flow to the endpoint while preventing new connections from going to the endpoint. This allows the container to complete requests during the "stop_grace_period" and then exit when finished without interruption of service. This change requires propagating the status of disabled service endpoints via the networkDB. Accordingly, the patch includes both code to generate and handle service update messages. It also augments the service structure with a ServiceDisabled boolean to convey whether an endpoint should ultimately be removed or just disabled. This, naturally, required a rebuild of the protocol buffer code. The protocol buffer encoding is designed to support additions of fields to messages in a backwards-compatible manner. Protocol buffer unmarshalling code automatically skips past any fields that it isn't aware of. As a result, an older moby daemon without this fix can receive and will process correctly networkDB messages from newer moby daemons with this patch. As it turns out, the additional field is simply a bool that is otherwise irrelevent on networkDB create and delete events. So its absence in older moby daemon processing has no impact. However, the fix leverages the "update" networkDB message which was previously unused in libnetwork. Although older libnetwork implementations parse the message cleanly, they will see the message as unexpected and as such issue a log at error level indicating the receipt of such. Other than this there should be no other negative impact for use of this patch in mixed environments. (Although older mobys won't be able to gracefully downgrade connections on their nodes of course.) Signed-off-by: Chris Telfer <ctelfer@docker.com>
164 lines
4.1 KiB
Go
164 lines
4.1 KiB
Go
package libnetwork
|
|
|
|
import (
|
|
"net"
|
|
|
|
"github.com/Microsoft/hcsshim"
|
|
"github.com/docker/docker/pkg/system"
|
|
"github.com/sirupsen/logrus"
|
|
)
|
|
|
|
type policyLists struct {
|
|
ilb *hcsshim.PolicyList
|
|
elb *hcsshim.PolicyList
|
|
}
|
|
|
|
var lbPolicylistMap map[*loadBalancer]*policyLists
|
|
|
|
func init() {
|
|
lbPolicylistMap = make(map[*loadBalancer]*policyLists)
|
|
}
|
|
|
|
func (n *network) addLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []*PortConfig) {
|
|
|
|
if system.GetOSVersion().Build > 16236 {
|
|
lb.Lock()
|
|
defer lb.Unlock()
|
|
//find the load balancer IP for the network.
|
|
var sourceVIP string
|
|
for _, e := range n.Endpoints() {
|
|
epInfo := e.Info()
|
|
if epInfo == nil {
|
|
continue
|
|
}
|
|
if epInfo.LoadBalancer() {
|
|
sourceVIP = epInfo.Iface().Address().IP.String()
|
|
break
|
|
}
|
|
}
|
|
|
|
if sourceVIP == "" {
|
|
logrus.Errorf("Failed to find load balancer IP for network %s", n.Name())
|
|
return
|
|
}
|
|
|
|
var endpoints []hcsshim.HNSEndpoint
|
|
|
|
for eid, be := range lb.backEnds {
|
|
if be.disabled {
|
|
continue
|
|
}
|
|
//Call HNS to get back ID (GUID) corresponding to the endpoint.
|
|
hnsEndpoint, err := hcsshim.GetHNSEndpointByName(eid)
|
|
if err != nil {
|
|
logrus.Errorf("Failed to find HNS ID for endpoint %v: %v", eid, err)
|
|
return
|
|
}
|
|
|
|
endpoints = append(endpoints, *hnsEndpoint)
|
|
}
|
|
|
|
if policies, ok := lbPolicylistMap[lb]; ok {
|
|
|
|
if policies.ilb != nil {
|
|
policies.ilb.Delete()
|
|
policies.ilb = nil
|
|
}
|
|
|
|
if policies.elb != nil {
|
|
policies.elb.Delete()
|
|
policies.elb = nil
|
|
}
|
|
delete(lbPolicylistMap, lb)
|
|
}
|
|
|
|
ilbPolicy, err := hcsshim.AddLoadBalancer(endpoints, true, sourceVIP, vip.String(), 0, 0, 0)
|
|
if err != nil {
|
|
logrus.Errorf("Failed to add ILB policy for service %s (%s) with endpoints %v using load balancer IP %s on network %s: %v",
|
|
lb.service.name, vip.String(), endpoints, sourceVIP, n.Name(), err)
|
|
return
|
|
}
|
|
|
|
lbPolicylistMap[lb] = &policyLists{
|
|
ilb: ilbPolicy,
|
|
}
|
|
|
|
publishedPorts := make(map[uint32]uint32)
|
|
|
|
for i, port := range ingressPorts {
|
|
protocol := uint16(6)
|
|
|
|
// Skip already published port
|
|
if publishedPorts[port.PublishedPort] == port.TargetPort {
|
|
continue
|
|
}
|
|
|
|
if port.Protocol == ProtocolUDP {
|
|
protocol = 17
|
|
}
|
|
|
|
// check if already has udp matching to add wild card publishing
|
|
for j := i + 1; j < len(ingressPorts); j++ {
|
|
if ingressPorts[j].TargetPort == port.TargetPort &&
|
|
ingressPorts[j].PublishedPort == port.PublishedPort {
|
|
protocol = 0
|
|
}
|
|
}
|
|
|
|
publishedPorts[port.PublishedPort] = port.TargetPort
|
|
|
|
lbPolicylistMap[lb].elb, err = hcsshim.AddLoadBalancer(endpoints, false, sourceVIP, "", protocol, uint16(port.TargetPort), uint16(port.PublishedPort))
|
|
if err != nil {
|
|
logrus.Errorf("Failed to add ELB policy for service %s (ip:%s target port:%v published port:%v) with endpoints %v using load balancer IP %s on network %s: %v",
|
|
lb.service.name, vip.String(), uint16(port.TargetPort), uint16(port.PublishedPort), endpoints, sourceVIP, n.Name(), err)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (n *network) rmLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []*PortConfig, rmService bool, fullRemove bool) {
|
|
if system.GetOSVersion().Build > 16236 {
|
|
if numEnabledBackends(lb) > 0 {
|
|
//Reprogram HNS (actually VFP) with the existing backends.
|
|
n.addLBBackend(ip, vip, lb, ingressPorts)
|
|
} else {
|
|
lb.Lock()
|
|
defer lb.Unlock()
|
|
logrus.Debugf("No more backends for service %s (ip:%s). Removing all policies", lb.service.name, lb.vip.String())
|
|
|
|
if policyLists, ok := lbPolicylistMap[lb]; ok {
|
|
if policyLists.ilb != nil {
|
|
policyLists.ilb.Delete()
|
|
policyLists.ilb = nil
|
|
}
|
|
|
|
if policyLists.elb != nil {
|
|
policyLists.elb.Delete()
|
|
policyLists.elb = nil
|
|
}
|
|
delete(lbPolicylistMap, lb)
|
|
|
|
} else {
|
|
logrus.Errorf("Failed to find policies for service %s (%s)", lb.service.name, lb.vip.String())
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func numEnabledBackends(lb *loadBalancer) int {
|
|
nEnabled := 0
|
|
for _, be := range lb.backEnds {
|
|
if !be.disabled {
|
|
nEnabled++
|
|
}
|
|
}
|
|
return nEnabled
|
|
}
|
|
|
|
func (sb *sandbox) populateLoadbalancers(ep *endpoint) {
|
|
}
|
|
|
|
func arrangeIngressFilterRule() {
|
|
}
|