kubernetes/pkg/proxy/localnodeportproxy/proxy.go

/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package localnodeportproxy

import (
	"context"
	"errors"
	"fmt"
	"io"
	"math/rand/v2"
	"net"
	"sync"
	"time"

	v1 "k8s.io/api/core/v1"
	"k8s.io/klog/v2"
	"k8s.io/kubernetes/pkg/proxy"
	"k8s.io/kubernetes/pkg/proxy/metrics"
	"k8s.io/utils/clock"
)

// dialTimeout is the timeout for connecting to a backend endpoint.
const dialTimeout = 5 * time.Second

// NodePortSpec describes a single NodePort that needs a localhost proxy.
// Callers build a slice of these during their own sync loop and hand it
// to SyncNodePorts.
type NodePortSpec struct {
	ServicePortName proxy.ServicePortName
	Protocol        v1.Protocol
	NodePort        int
	// Endpoints is the set of "ip:port" backends the listener round-robins over.
	Endpoints []string
	// SessionAffinityType mirrors Service.spec.sessionAffinity. When set to
	// ClientIP, consecutive connections from localhost are pinned to the same
	// endpoint for StickyMaxAgeSeconds.
	SessionAffinityType v1.ServiceAffinity
	// StickyMaxAgeSeconds is the affinity timeout in seconds, used only when
	// SessionAffinityType is ClientIP.
	StickyMaxAgeSeconds int
}

// affinityTimeout returns the effective affinity window for the spec, or 0
// when affinity is disabled.
func (s *NodePortSpec) affinityTimeout() time.Duration {
	if s.SessionAffinityType != v1.ServiceAffinityClientIP {
		return 0
	}
	return time.Duration(s.StickyMaxAgeSeconds) * time.Second
}

// LocalNodePortProxy manages userspace L4 proxy listeners on localhost
// for NodePort services in nftables mode
type LocalNodePortProxy struct {
	ctx      context.Context
	logger   klog.Logger
	listenIP string
	family   string
	clock    clock.Clock

	// mu guards active against concurrent Sync/Shutdown calls.
	mu     sync.Mutex
	active map[string]*nodePortListener
}

// NewLocalNodePortProxy creates a new proxy for the given IP family.
func NewLocalNodePortProxy(ctx context.Context, ipFamily v1.IPFamily) *LocalNodePortProxy {
	return newLocalNodePortProxy(ctx, ipFamily, clock.RealClock{})
}

func newLocalNodePortProxy(ctx context.Context, ipFamily v1.IPFamily, clock clock.Clock) *LocalNodePortProxy {
	listenIP := "127.0.0.1"
	if ipFamily == v1.IPv6Protocol {
		listenIP = "::1"
	}
	return &LocalNodePortProxy{
		ctx:      ctx,
		logger:   klog.FromContext(ctx),
		listenIP: listenIP,
		family:   string(ipFamily),
		clock:    clock,
		active:   make(map[string]*nodePortListener),
	}
}

// SyncNodePorts reconciles the set of active localhost NodePort proxies with
// the desired state.
func (p *LocalNodePortProxy) SyncNodePorts(desired []NodePortSpec) {
	byKey := make(map[string]*NodePortSpec, len(desired))
	for i := range desired {
		spec := &desired[i]
		if len(spec.Endpoints) == 0 {
			continue
		}
		byKey[spec.nodePortKey()] = spec
	}

	p.mu.Lock()
	defer p.mu.Unlock()

	// Remove stale listeners
	for key, l := range p.active {
		if _, ok := byKey[key]; !ok {
			p.logger.V(2).Info("Removing localhost nodeport proxy", "protocol", l.protocol, "port", l.port)
			l.shutdown()
			delete(p.active, key)
		}
	}

	// Add or update
	for key, spec := range byKey {
		if existing, ok := p.active[key]; ok {
			existing.update(spec)
			continue
		}
		if spec.Protocol != v1.ProtocolTCP {
			p.logger.Error(nil, "Unexpected non-TCP localhost nodeport proxy",
				"service", spec.ServicePortName, "protocol", spec.Protocol, "port", spec.NodePort)
			continue
		}
		l, err := p.newNodePortListener(spec)
		if err != nil {
			p.logger.Error(err, "Failed to create localhost nodeport proxy", "protocol", spec.Protocol, "port", spec.NodePort)
			metrics.LocalhostNodePortListenerCreationFailuresTotal.WithLabelValues(p.family).Inc()
			continue
		}
		p.active[key] = l
		p.logger.V(4).Info("Created localhost nodeport proxy", "protocol", spec.Protocol, "port", spec.NodePort, "endpoints", len(spec.Endpoints))
	}

	metrics.LocalhostNodePortListeners.WithLabelValues(p.family).Set(float64(len(p.active)))
}

// nodePortKey identifies this NodePort's listener in the active map.
func (s *NodePortSpec) nodePortKey() string {
	return fmt.Sprintf("%s/%d", s.Protocol, s.NodePort)
}

// Shutdown tears down all active listeners and closes any in-flight connections.
func (p *LocalNodePortProxy) Shutdown() {
	p.mu.Lock()
	defer p.mu.Unlock()

	for key, l := range p.active {
		l.shutdown()
		delete(p.active, key)
	}
	metrics.LocalhostNodePortListeners.WithLabelValues(p.family).Set(0)
}

func (p *LocalNodePortProxy) newNodePortListener(spec *NodePortSpec) (*nodePortListener, error) {
	addr := net.JoinHostPort(p.listenIP, fmt.Sprintf("%d", spec.NodePort))
	listener, err := net.Listen("tcp", addr)
	if err != nil {
		return nil, fmt.Errorf("failed to listen on %s: %w", addr, err)
	}

	ctx, cancel := context.WithCancel(p.ctx)
	l := &nodePortListener{
		protocol:        spec.Protocol,
		port:            spec.NodePort,
		logger:          p.logger,
		endpoints:       spec.Endpoints,
		affinityTimeout: spec.affinityTimeout(),
		listener:        listener,
		cancel:          cancel,
		clock:           p.clock,
	}
	go l.acceptLoop(ctx)
	return l, nil
}

type nodePortListener struct {
	// Immutable after construction.
	protocol v1.Protocol
	port     int
	logger   klog.Logger
	listener net.Listener
	cancel   context.CancelFunc
	clock    clock.Clock

	// mu guards the endpoint-selection state below
	mu        sync.Mutex
	endpoints []string
	// affinityTimeout is 0 when SessionAffinity is disabled; otherwise, it is
	// the duration for which a picked endpoint stays pinned for localhost
	// traffic. Since the source IP for all localhost traffic is 127.0.0.1 or
	// ::1, ClientIP affinity effectively pins all traffic through this
	// listener to a single endpoint until the pin goes stale.
	affinityTimeout time.Duration
	pinnedEndpoint  string // "ip:port" of the currently pinned endpoint, empty if none
	pinnedLastUsed  time.Time
}

func (l *nodePortListener) acceptLoop(ctx context.Context) {
	for {
		conn, err := l.listener.Accept()
		if err != nil {
			if ctx.Err() != nil || errors.Is(err, net.ErrClosed) {
				return
			}
			l.logger.V(4).Info("Accept error on localhost nodeport proxy", "err", err, "protocol", l.protocol, "port", l.port)
			continue
		}
		go l.handleTCPConn(ctx, conn)
	}
}

func (l *nodePortListener) handleTCPConn(ctx context.Context, clientConn net.Conn) {
	defer clientConn.Close() //nolint:errcheck

	ep := l.pickEndpoint()
	backendConn, err := net.DialTimeout("tcp", ep, dialTimeout)
	if err != nil {
		l.logger.V(4).Info("Failed to connect to backend", "err", err, "protocol", l.protocol, "port", l.port, "endpoint", ep)
		return
	}
	defer backendConn.Close() //nolint:errcheck

	// Force both sides closed on listener shutdown so io.Copy returns and
	// the handler goroutine doesn't leak on a long-lived/idle connection.
	connCtx, cancel := context.WithCancel(ctx)
	defer cancel()
	go func() {
		<-connCtx.Done()
		_ = clientConn.Close()
		_ = backendConn.Close()
	}()

	pipe := func(dst, src net.Conn, direction string) {
		_, err := io.Copy(dst, src)
		if err != nil && connCtx.Err() == nil {
			l.logger.V(4).Info("Error proxying localhost nodeport connection",
				"err", err, "protocol", l.protocol, "port", l.port, "direction", direction)
		}
		if tc, ok := dst.(*net.TCPConn); ok {
			_ = tc.CloseWrite()
		}
	}

	var wg sync.WaitGroup
	wg.Go(func() { pipe(backendConn, clientConn, "client->backend") })
	wg.Go(func() { pipe(clientConn, backendConn, "backend->client") })
	wg.Wait()
}

func (l *nodePortListener) pickEndpoint() string {
	l.mu.Lock()
	defer l.mu.Unlock()

	if len(l.endpoints) == 0 {
		return ""
	}
	// Session affinity is implemented as a sliding window (renewing the expiration
	// timer on every connection). Note that since all localhost traffic shares the
	// same source IP (127.0.0.1 or ::1), any active connection from localhost will
	// keep this pin alive indefinitely for all local clients. This matches the
	// behavior in standard iptables/nftables modes.
	now := l.clock.Now()
	if l.affinityTimeout > 0 && l.pinnedEndpoint != "" && now.Sub(l.pinnedLastUsed) <= l.affinityTimeout {
		for _, ep := range l.endpoints {
			if ep == l.pinnedEndpoint {
				l.pinnedLastUsed = now
				return ep
			}
		}
		// Pinned endpoint no longer in the set; fall through and pick a new one.
	}
	ep := l.endpoints[rand.IntN(len(l.endpoints))]
	if l.affinityTimeout > 0 {
		l.pinnedEndpoint = ep
		l.pinnedLastUsed = now
	}
	return ep
}

func (l *nodePortListener) update(spec *NodePortSpec) {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.endpoints = spec.Endpoints
	newTimeout := spec.affinityTimeout()
	if l.affinityTimeout != newTimeout {
		// Affinity config changed; drop any stale pin.
		l.pinnedEndpoint = ""
		l.affinityTimeout = newTimeout
	}
}

func (l *nodePortListener) shutdown() {
	l.cancel()
	_ = l.listener.Close()
}