integration/network: Fix flaky TestInspectNetwork/AfterLeaderChange

SwarmKit's Raft election timeout is 10 seconds (ElectionTick=10 ×
TickInterval=1 s). RestartNode (Stop+Start in sequence) can complete
in under 10 s on a fast machine, so followers may never detect a
leader gap: the restarted node reconnects before their election timer
fires and resumes leadership unchanged. This is the race that makes
the subtest flaky.

Fix by separating the stop and start steps:

1. Stop the leader and do NOT restart it immediately.
2. Poll a standby manager's API (standbyCli) until a different leader
   is elected. Using a non-leader manager avoids depending on the
   stopped node's API, which is unavailable during this window.
3. Start the stopped node back up. The new leader has advanced the
   Raft term, so the rejoining node is forced to become a follower.
4. Wait for c1 to be responsive again (HasLeader via c1) before
   asserting and running the inspect checks.

With the leader fully stopped (not immediately restarted), the
remaining two managers must elect a new leader - they have quorum
and the original node cannot reconnect to reset their election timer.
The election fires within the 10 s timeout, well inside NetworkPoll's
30 s ceiling (50 s on arm). The three-retry loop is removed; the
approach is deterministic.

Add HasLeaderOtherThan to integration/internal/swarm/states.go. Errors
from NodeList are treated as poll.Continue rather than poll.Error so a
brief period of cluster unavailability during the election does not
abort the wait.

Signed-off-by: Paweł Gronowski <pawel.gronowski@docker.com>
This commit is contained in:
Paweł Gronowski
2026-07-06 14:36:37 +02:00
parent 4cf23202e7
commit 5c7bcb64ff
2 changed files with 51 additions and 11 deletions

View File

@@ -157,6 +157,7 @@ func JobComplete(ctx context.Context, apiClient client.TaskAPIClient, service sw
}
}
// HasLeader polls until any manager node reports itself as the leader.
func HasLeader(ctx context.Context, apiClient client.NodeAPIClient) func(log poll.LogT) poll.Result {
return func(log poll.LogT) poll.Result {
result, err := apiClient.NodeList(ctx, client.NodeListOptions{
@@ -173,3 +174,29 @@ func HasLeader(ctx context.Context, apiClient client.NodeAPIClient) func(log pol
return poll.Continue("no leader elected yet")
}
}
// HasLeaderOtherThan polls until a manager node other than excludedNodeID
// reports itself as the leader.
// Pass a client connected to a node that is NOT excludedNodeID, so that
// the poll keeps working while excludedNodeID's API is unavailable.
// Errors from NodeList are treated as poll.Continue rather than poll.Error
// so a brief period of cluster unavailability during the election does not
// abort the wait.
func HasLeaderOtherThan(ctx context.Context, apiClient client.NodeAPIClient, excludedNodeID string) func(log poll.LogT) poll.Result {
return func(log poll.LogT) poll.Result {
result, err := apiClient.NodeList(ctx, client.NodeListOptions{
Filters: make(client.Filters).Add("role", "manager"),
})
if err != nil {
// The standby manager itself may be briefly unavailable during
// the election; keep retrying rather than failing the poll.
return poll.Continue("waiting for node list: %v", err)
}
for _, node := range result.Items {
if node.ManagerStatus != nil && node.ManagerStatus.Leader && node.ID != excludedNodeID {
return poll.Success()
}
}
return poll.Continue("no leader other than %s elected yet", excludedNodeID)
}
}

View File

@@ -181,23 +181,36 @@ func TestInspectNetwork(t *testing.T) {
t.Run("AfterLeaderChange", func(t *testing.T) {
oldLeader := leaderID()
var leader *daemon.Daemon
var leader, standby *daemon.Daemon
for _, d := range mgr {
if d.NodeID() == oldLeader {
leader = d
break
} else if standby == nil {
standby = d
}
}
assert.Assert(t, leader != nil)
// Force a leader change
for range 3 {
leader.RestartNode(t)
poll.WaitOn(t, swarm.HasLeader(ctx, c1), swarm.NetworkPoll)
if leaderID() != oldLeader {
break
}
t.Log("Restarting the node did not trigger a leader change")
}
assert.Assert(t, standby != nil)
// standbyCli connects to a non-leader manager so NodeList queries keep
// working while the stopped leader's API is unavailable.
standbyCli := standby.NewClientT(t)
defer standbyCli.Close()
// SwarmKit's Raft election timeout is 10 s (ElectionTick=10 ×
// TickInterval=1 s). RestartNode (Stop+Start together) completes in
// under 10 s on fast machines, so followers may never detect a gap and
// the original node resumes leadership. Instead, stop the
// leader and leave it stopped until a new leader is confirmed. The
// new leader advances the Raft term, so the restarted node is forced
// to rejoin as a follower.
leader.Stop(t)
poll.WaitOn(t, swarm.HasLeaderOtherThan(ctx, standbyCli, oldLeader), swarm.NetworkPoll)
leader.StartNode(t)
// Wait for c1 to be responsive and the cluster to be fully settled
// before asserting and running the inspect checks.
poll.WaitOn(t, swarm.HasLeader(ctx, c1), swarm.NetworkPoll)
assert.Assert(t, leaderID() != oldLeader, "leader did not change")
checkNetworkInspect(t)