mirror of
https://github.com/moby/moby.git
synced 2026-08-09 01:21:37 +00:00
integration/network: Fix flaky TestInspectNetwork/AfterLeaderChange
SwarmKit's Raft election timeout is 10 seconds (ElectionTick=10 × TickInterval=1 s). RestartNode (Stop+Start in sequence) can complete in under 10 s on a fast machine, so followers may never detect a leader gap: the restarted node reconnects before their election timer fires and resumes leadership unchanged. This is the race that makes the subtest flaky. Fix by separating the stop and start steps: 1. Stop the leader and do NOT restart it immediately. 2. Poll a standby manager's API (standbyCli) until a different leader is elected. Using a non-leader manager avoids depending on the stopped node's API, which is unavailable during this window. 3. Start the stopped node back up. The new leader has advanced the Raft term, so the rejoining node is forced to become a follower. 4. Wait for c1 to be responsive again (HasLeader via c1) before asserting and running the inspect checks. With the leader fully stopped (not immediately restarted), the remaining two managers must elect a new leader - they have quorum and the original node cannot reconnect to reset their election timer. The election fires within the 10 s timeout, well inside NetworkPoll's 30 s ceiling (50 s on arm). The three-retry loop is removed; the approach is deterministic. Add HasLeaderOtherThan to integration/internal/swarm/states.go. Errors from NodeList are treated as poll.Continue rather than poll.Error so a brief period of cluster unavailability during the election does not abort the wait. Signed-off-by: Paweł Gronowski <pawel.gronowski@docker.com>
This commit is contained in:
@@ -157,6 +157,7 @@ func JobComplete(ctx context.Context, apiClient client.TaskAPIClient, service sw
|
||||
}
|
||||
}
|
||||
|
||||
// HasLeader polls until any manager node reports itself as the leader.
|
||||
func HasLeader(ctx context.Context, apiClient client.NodeAPIClient) func(log poll.LogT) poll.Result {
|
||||
return func(log poll.LogT) poll.Result {
|
||||
result, err := apiClient.NodeList(ctx, client.NodeListOptions{
|
||||
@@ -173,3 +174,29 @@ func HasLeader(ctx context.Context, apiClient client.NodeAPIClient) func(log pol
|
||||
return poll.Continue("no leader elected yet")
|
||||
}
|
||||
}
|
||||
|
||||
// HasLeaderOtherThan polls until a manager node other than excludedNodeID
|
||||
// reports itself as the leader.
|
||||
// Pass a client connected to a node that is NOT excludedNodeID, so that
|
||||
// the poll keeps working while excludedNodeID's API is unavailable.
|
||||
// Errors from NodeList are treated as poll.Continue rather than poll.Error
|
||||
// so a brief period of cluster unavailability during the election does not
|
||||
// abort the wait.
|
||||
func HasLeaderOtherThan(ctx context.Context, apiClient client.NodeAPIClient, excludedNodeID string) func(log poll.LogT) poll.Result {
|
||||
return func(log poll.LogT) poll.Result {
|
||||
result, err := apiClient.NodeList(ctx, client.NodeListOptions{
|
||||
Filters: make(client.Filters).Add("role", "manager"),
|
||||
})
|
||||
if err != nil {
|
||||
// The standby manager itself may be briefly unavailable during
|
||||
// the election; keep retrying rather than failing the poll.
|
||||
return poll.Continue("waiting for node list: %v", err)
|
||||
}
|
||||
for _, node := range result.Items {
|
||||
if node.ManagerStatus != nil && node.ManagerStatus.Leader && node.ID != excludedNodeID {
|
||||
return poll.Success()
|
||||
}
|
||||
}
|
||||
return poll.Continue("no leader other than %s elected yet", excludedNodeID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,23 +181,36 @@ func TestInspectNetwork(t *testing.T) {
|
||||
|
||||
t.Run("AfterLeaderChange", func(t *testing.T) {
|
||||
oldLeader := leaderID()
|
||||
var leader *daemon.Daemon
|
||||
var leader, standby *daemon.Daemon
|
||||
for _, d := range mgr {
|
||||
if d.NodeID() == oldLeader {
|
||||
leader = d
|
||||
break
|
||||
} else if standby == nil {
|
||||
standby = d
|
||||
}
|
||||
}
|
||||
assert.Assert(t, leader != nil)
|
||||
// Force a leader change
|
||||
for range 3 {
|
||||
leader.RestartNode(t)
|
||||
poll.WaitOn(t, swarm.HasLeader(ctx, c1), swarm.NetworkPoll)
|
||||
if leaderID() != oldLeader {
|
||||
break
|
||||
}
|
||||
t.Log("Restarting the node did not trigger a leader change")
|
||||
}
|
||||
assert.Assert(t, standby != nil)
|
||||
|
||||
// standbyCli connects to a non-leader manager so NodeList queries keep
|
||||
// working while the stopped leader's API is unavailable.
|
||||
standbyCli := standby.NewClientT(t)
|
||||
defer standbyCli.Close()
|
||||
|
||||
// SwarmKit's Raft election timeout is 10 s (ElectionTick=10 ×
|
||||
// TickInterval=1 s). RestartNode (Stop+Start together) completes in
|
||||
// under 10 s on fast machines, so followers may never detect a gap and
|
||||
// the original node resumes leadership. Instead, stop the
|
||||
// leader and leave it stopped until a new leader is confirmed. The
|
||||
// new leader advances the Raft term, so the restarted node is forced
|
||||
// to rejoin as a follower.
|
||||
leader.Stop(t)
|
||||
poll.WaitOn(t, swarm.HasLeaderOtherThan(ctx, standbyCli, oldLeader), swarm.NetworkPoll)
|
||||
leader.StartNode(t)
|
||||
// Wait for c1 to be responsive and the cluster to be fully settled
|
||||
// before asserting and running the inspect checks.
|
||||
poll.WaitOn(t, swarm.HasLeader(ctx, c1), swarm.NetworkPoll)
|
||||
|
||||
assert.Assert(t, leaderID() != oldLeader, "leader did not change")
|
||||
|
||||
checkNetworkInspect(t)
|
||||
|
||||
Reference in New Issue
Block a user