diff --git a/integration/internal/swarm/states.go b/integration/internal/swarm/states.go index a9209cb3a7..b382d41c3d 100644 --- a/integration/internal/swarm/states.go +++ b/integration/internal/swarm/states.go @@ -157,6 +157,7 @@ func JobComplete(ctx context.Context, apiClient client.TaskAPIClient, service sw } } +// HasLeader polls until any manager node reports itself as the leader. func HasLeader(ctx context.Context, apiClient client.NodeAPIClient) func(log poll.LogT) poll.Result { return func(log poll.LogT) poll.Result { result, err := apiClient.NodeList(ctx, client.NodeListOptions{ @@ -173,3 +174,29 @@ func HasLeader(ctx context.Context, apiClient client.NodeAPIClient) func(log pol return poll.Continue("no leader elected yet") } } + +// HasLeaderOtherThan polls until a manager node other than excludedNodeID +// reports itself as the leader. +// Pass a client connected to a node that is NOT excludedNodeID, so that +// the poll keeps working while excludedNodeID's API is unavailable. +// Errors from NodeList are treated as poll.Continue rather than poll.Error +// so a brief period of cluster unavailability during the election does not +// abort the wait. +func HasLeaderOtherThan(ctx context.Context, apiClient client.NodeAPIClient, excludedNodeID string) func(log poll.LogT) poll.Result { + return func(log poll.LogT) poll.Result { + result, err := apiClient.NodeList(ctx, client.NodeListOptions{ + Filters: make(client.Filters).Add("role", "manager"), + }) + if err != nil { + // The standby manager itself may be briefly unavailable during + // the election; keep retrying rather than failing the poll. + return poll.Continue("waiting for node list: %v", err) + } + for _, node := range result.Items { + if node.ManagerStatus != nil && node.ManagerStatus.Leader && node.ID != excludedNodeID { + return poll.Success() + } + } + return poll.Continue("no leader other than %s elected yet", excludedNodeID) + } +} diff --git a/integration/network/inspect_test.go b/integration/network/inspect_test.go index 38e8b73bba..cc3133772e 100644 --- a/integration/network/inspect_test.go +++ b/integration/network/inspect_test.go @@ -24,12 +24,14 @@ func TestInspectNetwork(t *testing.T) { var mgr [3]*daemon.Daemon mgr[0] = swarm.NewSwarm(ctx, t, testEnv, daemon.WithSwarmListenAddr("127.0.0.2")) + defer mgr[0].Cleanup(t) defer mgr[0].Stop(t) for i := range mgr { if i != 0 { mgr[i] = daemon.New(t, daemon.WithSwarmListenAddr("127.0.0."+strconv.Itoa(i+2))) mgr[i].StartAndSwarmJoin(ctx, t, mgr[0], true) + defer mgr[i].Cleanup(t) defer mgr[i].Stop(t) } t.Logf("Daemon %s is Swarm Node %s", mgr[i].ID(), mgr[i].NodeID()) @@ -40,6 +42,7 @@ func TestInspectNetwork(t *testing.T) { worker1 := daemon.New(t, daemon.WithSwarmListenAddr("127.0.0."+strconv.Itoa(len(mgr)+2))) worker1.StartAndSwarmJoin(ctx, t, mgr[0], false) + defer worker1.Cleanup(t) defer worker1.Stop(t) t.Logf("Daemon %s is Swarm Node %s", worker1.ID(), worker1.NodeID()) @@ -181,23 +184,36 @@ func TestInspectNetwork(t *testing.T) { t.Run("AfterLeaderChange", func(t *testing.T) { oldLeader := leaderID() - var leader *daemon.Daemon + var leader, standby *daemon.Daemon for _, d := range mgr { if d.NodeID() == oldLeader { leader = d - break + } else if standby == nil { + standby = d } } assert.Assert(t, leader != nil) - // Force a leader change - for range 3 { - leader.RestartNode(t) - poll.WaitOn(t, swarm.HasLeader(ctx, c1), swarm.NetworkPoll) - if leaderID() != oldLeader { - break - } - t.Log("Restarting the node did not trigger a leader change") - } + assert.Assert(t, standby != nil) + + // standbyCli connects to a non-leader manager so NodeList queries keep + // working while the stopped leader's API is unavailable. + standbyCli := standby.NewClientT(t) + defer standbyCli.Close() + + // SwarmKit's Raft election timeout is 10 s (ElectionTick=10 × + // TickInterval=1 s). RestartNode (Stop+Start together) completes in + // under 10 s on fast machines, so followers may never detect a gap and + // the original node resumes leadership. Instead, stop the + // leader and leave it stopped until a new leader is confirmed. The + // new leader advances the Raft term, so the restarted node is forced + // to rejoin as a follower. + leader.Stop(t) + poll.WaitOn(t, swarm.HasLeaderOtherThan(ctx, standbyCli, oldLeader), swarm.NetworkPoll) + leader.StartNode(t) + // Wait for c1 to be responsive and the cluster to be fully settled + // before asserting and running the inspect checks. + poll.WaitOn(t, swarm.HasLeader(ctx, c1), swarm.NetworkPoll) + assert.Assert(t, leaderID() != oldLeader, "leader did not change") checkNetworkInspect(t)