Merge pull request #53017 from vvoland/flaky-inspectnetwork

integration/network: Fix flaky TestInspectNetwork/AfterLeaderChange

integration/internal/swarm/states.go

@@ -157,6 +157,7 @@ func JobComplete(ctx context.Context, apiClient client.TaskAPIClient, service sw
 	}
 }
 
+// HasLeader polls until any manager node reports itself as the leader.
 func HasLeader(ctx context.Context, apiClient client.NodeAPIClient) func(log poll.LogT) poll.Result {
 	return func(log poll.LogT) poll.Result {
 		result, err := apiClient.NodeList(ctx, client.NodeListOptions{
@@ -173,3 +174,29 @@ func HasLeader(ctx context.Context, apiClient client.NodeAPIClient) func(log pol
 		return poll.Continue("no leader elected yet")
 	}
 }
+
+// HasLeaderOtherThan polls until a manager node other than excludedNodeID
+// reports itself as the leader.
+// Pass a client connected to a node that is NOT excludedNodeID, so that
+// the poll keeps working while excludedNodeID's API is unavailable.
+// Errors from NodeList are treated as poll.Continue rather than poll.Error
+// so a brief period of cluster unavailability during the election does not
+// abort the wait.
+func HasLeaderOtherThan(ctx context.Context, apiClient client.NodeAPIClient, excludedNodeID string) func(log poll.LogT) poll.Result {
+	return func(log poll.LogT) poll.Result {
+		result, err := apiClient.NodeList(ctx, client.NodeListOptions{
+			Filters: make(client.Filters).Add("role", "manager"),
+		})
+		if err != nil {
+			// The standby manager itself may be briefly unavailable during
+			// the election; keep retrying rather than failing the poll.
+			return poll.Continue("waiting for node list: %v", err)
+		}
+		for _, node := range result.Items {
+			if node.ManagerStatus != nil && node.ManagerStatus.Leader && node.ID != excludedNodeID {
+				return poll.Success()
+			}
+		}
+		return poll.Continue("no leader other than %s elected yet", excludedNodeID)
+	}
+}

integration/network/inspect_test.go

@@ -24,12 +24,14 @@ func TestInspectNetwork(t *testing.T) {
 
 	var mgr [3]*daemon.Daemon
 	mgr[0] = swarm.NewSwarm(ctx, t, testEnv, daemon.WithSwarmListenAddr("127.0.0.2"))
+	defer mgr[0].Cleanup(t)
 	defer mgr[0].Stop(t)
 
 	for i := range mgr {
 		if i != 0 {
 			mgr[i] = daemon.New(t, daemon.WithSwarmListenAddr("127.0.0."+strconv.Itoa(i+2)))
 			mgr[i].StartAndSwarmJoin(ctx, t, mgr[0], true)
+			defer mgr[i].Cleanup(t)
 			defer mgr[i].Stop(t)
 		}
 		t.Logf("Daemon %s is Swarm Node %s", mgr[i].ID(), mgr[i].NodeID())
@@ -40,6 +42,7 @@ func TestInspectNetwork(t *testing.T) {
 
 	worker1 := daemon.New(t, daemon.WithSwarmListenAddr("127.0.0."+strconv.Itoa(len(mgr)+2)))
 	worker1.StartAndSwarmJoin(ctx, t, mgr[0], false)
+	defer worker1.Cleanup(t)
 	defer worker1.Stop(t)
 	t.Logf("Daemon %s is Swarm Node %s", worker1.ID(), worker1.NodeID())
 
@@ -181,23 +184,36 @@ func TestInspectNetwork(t *testing.T) {
 
 	t.Run("AfterLeaderChange", func(t *testing.T) {
 		oldLeader := leaderID()
-		var leader *daemon.Daemon
+		var leader, standby *daemon.Daemon
 		for _, d := range mgr {
 			if d.NodeID() == oldLeader {
 				leader = d
-				break
+			} else if standby == nil {
+				standby = d
 			}
 		}
 		assert.Assert(t, leader != nil)
-		// Force a leader change
-		for range 3 {
-			leader.RestartNode(t)
-			poll.WaitOn(t, swarm.HasLeader(ctx, c1), swarm.NetworkPoll)
-			if leaderID() != oldLeader {
-				break
-			}
-			t.Log("Restarting the node did not trigger a leader change")
-		}
+		assert.Assert(t, standby != nil)
+
+		// standbyCli connects to a non-leader manager so NodeList queries keep
+		// working while the stopped leader's API is unavailable.
+		standbyCli := standby.NewClientT(t)
+		defer standbyCli.Close()
+
+		// SwarmKit's Raft election timeout is 10 s (ElectionTick=10 ×
+		// TickInterval=1 s). RestartNode (Stop+Start together) completes in
+		// under 10 s on fast machines, so followers may never detect a gap and
+		// the original node resumes leadership. Instead, stop the
+		// leader and leave it stopped until a new leader is confirmed. The
+		// new leader advances the Raft term, so the restarted node is forced
+		// to rejoin as a follower.
+		leader.Stop(t)
+		poll.WaitOn(t, swarm.HasLeaderOtherThan(ctx, standbyCli, oldLeader), swarm.NetworkPoll)
+		leader.StartNode(t)
+		// Wait for c1 to be responsive and the cluster to be fully settled
+		// before asserting and running the inspect checks.
+		poll.WaitOn(t, swarm.HasLeader(ctx, c1), swarm.NetworkPoll)
+
 		assert.Assert(t, leaderID() != oldLeader, "leader did not change")
 
 		checkNetworkInspect(t)