From 5c7bcb64ff80d4f19f0e35bd5a4c603a29d352bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 6 Jul 2026 14:36:37 +0200 Subject: [PATCH 1/2] integration/network: Fix flaky TestInspectNetwork/AfterLeaderChange MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SwarmKit's Raft election timeout is 10 seconds (ElectionTick=10 × TickInterval=1 s). RestartNode (Stop+Start in sequence) can complete in under 10 s on a fast machine, so followers may never detect a leader gap: the restarted node reconnects before their election timer fires and resumes leadership unchanged. This is the race that makes the subtest flaky. Fix by separating the stop and start steps: 1. Stop the leader and do NOT restart it immediately. 2. Poll a standby manager's API (standbyCli) until a different leader is elected. Using a non-leader manager avoids depending on the stopped node's API, which is unavailable during this window. 3. Start the stopped node back up. The new leader has advanced the Raft term, so the rejoining node is forced to become a follower. 4. Wait for c1 to be responsive again (HasLeader via c1) before asserting and running the inspect checks. With the leader fully stopped (not immediately restarted), the remaining two managers must elect a new leader - they have quorum and the original node cannot reconnect to reset their election timer. The election fires within the 10 s timeout, well inside NetworkPoll's 30 s ceiling (50 s on arm). The three-retry loop is removed; the approach is deterministic. Add HasLeaderOtherThan to integration/internal/swarm/states.go. Errors from NodeList are treated as poll.Continue rather than poll.Error so a brief period of cluster unavailability during the election does not abort the wait. Signed-off-by: Paweł Gronowski --- integration/internal/swarm/states.go | 27 +++++++++++++++++++++ integration/network/inspect_test.go | 35 +++++++++++++++++++--------- 2 files changed, 51 insertions(+), 11 deletions(-) diff --git a/integration/internal/swarm/states.go b/integration/internal/swarm/states.go index a9209cb3a7..b382d41c3d 100644 --- a/integration/internal/swarm/states.go +++ b/integration/internal/swarm/states.go @@ -157,6 +157,7 @@ func JobComplete(ctx context.Context, apiClient client.TaskAPIClient, service sw } } +// HasLeader polls until any manager node reports itself as the leader. func HasLeader(ctx context.Context, apiClient client.NodeAPIClient) func(log poll.LogT) poll.Result { return func(log poll.LogT) poll.Result { result, err := apiClient.NodeList(ctx, client.NodeListOptions{ @@ -173,3 +174,29 @@ func HasLeader(ctx context.Context, apiClient client.NodeAPIClient) func(log pol return poll.Continue("no leader elected yet") } } + +// HasLeaderOtherThan polls until a manager node other than excludedNodeID +// reports itself as the leader. +// Pass a client connected to a node that is NOT excludedNodeID, so that +// the poll keeps working while excludedNodeID's API is unavailable. +// Errors from NodeList are treated as poll.Continue rather than poll.Error +// so a brief period of cluster unavailability during the election does not +// abort the wait. +func HasLeaderOtherThan(ctx context.Context, apiClient client.NodeAPIClient, excludedNodeID string) func(log poll.LogT) poll.Result { + return func(log poll.LogT) poll.Result { + result, err := apiClient.NodeList(ctx, client.NodeListOptions{ + Filters: make(client.Filters).Add("role", "manager"), + }) + if err != nil { + // The standby manager itself may be briefly unavailable during + // the election; keep retrying rather than failing the poll. + return poll.Continue("waiting for node list: %v", err) + } + for _, node := range result.Items { + if node.ManagerStatus != nil && node.ManagerStatus.Leader && node.ID != excludedNodeID { + return poll.Success() + } + } + return poll.Continue("no leader other than %s elected yet", excludedNodeID) + } +} diff --git a/integration/network/inspect_test.go b/integration/network/inspect_test.go index 38e8b73bba..7906f33ad2 100644 --- a/integration/network/inspect_test.go +++ b/integration/network/inspect_test.go @@ -181,23 +181,36 @@ func TestInspectNetwork(t *testing.T) { t.Run("AfterLeaderChange", func(t *testing.T) { oldLeader := leaderID() - var leader *daemon.Daemon + var leader, standby *daemon.Daemon for _, d := range mgr { if d.NodeID() == oldLeader { leader = d - break + } else if standby == nil { + standby = d } } assert.Assert(t, leader != nil) - // Force a leader change - for range 3 { - leader.RestartNode(t) - poll.WaitOn(t, swarm.HasLeader(ctx, c1), swarm.NetworkPoll) - if leaderID() != oldLeader { - break - } - t.Log("Restarting the node did not trigger a leader change") - } + assert.Assert(t, standby != nil) + + // standbyCli connects to a non-leader manager so NodeList queries keep + // working while the stopped leader's API is unavailable. + standbyCli := standby.NewClientT(t) + defer standbyCli.Close() + + // SwarmKit's Raft election timeout is 10 s (ElectionTick=10 × + // TickInterval=1 s). RestartNode (Stop+Start together) completes in + // under 10 s on fast machines, so followers may never detect a gap and + // the original node resumes leadership. Instead, stop the + // leader and leave it stopped until a new leader is confirmed. The + // new leader advances the Raft term, so the restarted node is forced + // to rejoin as a follower. + leader.Stop(t) + poll.WaitOn(t, swarm.HasLeaderOtherThan(ctx, standbyCli, oldLeader), swarm.NetworkPoll) + leader.StartNode(t) + // Wait for c1 to be responsive and the cluster to be fully settled + // before asserting and running the inspect checks. + poll.WaitOn(t, swarm.HasLeader(ctx, c1), swarm.NetworkPoll) + assert.Assert(t, leaderID() != oldLeader, "leader did not change") checkNetworkInspect(t) From 6e9f1965ed7055e0810ddfd7c369ce641dab3b40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 6 Jul 2026 17:42:32 +0200 Subject: [PATCH 2/2] integration/network: Clean up daemon storage in TestInspectNetwork MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestInspectNetwork creates 4 daemons (3 managers + 1 worker) per run but never called Cleanup, leaving overlay storage, Raft state and logs on disk. When the stress runner executes the test 10 times in a row with -test.count 10, the accumulated directories fill the disk: no space left on device: could not create daemon root ".../bundles/test-integration-flaky/TestInspectNetwork/d8488af0.../root" Defer Cleanup after Stop for every daemon. Defers run LIFO so Stop fires before Cleanup for each daemon, which is required because Cleanup unmounts overlay mounts the running daemon still holds. Signed-off-by: Paweł Gronowski --- integration/network/inspect_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration/network/inspect_test.go b/integration/network/inspect_test.go index 7906f33ad2..cc3133772e 100644 --- a/integration/network/inspect_test.go +++ b/integration/network/inspect_test.go @@ -24,12 +24,14 @@ func TestInspectNetwork(t *testing.T) { var mgr [3]*daemon.Daemon mgr[0] = swarm.NewSwarm(ctx, t, testEnv, daemon.WithSwarmListenAddr("127.0.0.2")) + defer mgr[0].Cleanup(t) defer mgr[0].Stop(t) for i := range mgr { if i != 0 { mgr[i] = daemon.New(t, daemon.WithSwarmListenAddr("127.0.0."+strconv.Itoa(i+2))) mgr[i].StartAndSwarmJoin(ctx, t, mgr[0], true) + defer mgr[i].Cleanup(t) defer mgr[i].Stop(t) } t.Logf("Daemon %s is Swarm Node %s", mgr[i].ID(), mgr[i].NodeID()) @@ -40,6 +42,7 @@ func TestInspectNetwork(t *testing.T) { worker1 := daemon.New(t, daemon.WithSwarmListenAddr("127.0.0."+strconv.Itoa(len(mgr)+2))) worker1.StartAndSwarmJoin(ctx, t, mgr[0], false) + defer worker1.Cleanup(t) defer worker1.Stop(t) t.Logf("Daemon %s is Swarm Node %s", worker1.ID(), worker1.NodeID())