From 33a3e05683100ef6a5e9fbf62dcf4c561ba212fd Mon Sep 17 00:00:00 2001 From: Andy Ye <35905412+TurboTheTurtle@users.noreply.github.com> Date: Wed, 10 Jun 2026 09:33:21 -0700 Subject: [PATCH] fix(webchat): finalize provider failure lifecycle (#91895) * fix(webchat): finalize provider failure lifecycle * chore: narrow fallback failure lifecycle marker Signed-off-by: sallyom --------- Signed-off-by: sallyom Co-authored-by: sallyom --- .../reply/agent-runner-execution.test.ts | 19 ++++++++- .../reply/agent-runner-execution.ts | 11 +++++ src/gateway/server-chat.agent-events.test.ts | 41 +++++++++++++++++++ src/gateway/server-chat.ts | 6 ++- .../check-deadcode-unused-files.test.ts | 5 ++- 5 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/auto-reply/reply/agent-runner-execution.test.ts b/src/auto-reply/reply/agent-runner-execution.test.ts index e8334f45b181..37dc39d3ae78 100644 --- a/src/auto-reply/reply/agent-runner-execution.test.ts +++ b/src/auto-reply/reply/agent-runner-execution.test.ts @@ -5198,6 +5198,8 @@ describe("runAgentTurnWithFallback", () => { }); it("uses compact generic copy for raw external chat errors when verbose is off", async () => { + const agentEvents = await import("../../infra/agent-events.js"); + const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent); state.runEmbeddedAgentMock.mockRejectedValueOnce( new Error("INVALID_ARGUMENT: some other failure"), ); @@ -5210,7 +5212,7 @@ describe("runAgentTurnWithFallback", () => { Provider: "whatsapp", MessageSid: "msg", } as unknown as TemplateContext, - opts: {}, + opts: { runId: "run-provider-failure" } as GetReplyOptions, typingSignals: createMockTypingSignaler(), blockReplyPipeline: null, blockStreamingEnabled: false, @@ -5230,6 +5232,21 @@ describe("runAgentTurnWithFallback", () => { if (result.kind === "final") { expect(result.payload.text).toBe(GENERIC_RUN_FAILURE_TEXT); } + const terminalFailureEvent = emitAgentEvent.mock.calls + .map((call) => call[0]) + .find((event) => { + if (!event || typeof event !== "object") { + return false; + } + const data = (event as { data?: Record }).data; + return ( + (event as { runId?: unknown }).runId === "run-provider-failure" && + (event as { stream?: unknown }).stream === "lifecycle" && + data?.phase === "error" && + data.fallbackExhaustedFailure === true + ); + }); + expect(terminalFailureEvent).toBeDefined(); }); it("uses heartbeat failure copy for raw external errors during heartbeat runs", async () => { diff --git a/src/auto-reply/reply/agent-runner-execution.ts b/src/auto-reply/reply/agent-runner-execution.ts index d9858a8e061e..37d993404bd5 100644 --- a/src/auto-reply/reply/agent-runner-execution.ts +++ b/src/auto-reply/reply/agent-runner-execution.ts @@ -2932,6 +2932,17 @@ export async function runAgentTurnWithFallback(params: { cfg: params.followupRun.run.config, }); + emitAgentEvent({ + runId, + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + stream: "lifecycle", + data: { + phase: "error", + error: message, + endedAt: Date.now(), + fallbackExhaustedFailure: true, + }, + }); params.replyOperation?.fail("run_failed", err); return { kind: "final", diff --git a/src/gateway/server-chat.agent-events.test.ts b/src/gateway/server-chat.agent-events.test.ts index 64c1ace2946f..f2042df5d387 100644 --- a/src/gateway/server-chat.agent-events.test.ts +++ b/src/gateway/server-chat.agent-events.test.ts @@ -2641,6 +2641,47 @@ describe("agent event handler", () => { expect(agentRunSeq.has("run-terminal-error")).toBe(false); }); + it("finalizes fallback-exhausted lifecycle errors without waiting for retry grace", () => { + vi.useFakeTimers(); + const { broadcast, clearAgentRunContext, agentRunSeq, handler } = createHarness({ + resolveSessionKeyForRun: () => "session-terminal-error", + lifecycleErrorRetryGraceMs: 100, + }); + registerAgentRunContext("run-terminal-final-failure", { + sessionKey: "session-terminal-error", + }); + + handler({ + runId: "run-terminal-final-failure", + seq: 1, + stream: "lifecycle", + ts: Date.now(), + data: { + phase: "error", + error: "LLM request failed: network connection error.", + fallbackExhaustedFailure: true, + }, + }); + + const finalPayload = chatBroadcastCalls(broadcast).at(-1)?.[1] as { + state?: string; + runId?: string; + errorMessage?: string; + }; + expect(finalPayload.state).toBe("error"); + expect(finalPayload.runId).toBe("run-terminal-final-failure"); + expect(finalPayload.errorMessage).toContain("network connection error"); + expect(clearAgentRunContext).toHaveBeenCalledWith("run-terminal-final-failure"); + expect(agentRunSeq.has("run-terminal-final-failure")).toBe(false); + expect( + persistGatewaySessionLifecycleEventMock.mock.calls.some( + ([params]) => + (params as { event?: { data?: { fallbackExhaustedFailure?: boolean } } }).event?.data + ?.fallbackExhaustedFailure === true, + ), + ).toBe(true); + }); + it("keeps deferred lifecycle-error cleanup across later non-terminal events", () => { vi.useFakeTimers(); const { broadcast, clearAgentRunContext, agentRunSeq, handler } = createHarness({ diff --git a/src/gateway/server-chat.ts b/src/gateway/server-chat.ts index 18053ee0b13f..3efb289d4d07 100644 --- a/src/gateway/server-chat.ts +++ b/src/gateway/server-chat.ts @@ -1232,7 +1232,11 @@ export function createAgentEventHandler({ if (lifecyclePhase === "error") { clearBufferedChatState(clientRunId); const skipChatErrorFinal = isChatSendRunActive(evt.runId) && !chatLink; - if (isAborted || lifecycleErrorRetryGraceMs <= 0) { + const isFallbackExhaustedFailure = evt.data?.fallbackExhaustedFailure === true; + // Per-attempt provider errors keep the retry grace so fallback can reuse + // the runId. Once the runner marks fallback as exhausted, clear chat state + // immediately so webchat sessions do not stay in progress until the timer. + if (isAborted || isFallbackExhaustedFailure || lifecycleErrorRetryGraceMs <= 0) { finalizeLifecycleEvent(evt, { skipChatErrorFinal }); } else { scheduleTerminalLifecycleError(evt, { skipChatErrorFinal }); diff --git a/test/scripts/check-deadcode-unused-files.test.ts b/test/scripts/check-deadcode-unused-files.test.ts index f9afc9f5a05a..8c22658e2f67 100644 --- a/test/scripts/check-deadcode-unused-files.test.ts +++ b/test/scripts/check-deadcode-unused-files.test.ts @@ -210,7 +210,9 @@ src/a.ts: src/a.ts await resultPromise; - expect(calls[0]).toMatchObject({ + const call = calls[0] as { command: string }; + expect(path.basename(call.command)).toBe("pnpm"); + expect(call).toMatchObject({ args: [ "--config.minimum-release-age=0", "dlx", @@ -226,7 +228,6 @@ src/a.ts: src/a.ts "--files", "--no-config-hints", ], - command: "pnpm", options: { detached: process.platform !== "win32", shell: false,