fix(webchat): finalize provider failure lifecycle (#91895)

* fix(webchat): finalize provider failure lifecycle

* chore: narrow fallback failure lifecycle marker

Signed-off-by: sallyom <somalley@redhat.com>

---------

Signed-off-by: sallyom <somalley@redhat.com>
Co-authored-by: sallyom <somalley@redhat.com>

src/auto-reply/reply/agent-runner-execution.test.ts

@@ -5198,6 +5198,8 @@ describe("runAgentTurnWithFallback", () => {
   });
 
   it("uses compact generic copy for raw external chat errors when verbose is off", async () => {
+    const agentEvents = await import("../../infra/agent-events.js");
+    const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent);
     state.runEmbeddedAgentMock.mockRejectedValueOnce(
       new Error("INVALID_ARGUMENT: some other failure"),
     );
@@ -5210,7 +5212,7 @@ describe("runAgentTurnWithFallback", () => {
         Provider: "whatsapp",
         MessageSid: "msg",
       } as unknown as TemplateContext,
-      opts: {},
+      opts: { runId: "run-provider-failure" } as GetReplyOptions,
       typingSignals: createMockTypingSignaler(),
       blockReplyPipeline: null,
       blockStreamingEnabled: false,
@@ -5230,6 +5232,21 @@ describe("runAgentTurnWithFallback", () => {
     if (result.kind === "final") {
       expect(result.payload.text).toBe(GENERIC_RUN_FAILURE_TEXT);
     }
+    const terminalFailureEvent = emitAgentEvent.mock.calls
+      .map((call) => call[0])
+      .find((event) => {
+        if (!event || typeof event !== "object") {
+          return false;
+        }
+        const data = (event as { data?: Record<string, unknown> }).data;
+        return (
+          (event as { runId?: unknown }).runId === "run-provider-failure" &&
+          (event as { stream?: unknown }).stream === "lifecycle" &&
+          data?.phase === "error" &&
+          data.fallbackExhaustedFailure === true
+        );
+      });
+    expect(terminalFailureEvent).toBeDefined();
   });
 
   it("uses heartbeat failure copy for raw external errors during heartbeat runs", async () => {

src/auto-reply/reply/agent-runner-execution.ts

@@ -2932,6 +2932,17 @@ export async function runAgentTurnWithFallback(params: {
         cfg: params.followupRun.run.config,
       });
 
+      emitAgentEvent({
+        runId,
+        ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}),
+        stream: "lifecycle",
+        data: {
+          phase: "error",
+          error: message,
+          endedAt: Date.now(),
+          fallbackExhaustedFailure: true,
+        },
+      });
       params.replyOperation?.fail("run_failed", err);
       return {
         kind: "final",

src/gateway/server-chat.agent-events.test.ts

@@ -2641,6 +2641,47 @@ describe("agent event handler", () => {
     expect(agentRunSeq.has("run-terminal-error")).toBe(false);
   });
 
+  it("finalizes fallback-exhausted lifecycle errors without waiting for retry grace", () => {
+    vi.useFakeTimers();
+    const { broadcast, clearAgentRunContext, agentRunSeq, handler } = createHarness({
+      resolveSessionKeyForRun: () => "session-terminal-error",
+      lifecycleErrorRetryGraceMs: 100,
+    });
+    registerAgentRunContext("run-terminal-final-failure", {
+      sessionKey: "session-terminal-error",
+    });
+
+    handler({
+      runId: "run-terminal-final-failure",
+      seq: 1,
+      stream: "lifecycle",
+      ts: Date.now(),
+      data: {
+        phase: "error",
+        error: "LLM request failed: network connection error.",
+        fallbackExhaustedFailure: true,
+      },
+    });
+
+    const finalPayload = chatBroadcastCalls(broadcast).at(-1)?.[1] as {
+      state?: string;
+      runId?: string;
+      errorMessage?: string;
+    };
+    expect(finalPayload.state).toBe("error");
+    expect(finalPayload.runId).toBe("run-terminal-final-failure");
+    expect(finalPayload.errorMessage).toContain("network connection error");
+    expect(clearAgentRunContext).toHaveBeenCalledWith("run-terminal-final-failure");
+    expect(agentRunSeq.has("run-terminal-final-failure")).toBe(false);
+    expect(
+      persistGatewaySessionLifecycleEventMock.mock.calls.some(
+        ([params]) =>
+          (params as { event?: { data?: { fallbackExhaustedFailure?: boolean } } }).event?.data
+            ?.fallbackExhaustedFailure === true,
+      ),
+    ).toBe(true);
+  });
+
   it("keeps deferred lifecycle-error cleanup across later non-terminal events", () => {
     vi.useFakeTimers();
     const { broadcast, clearAgentRunContext, agentRunSeq, handler } = createHarness({

src/gateway/server-chat.ts

@@ -1232,7 +1232,11 @@ export function createAgentEventHandler({
     if (lifecyclePhase === "error") {
       clearBufferedChatState(clientRunId);
       const skipChatErrorFinal = isChatSendRunActive(evt.runId) && !chatLink;
-      if (isAborted || lifecycleErrorRetryGraceMs <= 0) {
+      const isFallbackExhaustedFailure = evt.data?.fallbackExhaustedFailure === true;
+      // Per-attempt provider errors keep the retry grace so fallback can reuse
+      // the runId. Once the runner marks fallback as exhausted, clear chat state
+      // immediately so webchat sessions do not stay in progress until the timer.
+      if (isAborted || isFallbackExhaustedFailure || lifecycleErrorRetryGraceMs <= 0) {
         finalizeLifecycleEvent(evt, { skipChatErrorFinal });
       } else {
         scheduleTerminalLifecycleError(evt, { skipChatErrorFinal });

test/scripts/check-deadcode-unused-files.test.ts

@@ -210,7 +210,9 @@ src/a.ts: src/a.ts
 
     await resultPromise;
 
-    expect(calls[0]).toMatchObject({
+    const call = calls[0] as { command: string };
+    expect(path.basename(call.command)).toBe("pnpm");
+    expect(call).toMatchObject({
       args: [
         "--config.minimum-release-age=0",
         "dlx",
@@ -226,7 +228,6 @@ src/a.ts: src/a.ts
         "--files",
         "--no-config-hints",
       ],
-      command: "pnpm",
       options: {
         detached: process.platform !== "win32",
         shell: false,