diff --git a/docs/channels/troubleshooting.md b/docs/channels/troubleshooting.md index 92051a17f003..b8ad3eba86c5 100644 --- a/docs/channels/troubleshooting.md +++ b/docs/channels/troubleshooting.md @@ -161,8 +161,9 @@ If the gateway process is healthy but a channel stays stopped after repeated unclean boots, the [crash-loop breaker](/gateway/restart-recovery#safety-valves-and-observability) may be suppressing channel auto-start. Use `openclaw gateway call channels.start --params '{"channel":""}'` to -override, or wait for the unclean-boot window to drain and then restart the -gateway. +override immediately, or leave the healthy gateway running. After the full +unclean-boot window drains, the same process rechecks the breaker and resumes +deferred channel auto-start. ## Related diff --git a/docs/gateway/restart-recovery.md b/docs/gateway/restart-recovery.md index 7089f63dd858..96e63e679149 100644 --- a/docs/gateway/restart-recovery.md +++ b/docs/gateway/restart-recovery.md @@ -182,14 +182,15 @@ restart handling continues. - **Crash-loop breaker:** 3 unclean boots within 5 minutes trip a breaker that suppresses auto-start side services on the next boot, so a crashing gateway - does not amplify itself. A later boot recovers once the unclean-boot window - drains. + does not amplify itself. A continuously stable safe-mode gateway rechecks the + breaker after the full unclean-boot window drains and then resumes deferred + channel auto-start without requiring another gateway restart. When the breaker is tripped, the **control plane still starts**, but channel - plugins (and other auto-started side services) stay down for the current boot - unless an operator manually overrides the suppression. Automatic startup - resumes on a later boot after the unclean-boot window drains. Gateway logs - look like: + plugins (and other auto-started side services) stay down until an operator + manually overrides the suppression or the full window drains with no unclean + boots. Recovery preserves channels that an operator manually stopped and any + separate development-mode suppression. Gateway logs look like: `channel autostart suppressed by crash-loop breaker; refusing automatic start for … Start a channel manually with: openclaw gateway call channels.start --params '{"channel":""}'` @@ -214,8 +215,12 @@ channels.start --params '{"channel":""}'` `channels.start` is a **manual** override; it does not disable the breaker for other channels. - 5. Or wait for the unclean-boot window to drain, then restart the gateway. - The next boot logs whether channel auto-start is restored. + 5. Or leave the healthy gateway running until the full unclean-boot window + drains. The same process logs that the restart-loop breaker recovered and + starts the deferred configured channels. + If that message does not appear after the window plus one health-monitor + interval, inspect the gateway logs and run `openclaw doctor` before + restarting. See also [Gateway](/gateway) (safe mode paragraph) for the same control-plane vs channel-autostart split. diff --git a/src/cli/gateway-cli/run.option-collisions.test.ts b/src/cli/gateway-cli/run.option-collisions.test.ts index fad5c30b1e20..667965641759 100644 --- a/src/cli/gateway-cli/run.option-collisions.test.ts +++ b/src/cli/gateway-cli/run.option-collisions.test.ts @@ -123,6 +123,10 @@ const bootLifecycle = vi.hoisted(() => ({ record: vi.fn( (_env?: NodeJS.ProcessEnv, _nowMs?: number, _reason?: string): string | undefined => "boot-id", ), + recover: vi.fn( + (_bootId?: string, _env?: NodeJS.ProcessEnv, _nowMs?: number): string | undefined => + "recovered-boot-id", + ), complete: vi.fn(), })); const netState = vi.hoisted(() => ({ @@ -317,6 +321,8 @@ vi.mock("../../infra/gateway-boot-lifecycle.js", () => ({ bootLifecycle.inspect(env, nowMs), recordGatewayBootStart: (env?: NodeJS.ProcessEnv, nowMs?: number, reason?: string) => bootLifecycle.record(env, nowMs, reason), + recordGatewayCrashLoopRecovery: (bootId?: string, env?: NodeJS.ProcessEnv, nowMs?: number) => + bootLifecycle.recover(bootId, env, nowMs), completeGatewayBootLifecycle: (bootId: string | undefined, completion: unknown) => bootLifecycle.complete(bootId, completion), })); @@ -409,6 +415,7 @@ describe("gateway run option collisions", () => { bootLifecycle.decisions.length = 0; bootLifecycle.inspect.mockClear(); bootLifecycle.record.mockClear(); + bootLifecycle.recover.mockClear(); bootLifecycle.complete.mockClear(); startGatewayServer.mockClear(); setGatewayWsLogStyle.mockClear(); @@ -467,6 +474,7 @@ describe("gateway run option collisions", () => { auth?: { mode?: string; token?: string; password?: string }; bind?: string; channelAutostartSuppression?: { reason?: string; message?: string }; + tryRecoverChannelAutostartSuppression?: () => boolean; ambientEnvTriggers?: "allow" | "suppress"; startupConfigSnapshotRead?: { snapshot?: Record }; startupStartedAt?: number; @@ -1594,6 +1602,55 @@ describe("gateway run option collisions", () => { expect(gatewayLogMessages.some((message) => message.includes("breaker recovered"))).toBe(true); }); + it("recovers channel autostart only after the full breaker window drains", async () => { + runGatewayLoop.mockImplementationOnce( + async ({ + beginBoot, + start, + }: { + beginBoot?: (startedAtMs: number) => Promise | void; + start: GatewayLoopStart; + }) => { + await beginBoot?.(1000); + await start({ startupStartedAt: 1000 }); + }, + ); + bootLifecycle.decisions.push({ + tripped: true, + uncleanBoots: 3, + windowMs: 300_000, + shouldWriteStabilityBundle: false, + recovered: false, + }); + + await runGatewayCli(["gateway", "run", "--allow-unconfigured"]); + + const recover = gatewayStartOptions().tryRecoverChannelAutostartSuppression; + expect(recover).toBeTypeOf("function"); + bootLifecycle.decisions.push( + { + tripped: false, + uncleanBoots: 1, + windowMs: 300_000, + shouldWriteStabilityBundle: false, + recovered: true, + }, + { + tripped: false, + uncleanBoots: 0, + windowMs: 300_000, + shouldWriteStabilityBundle: false, + recovered: true, + }, + ); + + expect(recover?.()).toBe(false); + expect(bootLifecycle.recover).not.toHaveBeenCalled(); + expect(recover?.()).toBe(true); + expect(bootLifecycle.recover).toHaveBeenCalledWith("boot-id", process.env, undefined); + expect(gatewayLogMessages.some((message) => message.includes("breaker recovered"))).toBe(true); + }); + it("skips failure bundles but exits nonzero for unconfirmed gateway lock conflicts", async () => { const port = await getFreePort(); configState.snapshot = { diff --git a/src/cli/gateway-cli/run.ts b/src/cli/gateway-cli/run.ts index 3f039de99ea6..1028761026c6 100644 --- a/src/cli/gateway-cli/run.ts +++ b/src/cli/gateway-cli/run.ts @@ -43,6 +43,7 @@ import { GATEWAY_CRASH_LOOP_RECOVERED_REASON, inspectGatewayCrashLoopBreaker, recordGatewayBootStart, + recordGatewayCrashLoopRecovery, type GatewayCrashLoopBreakerDecision, type GatewayBootLifecycleCompletion, } from "../../infra/gateway-boot-lifecycle.js"; @@ -1136,6 +1137,22 @@ async function runGatewayCommandOnce(opts: GatewayRunOpts, hooks: GatewayRunRunt let crashLoopDecision: GatewayCrashLoopBreakerDecision | undefined; let channelAutostartSuppression: { reason: "crash-loop-breaker"; message: string } | undefined; let activeBootId: string | undefined; + const tryRecoverChannelAutostartSuppression = () => { + const decision = inspectGatewayCrashLoopBreaker(process.env); + // The current safe-mode boot remains an open row until the full window has + // drained. Requiring zero prevents a near-expiry history from restoring + // channels before this process itself has proven stable for the whole window. + if (!decision.recovered || decision.uncleanBoots !== 0) { + return false; + } + const recoveredBootId = recordGatewayCrashLoopRecovery(activeBootId, process.env); + if (!recoveredBootId) { + return false; + } + activeBootId = recoveredBootId; + gatewayLog.info("gateway restart-loop breaker recovered; channel auto-start restored"); + return true; + }; const beginBoot = async (startedAtMs: number) => { // run-loop calls beginBoot before every startGatewayServer invocation, so // in-process restarts re-evaluate breaker state instead of reusing stale mode. @@ -1194,6 +1211,7 @@ async function runGatewayCommandOnce(opts: GatewayRunOpts, hooks: GatewayRunRunt : {}), ...(envSidecarStartupMode !== "start" ? { sidecarStartup: envSidecarStartupMode } : {}), ...(channelAutostartSuppression ? { channelAutostartSuppression } : {}), + ...(channelAutostartSuppression ? { tryRecoverChannelAutostartSuppression } : {}), ...(devMode ? { ambientEnvTriggers: devAmbientEnvTriggers, diff --git a/src/gateway/channel-health-monitor.test.ts b/src/gateway/channel-health-monitor.test.ts index e04dc62483ca..c590f7bcb0f4 100644 --- a/src/gateway/channel-health-monitor.test.ts +++ b/src/gateway/channel-health-monitor.test.ts @@ -17,6 +17,7 @@ function createMockChannelManager(overrides?: Partial): ChannelM stopChannel: vi.fn(async () => {}), setAutostartSuppression: vi.fn(), getAutostartSuppression: vi.fn(() => null), + recoverAutostartSuppression: vi.fn(async () => false), setAmbientAutostartSuppressedChannelIds: vi.fn(), isAmbientAutostartSuppressed: vi.fn(() => false), markChannelLoggedOut: vi.fn(), @@ -283,7 +284,12 @@ describe("channel-health-monitor", () => { it("treats crash-loop suppressed accounts as expected stopped", async () => { let suppressed = true; + let allowRecovery = false; const suppression = { reason: "crash-loop-breaker" as const, message: "safe mode" }; + const recoverAutostartSuppression = vi.fn(async () => { + suppressed = !allowRecovery; + return allowRecovery; + }); const manager = createSnapshotManager( { discord: { @@ -292,6 +298,7 @@ describe("channel-health-monitor", () => { }, { getAutostartSuppression: vi.fn(() => (suppressed ? suppression : null)), + recoverAutostartSuppression, }, ); const monitor = startDefaultMonitor(manager, { @@ -305,9 +312,10 @@ describe("channel-health-monitor", () => { expect(manager.resetRestartAttempts).not.toHaveBeenCalled(); expect(manager.startChannel).not.toHaveBeenCalled(); - suppressed = false; + allowRecovery = true; await vi.advanceTimersByTimeAsync(101); + expect(recoverAutostartSuppression).toHaveBeenCalled(); expect(manager.resetRestartAttempts).toHaveBeenCalledWith("discord", "default"); expect(manager.startChannel).toHaveBeenCalledWith("discord", "default"); monitor.stop(); diff --git a/src/gateway/channel-health-monitor.ts b/src/gateway/channel-health-monitor.ts index 3088001c5e8a..0c0ebc901c98 100644 --- a/src/gateway/channel-health-monitor.ts +++ b/src/gateway/channel-health-monitor.ts @@ -100,6 +100,9 @@ export function startChannelHealthMonitor(deps: ChannelHealthMonitorDeps): Chann return; } + if (channelManager.getAutostartSuppression() !== null) { + await channelManager.recoverAutostartSuppression(); + } const snapshot = channelManager.getRuntimeSnapshot(); const globalAutostartSuppression = channelManager.getAutostartSuppression(); diff --git a/src/gateway/server-channels.test.ts b/src/gateway/server-channels.test.ts index e11466936184..61f76572e654 100644 --- a/src/gateway/server-channels.test.ts +++ b/src/gateway/server-channels.test.ts @@ -253,6 +253,7 @@ function createManager(options?: { deferStartupAccountStartsUntil?: Promise; fillChannelDependencies?: boolean; ambientAutostartSuppressedChannelIds?: ReadonlySet; + tryRecoverAutostartSuppression?: () => boolean; getNativeApprovalRuntime?: () => GatewayNativeApprovalRuntime | undefined; }) { const log = createSubsystemLogger("gateway/server-channels-test"); @@ -281,6 +282,9 @@ function createManager(options?: { ...(options?.ambientAutostartSuppressedChannelIds ? { ambientAutostartSuppressedChannelIds: options.ambientAutostartSuppressedChannelIds } : {}), + ...(options?.tryRecoverAutostartSuppression + ? { tryRecoverAutostartSuppression: options.tryRecoverAutostartSuppression } + : {}), ...(options?.getNativeApprovalRuntime ? { getNativeApprovalRuntime: options.getNativeApprovalRuntime } : {}), @@ -1731,6 +1735,83 @@ describe("server-channels auto restart", () => { expect(manager.getAutostartSuppression()?.reason).toBe("crash-loop-breaker"); }); + it("recovers suppressed autostart without undoing manual stops", async () => { + const startAccount = vi.fn( + async ({ abortSignal }: ChannelGatewayContext) => + await new Promise((resolve) => { + abortSignal.addEventListener("abort", () => resolve(), { once: true }); + }), + ); + installTestRegistry( + createTestPlugin({ + startAccount, + listAccountIds: () => [DEFAULT_ACCOUNT_ID, "work"], + }), + ); + const tryRecover = vi.fn(() => true); + const manager = createManager({ + tryRecoverAutostartSuppression: tryRecover, + getRuntimeConfig: () => ({ + channels: { discord: { healthMonitor: { enabled: false } } }, + }), + }); + manager.setAutostartSuppression({ + reason: "crash-loop-breaker", + message: "safe mode", + }); + + await manager.startChannels(); + await manager.startChannel("discord", DEFAULT_ACCOUNT_ID, { manual: true }); + await manager.stopChannel("discord", DEFAULT_ACCOUNT_ID); + await manager.recoverAutostartSuppression(); + await flushMicrotasks(); + + expect(tryRecover).toHaveBeenCalledOnce(); + expect(manager.getAutostartSuppression()).toBeNull(); + expect(startAccount.mock.calls.map(([ctx]) => ctx.accountId)).toEqual([ + DEFAULT_ACCOUNT_ID, + "work", + ]); + expect(manager.isHealthMonitorEnabled("discord", "work")).toBe(false); + expect(manager.isManuallyStopped("discord", DEFAULT_ACCOUNT_ID)).toBe(true); + }); + + it("keeps suppression when persisted recovery is not proven", async () => { + const startAccount = vi.fn(async () => {}); + installTestRegistry(createTestPlugin({ startAccount })); + const manager = createManager({ tryRecoverAutostartSuppression: () => false }); + manager.setAutostartSuppression({ + reason: "crash-loop-breaker", + message: "safe mode", + }); + + await expect(manager.recoverAutostartSuppression()).resolves.toBe(false); + + expect(manager.getAutostartSuppression()?.reason).toBe("crash-loop-breaker"); + expect(startAccount).not.toHaveBeenCalled(); + }); + + it("keeps ambient channel suppression after crash-loop recovery", async () => { + const startAccount = vi.fn(async () => {}); + installTestRegistry(createTestPlugin({ startAccount })); + const manager = createManager({ + ambientAutostartSuppressedChannelIds: new Set(["discord"]), + tryRecoverAutostartSuppression: () => true, + }); + manager.setAutostartSuppression({ + reason: "crash-loop-breaker", + message: "safe mode", + }); + + await expect(manager.recoverAutostartSuppression()).resolves.toBe(true); + + expect(manager.getAutostartSuppression()).toBeNull(); + expect(startAccount).not.toHaveBeenCalled(); + expect(manager.getRuntimeSnapshot().channelAccounts.discord?.default?.lastError).toBe( + "ambient channel credentials suppressed for dev gateway", + ); + }); + it("suppresses ambient dev channel autostart while allowing manual starts", async () => { const startAccount = vi.fn(async (_ctx: ChannelGatewayContext) => {}); installTestRegistry(createTestPlugin({ startAccount })); diff --git a/src/gateway/server-channels.ts b/src/gateway/server-channels.ts index 38c2d0416cfd..1b09b6d22847 100644 --- a/src/gateway/server-channels.ts +++ b/src/gateway/server-channels.ts @@ -221,6 +221,7 @@ type ChannelManagerOptions = { deferStartupAccountStartsUntil?: Promise; getNativeApprovalRuntime?: () => GatewayNativeApprovalRuntime | undefined; ambientAutostartSuppressedChannelIds?: ReadonlySet; + tryRecoverAutostartSuppression?: () => boolean; }; type StopChannelOptions = { @@ -259,6 +260,7 @@ export type ChannelManager = { stopChannel: (channel: ChannelId, accountId?: string, opts?: StopChannelOptions) => Promise; setAutostartSuppression: (suppression: ChannelAutostartSuppression | null) => void; getAutostartSuppression: () => ChannelAutostartSuppression | null; + recoverAutostartSuppression: () => Promise; setAmbientAutostartSuppressedChannelIds: (channelIds: ReadonlySet) => void; isAmbientAutostartSuppressed: (channelId: string) => boolean; markChannelLoggedOut: (channelId: ChannelId, cleared: boolean, accountId?: string) => void; @@ -1179,7 +1181,7 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage } }; - const startChannels = async () => { + const startChannelsWithOptions = async (startOptions: StartChannelOptions = {}) => { let releaseAccountStarts: (() => void) | undefined; const deferAccountStartUntil = opts.deferStartupAccountStartsUntil ?? @@ -1197,11 +1199,10 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage tasks: [...listChannelPlugins()].map((plugin) => async () => { try { await measureStartup(`channels.${plugin.id}.start`, () => - startChannelInternal( - plugin.id, - undefined, - deferAccountStartUntil ? { deferAccountStartUntil } : {}, - ), + startChannelInternal(plugin.id, undefined, { + ...startOptions, + ...(deferAccountStartUntil ? { deferAccountStartUntil } : {}), + }), ); } catch (err) { ensureChannelLog(plugin.id).error?.( @@ -1215,6 +1216,19 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage } }; + const startChannels = async () => await startChannelsWithOptions(); + + const recoverAutostartSuppression = async (): Promise => { + if (!autostartSuppression || !opts.tryRecoverAutostartSuppression?.()) { + return false; + } + autostartSuppression = null; + // Recovery resumes the autostart attempt that safe mode deferred. Preserve + // explicit operator stops while still covering health-monitor opt-outs. + await startChannelsWithOptions({ preserveManualStop: true }); + return true; + }; + const markChannelLoggedOut = (channelId: ChannelId, cleared: boolean, accountId?: string) => { const plugin = getChannelPlugin(channelId); if (!plugin) { @@ -1305,6 +1319,7 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage autostartSuppression = suppression; }, getAutostartSuppression: () => autostartSuppression, + recoverAutostartSuppression, setAmbientAutostartSuppressedChannelIds: (channelIds) => { ambientAutostartSuppressedChannelIds = new Set(channelIds); }, diff --git a/src/gateway/server-public.ts b/src/gateway/server-public.ts index 0a6be774e3d0..249468aec26d 100644 --- a/src/gateway/server-public.ts +++ b/src/gateway/server-public.ts @@ -54,6 +54,8 @@ export type GatewayServerOptions = { channelWizardRunner?: import("./server-methods/wizard.js").ChannelSetupWizardRunner; sidecarStartup?: GatewaySidecarStartupMode; channelAutostartSuppression?: ChannelAutostartSuppression; + /** Internal lifecycle callback that re-proves and records crash-loop recovery. */ + tryRecoverChannelAutostartSuppression?: () => boolean; ambientEnvTriggers?: AmbientEnvTriggerPolicy; /** Optional startup timestamp used for concise readiness logging. */ startupStartedAt?: number; diff --git a/src/gateway/server-runtime-state-prepare.ts b/src/gateway/server-runtime-state-prepare.ts index 4b9294ddb119..9ab109ad1a09 100644 --- a/src/gateway/server-runtime-state-prepare.ts +++ b/src/gateway/server-runtime-state-prepare.ts @@ -323,6 +323,9 @@ export async function prepareGatewayRuntimeState(params: { deferStartupAccountStartsUntil: startupAccountStartsReady, getNativeApprovalRuntime: () => gatewayInstanceRuntimeRef.current?.nativeApprovals, ambientAutostartSuppressedChannelIds, + ...(opts.tryRecoverChannelAutostartSuppression + ? { tryRecoverAutostartSuppression: opts.tryRecoverChannelAutostartSuppression } + : {}), }); channelManager.setAutostartSuppression(opts.channelAutostartSuppression ?? null); const sidecarStartup = opts.sidecarStartup ?? "start"; diff --git a/src/gateway/server/readiness.test.ts b/src/gateway/server/readiness.test.ts index b327d9788b2a..e15e525fc396 100644 --- a/src/gateway/server/readiness.test.ts +++ b/src/gateway/server/readiness.test.ts @@ -35,6 +35,7 @@ function createManager(snapshot: ChannelRuntimeSnapshot): ChannelManager { stopChannel: vi.fn(), setAutostartSuppression: vi.fn(), getAutostartSuppression: vi.fn(() => null), + recoverAutostartSuppression: vi.fn(async () => false), setAmbientAutostartSuppressedChannelIds: vi.fn(), isAmbientAutostartSuppressed: vi.fn(() => false), markChannelLoggedOut: vi.fn(), diff --git a/src/infra/gateway-boot-lifecycle.test.ts b/src/infra/gateway-boot-lifecycle.test.ts index 7610768f642c..809b41f8a3e1 100644 --- a/src/infra/gateway-boot-lifecycle.test.ts +++ b/src/infra/gateway-boot-lifecycle.test.ts @@ -21,6 +21,7 @@ import { formatGatewayCrashLoopManualChannelStartHint, inspectGatewayCrashLoopBreaker, recordGatewayBootStart, + recordGatewayCrashLoopRecovery, repairGatewayAgentMediaMigrationStartupFailures, } from "./gateway-boot-lifecycle.js"; import { executeSqliteQuerySync, getNodeSqliteKysely } from "./kysely-sync.js"; @@ -178,6 +179,59 @@ describe("gateway crash-loop breaker", () => { expect(secondDecision).toMatchObject({ tripped: false, recovered: false }); }); + it("records a fresh lifecycle segment before recovered channel startup", () => { + const db = createLifecycleDb(); + const nowMs = 1_000_000; + const safeModeBootId = recordGatewayBootStart( + db.env, + nowMs - GATEWAY_BOOT_LOOP_WINDOW_MS - 1, + GATEWAY_CRASH_LOOP_BREAKER_REASON, + ); + + expect(inspectGatewayCrashLoopBreaker(db.env, nowMs)).toMatchObject({ + recovered: true, + uncleanBoots: 0, + }); + + const recoveredBootId = recordGatewayCrashLoopRecovery(safeModeBootId, db.env, nowMs); + + expect(recoveredBootId).toBeDefined(); + expect( + executeSqliteQuerySync( + db.db, + db.kysely + .selectFrom("gateway_boot_lifecycle") + .select(["boot_id", "completed_at_ms", "outcome", "startup_reason"]) + .orderBy("started_at_ms"), + ).rows, + ).toEqual([ + { + boot_id: safeModeBootId, + completed_at_ms: nowMs, + outcome: "safe_mode_stable", + startup_reason: GATEWAY_CRASH_LOOP_BREAKER_REASON, + }, + { + boot_id: recoveredBootId, + completed_at_ms: null, + outcome: null, + startup_reason: GATEWAY_CRASH_LOOP_RECOVERED_REASON, + }, + ]); + expect(inspectGatewayCrashLoopBreaker(db.env, nowMs + 1)).toMatchObject({ + recovered: false, + uncleanBoots: 1, + }); + insertBootRows(db, [ + { bootId: "post-recovery-crash-a", startedAtMs: nowMs + 2 }, + { bootId: "post-recovery-crash-b", startedAtMs: nowMs + 3 }, + ]); + expect(inspectGatewayCrashLoopBreaker(db.env, nowMs + 4)).toMatchObject({ + tripped: true, + uncleanBoots: GATEWAY_BOOT_LOOP_UNCLEAN_THRESHOLD, + }); + }); + it("records forced stops without tripping the breaker", () => { const db = createLifecycleDb(); const nowMs = 1_000_000; diff --git a/src/infra/gateway-boot-lifecycle.ts b/src/infra/gateway-boot-lifecycle.ts index 5781c45e6198..92ce03579cc1 100644 --- a/src/infra/gateway-boot-lifecycle.ts +++ b/src/infra/gateway-boot-lifecycle.ts @@ -23,17 +23,16 @@ import { // can inspect a stable process instead of a flap. const GATEWAY_BOOT_LOOP_UNCLEAN_THRESHOLD = 3; const GATEWAY_BOOT_LOOP_WINDOW_MS = 5 * 60_000; -// Keep enough history for operator forensics while bounding one-row-per-boot +// Keep enough history for operator forensics while bounding lifecycle-segment // growth. Retention must comfortably exceed GATEWAY_BOOT_LOOP_WINDOW_MS. const GATEWAY_BOOT_LIFECYCLE_RETENTION_MS = 24 * 60 * 60_000; export const GATEWAY_BOOT_REASON_MAX_UTF16_CODE_UNITS = 500; export const GATEWAY_CRASH_LOOP_BREAKER_REASON = "gateway.crash_loop_breaker"; export const GATEWAY_CRASH_LOOP_RECOVERED_REASON = "gateway.crash_loop_recovered"; /** - * The breaker never self-clears within its window, so every operator-facing surface must name the - * manual override command instead of the internal RPC name. Account-scoped suppression must carry - * its accountId: `channels.start` resolves an omitted account to the channel default, so a hint - * without it would start a different account than the one the message named. + * The breaker only self-clears after the full window drains. Operator surfaces name the manual + * override command, not the internal RPC. Account hints carry accountId to avoid starting a + * different default account than the warning named. */ export function formatGatewayCrashLoopManualChannelStartHint(target?: { channelId: string; @@ -55,6 +54,7 @@ type GatewayBootLifecycleDatabase = Pick { + const kysely = getNodeSqliteKysely(db); + if (bootId) { + executeSqliteQuerySync( + db, + kysely + .updateTable("gateway_boot_lifecycle") + .set({ + completed_at_ms: nowMs, + outcome: "safe_mode_stable", + reason: null, + }) + .where("boot_id", "=", bootId), + ); + } + executeSqliteQuerySync( + db, + kysely.insertInto("gateway_boot_lifecycle").values({ + boot_id: recoveredBootId, + pid: process.pid, + started_at_ms: nowMs, + completed_at_ms: null, + outcome: null, + startup_reason: GATEWAY_CRASH_LOOP_RECOVERED_REASON, + reason: null, + }), + ); + }, + { env }, + ); + return recoveredBootId; + } catch (err) { + gatewayLifecycleLog.warn( + `failed to persist gateway crash-loop recovery; fail-safe: ${String(err)}`, + ); + return undefined; + } +} + export function completeGatewayBootLifecycle( bootId: string | undefined, completion: GatewayBootLifecycleCompletion,