dilim/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 22:17:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(gateway): recover channel autostart after crash loops (#118311)

Browse Source 
* fix(gateway): recover channel autostart after crash loops

* docs(gateway): clarify crash-loop recovery steps

* test(gateway): type crash-loop recovery context

* test(gateway): correct recovery mock context

* test(gateway): keep recovery monitor coverage compact

* chore: leave crash-loop note to release flow

---------

Co-authored-by: Peter Steinberger <steipete@mac-studio-sf2.local>
This commit is contained in:
Peter Steinberger
2026-08-02 18:01:39 -07:00
committed by GitHub
parent 899be356ac
commit ec7e5dd769
13 changed files with 322 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
docs/channels/troubleshooting.md
View File

@@ -161,8 +161,9 @@ If the gateway process is healthy but a channel stays stopped after repeated
unclean boots, the [crash-loop breaker](/gateway/restart-recovery#safety-valves-and-observability)
may be suppressing channel auto-start. Use
`openclaw gateway call channels.start --params '{"channel":"<id>"}'` to
override, or wait for the unclean-boot window to drain and then restart the
gateway.
override immediately, or leave the healthy gateway running. After the full
unclean-boot window drains, the same process rechecks the breaker and resumes
deferred channel auto-start.
## Related

21
docs/gateway/restart-recovery.md
View File

@@ -182,14 +182,15 @@ restart handling continues.
- **Crash-loop breaker:** 3 unclean boots within 5 minutes trip a breaker that
suppresses auto-start side services on the next boot, so a crashing gateway
does not amplify itself. A later boot recovers once the unclean-boot window
drains.
does not amplify itself. A continuously stable safe-mode gateway rechecks the
breaker after the full unclean-boot window drains and then resumes deferred
channel auto-start without requiring another gateway restart.
When the breaker is tripped, the **control plane still starts**, but channel
plugins (and other auto-started side services) stay down for the current boot
unless an operator manually overrides the suppression. Automatic startup
resumes on a later boot after the unclean-boot window drains. Gateway logs
look like:
plugins (and other auto-started side services) stay down until an operator
manually overrides the suppression or the full window drains with no unclean
boots. Recovery preserves channels that an operator manually stopped and any
separate development-mode suppression. Gateway logs look like:
`channel autostart suppressed by crash-loop breaker; refusing automatic
start for <channel>… Start a channel manually with: openclaw gateway call
channels.start --params '{"channel":"<id>"}'`
@@ -214,8 +215,12 @@ channels.start --params '{"channel":"<id>"}'`
`channels.start` is a **manual** override; it does not disable the
breaker for other channels.
5. Or wait for the unclean-boot window to drain, then restart the gateway.
The next boot logs whether channel auto-start is restored.
5. Or leave the healthy gateway running until the full unclean-boot window
drains. The same process logs that the restart-loop breaker recovered and
starts the deferred configured channels.
If that message does not appear after the window plus one health-monitor
interval, inspect the gateway logs and run `openclaw doctor` before
restarting.
See also [Gateway](/gateway) (safe mode paragraph) for the same control-plane
vs channel-autostart split.

57
src/cli/gateway-cli/run.option-collisions.test.ts
View File

@@ -123,6 +123,10 @@ const bootLifecycle = vi.hoisted(() => ({
record: vi.fn(
(_env?: NodeJS.ProcessEnv, _nowMs?: number, _reason?: string): string | undefined => "boot-id",
),
recover: vi.fn(
(_bootId?: string, _env?: NodeJS.ProcessEnv, _nowMs?: number): string | undefined =>
"recovered-boot-id",
),
complete: vi.fn(),
}));
const netState = vi.hoisted(() => ({
@@ -317,6 +321,8 @@ vi.mock("../../infra/gateway-boot-lifecycle.js", () => ({
bootLifecycle.inspect(env, nowMs),
recordGatewayBootStart: (env?: NodeJS.ProcessEnv, nowMs?: number, reason?: string) =>
bootLifecycle.record(env, nowMs, reason),
recordGatewayCrashLoopRecovery: (bootId?: string, env?: NodeJS.ProcessEnv, nowMs?: number) =>
bootLifecycle.recover(bootId, env, nowMs),
completeGatewayBootLifecycle: (bootId: string | undefined, completion: unknown) =>
bootLifecycle.complete(bootId, completion),
}));
@@ -409,6 +415,7 @@ describe("gateway run option collisions", () => {
bootLifecycle.decisions.length = 0;
bootLifecycle.inspect.mockClear();
bootLifecycle.record.mockClear();
bootLifecycle.recover.mockClear();
bootLifecycle.complete.mockClear();
startGatewayServer.mockClear();
setGatewayWsLogStyle.mockClear();
@@ -467,6 +474,7 @@ describe("gateway run option collisions", () => {
auth?: { mode?: string; token?: string; password?: string };
bind?: string;
channelAutostartSuppression?: { reason?: string; message?: string };
tryRecoverChannelAutostartSuppression?: () => boolean;
ambientEnvTriggers?: "allow" | "suppress";
startupConfigSnapshotRead?: { snapshot?: Record<string, unknown> };
startupStartedAt?: number;
@@ -1594,6 +1602,55 @@ describe("gateway run option collisions", () => {
expect(gatewayLogMessages.some((message) => message.includes("breaker recovered"))).toBe(true);
});
it("recovers channel autostart only after the full breaker window drains", async () => {
runGatewayLoop.mockImplementationOnce(
async ({
beginBoot,
start,
}: {
beginBoot?: (startedAtMs: number) => Promise<void> | void;
start: GatewayLoopStart;
}) => {
await beginBoot?.(1000);
await start({ startupStartedAt: 1000 });
},
);
bootLifecycle.decisions.push({
tripped: true,
uncleanBoots: 3,
windowMs: 300_000,
shouldWriteStabilityBundle: false,
recovered: false,
});
await runGatewayCli(["gateway", "run", "--allow-unconfigured"]);
const recover = gatewayStartOptions().tryRecoverChannelAutostartSuppression;
expect(recover).toBeTypeOf("function");
bootLifecycle.decisions.push(
{
tripped: false,
uncleanBoots: 1,
windowMs: 300_000,
shouldWriteStabilityBundle: false,
recovered: true,
},
{
tripped: false,
uncleanBoots: 0,
windowMs: 300_000,
shouldWriteStabilityBundle: false,
recovered: true,
},
);
expect(recover?.()).toBe(false);
expect(bootLifecycle.recover).not.toHaveBeenCalled();
expect(recover?.()).toBe(true);
expect(bootLifecycle.recover).toHaveBeenCalledWith("boot-id", process.env, undefined);
expect(gatewayLogMessages.some((message) => message.includes("breaker recovered"))).toBe(true);
});
it("skips failure bundles but exits nonzero for unconfirmed gateway lock conflicts", async () => {
const port = await getFreePort();
configState.snapshot = {

18
src/cli/gateway-cli/run.ts
View File

@@ -43,6 +43,7 @@ import {
GATEWAY_CRASH_LOOP_RECOVERED_REASON,
inspectGatewayCrashLoopBreaker,
recordGatewayBootStart,
recordGatewayCrashLoopRecovery,
type GatewayCrashLoopBreakerDecision,
type GatewayBootLifecycleCompletion,
} from "../../infra/gateway-boot-lifecycle.js";
@@ -1136,6 +1137,22 @@ async function runGatewayCommandOnce(opts: GatewayRunOpts, hooks: GatewayRunRunt
let crashLoopDecision: GatewayCrashLoopBreakerDecision | undefined;
let channelAutostartSuppression: { reason: "crash-loop-breaker"; message: string } | undefined;
let activeBootId: string | undefined;
const tryRecoverChannelAutostartSuppression = () => {
const decision = inspectGatewayCrashLoopBreaker(process.env);
// The current safe-mode boot remains an open row until the full window has
// drained. Requiring zero prevents a near-expiry history from restoring
// channels before this process itself has proven stable for the whole window.
if (!decision.recovered || decision.uncleanBoots !== 0) {
return false;
}
const recoveredBootId = recordGatewayCrashLoopRecovery(activeBootId, process.env);
if (!recoveredBootId) {
return false;
}
activeBootId = recoveredBootId;
gatewayLog.info("gateway restart-loop breaker recovered; channel auto-start restored");
return true;
};
const beginBoot = async (startedAtMs: number) => {
// run-loop calls beginBoot before every startGatewayServer invocation, so
// in-process restarts re-evaluate breaker state instead of reusing stale mode.
@@ -1194,6 +1211,7 @@ async function runGatewayCommandOnce(opts: GatewayRunOpts, hooks: GatewayRunRunt
: {}),
...(envSidecarStartupMode !== "start" ? { sidecarStartup: envSidecarStartupMode } : {}),
...(channelAutostartSuppression ? { channelAutostartSuppression } : {}),
...(channelAutostartSuppression ? { tryRecoverChannelAutostartSuppression } : {}),
...(devMode
? {
ambientEnvTriggers: devAmbientEnvTriggers,

10
src/gateway/channel-health-monitor.test.ts
View File

@@ -17,6 +17,7 @@ function createMockChannelManager(overrides?: Partial<ChannelManager>): ChannelM
stopChannel: vi.fn(async () => {}),
setAutostartSuppression: vi.fn(),
getAutostartSuppression: vi.fn(() => null),
recoverAutostartSuppression: vi.fn(async () => false),
setAmbientAutostartSuppressedChannelIds: vi.fn(),
isAmbientAutostartSuppressed: vi.fn(() => false),
markChannelLoggedOut: vi.fn(),
@@ -283,7 +284,12 @@ describe("channel-health-monitor", () => {
it("treats crash-loop suppressed accounts as expected stopped", async () => {
let suppressed = true;
let allowRecovery = false;
const suppression = { reason: "crash-loop-breaker" as const, message: "safe mode" };
const recoverAutostartSuppression = vi.fn(async () => {
suppressed = !allowRecovery;
return allowRecovery;
});
const manager = createSnapshotManager(
{
discord: {
@@ -292,6 +298,7 @@ describe("channel-health-monitor", () => {
},
{
getAutostartSuppression: vi.fn(() => (suppressed ? suppression : null)),
recoverAutostartSuppression,
},
);
const monitor = startDefaultMonitor(manager, {
@@ -305,9 +312,10 @@ describe("channel-health-monitor", () => {
expect(manager.resetRestartAttempts).not.toHaveBeenCalled();
expect(manager.startChannel).not.toHaveBeenCalled();
suppressed = false;
allowRecovery = true;
await vi.advanceTimersByTimeAsync(101);
expect(recoverAutostartSuppression).toHaveBeenCalled();
expect(manager.resetRestartAttempts).toHaveBeenCalledWith("discord", "default");
expect(manager.startChannel).toHaveBeenCalledWith("discord", "default");
monitor.stop();

3
src/gateway/channel-health-monitor.ts
View File

@@ -100,6 +100,9 @@ export function startChannelHealthMonitor(deps: ChannelHealthMonitorDeps): Chann
return;
}
if (channelManager.getAutostartSuppression() !== null) {
await channelManager.recoverAutostartSuppression();
}
const snapshot = channelManager.getRuntimeSnapshot();
const globalAutostartSuppression = channelManager.getAutostartSuppression();

81
src/gateway/server-channels.test.ts
View File

@@ -253,6 +253,7 @@ function createManager(options?: {
deferStartupAccountStartsUntil?: Promise<void>;
fillChannelDependencies?: boolean;
ambientAutostartSuppressedChannelIds?: ReadonlySet<string>;
tryRecoverAutostartSuppression?: () => boolean;
getNativeApprovalRuntime?: () => GatewayNativeApprovalRuntime | undefined;
}) {
const log = createSubsystemLogger("gateway/server-channels-test");
@@ -281,6 +282,9 @@ function createManager(options?: {
...(options?.ambientAutostartSuppressedChannelIds
? { ambientAutostartSuppressedChannelIds: options.ambientAutostartSuppressedChannelIds }
: {}),
...(options?.tryRecoverAutostartSuppression
? { tryRecoverAutostartSuppression: options.tryRecoverAutostartSuppression }
: {}),
...(options?.getNativeApprovalRuntime
? { getNativeApprovalRuntime: options.getNativeApprovalRuntime }
: {}),
@@ -1731,6 +1735,83 @@ describe("server-channels auto restart", () => {
expect(manager.getAutostartSuppression()?.reason).toBe("crash-loop-breaker");
});
it("recovers suppressed autostart without undoing manual stops", async () => {
const startAccount = vi.fn(
async ({ abortSignal }: ChannelGatewayContext<TestAccount>) =>
await new Promise<void>((resolve) => {
abortSignal.addEventListener("abort", () => resolve(), { once: true });
}),
);
installTestRegistry(
createTestPlugin({
startAccount,
listAccountIds: () => [DEFAULT_ACCOUNT_ID, "work"],
}),
);
const tryRecover = vi.fn(() => true);
const manager = createManager({
tryRecoverAutostartSuppression: tryRecover,
getRuntimeConfig: () => ({
channels: { discord: { healthMonitor: { enabled: false } } },
}),
});
manager.setAutostartSuppression({
reason: "crash-loop-breaker",
message: "safe mode",
});
await manager.startChannels();
await manager.startChannel("discord", DEFAULT_ACCOUNT_ID, { manual: true });
await manager.stopChannel("discord", DEFAULT_ACCOUNT_ID);
await manager.recoverAutostartSuppression();
await flushMicrotasks();
expect(tryRecover).toHaveBeenCalledOnce();
expect(manager.getAutostartSuppression()).toBeNull();
expect(startAccount.mock.calls.map(([ctx]) => ctx.accountId)).toEqual([
DEFAULT_ACCOUNT_ID,
"work",
]);
expect(manager.isHealthMonitorEnabled("discord", "work")).toBe(false);
expect(manager.isManuallyStopped("discord", DEFAULT_ACCOUNT_ID)).toBe(true);
});
it("keeps suppression when persisted recovery is not proven", async () => {
const startAccount = vi.fn(async () => {});
installTestRegistry(createTestPlugin({ startAccount }));
const manager = createManager({ tryRecoverAutostartSuppression: () => false });
manager.setAutostartSuppression({
reason: "crash-loop-breaker",
message: "safe mode",
});
await expect(manager.recoverAutostartSuppression()).resolves.toBe(false);
expect(manager.getAutostartSuppression()?.reason).toBe("crash-loop-breaker");
expect(startAccount).not.toHaveBeenCalled();
});
it("keeps ambient channel suppression after crash-loop recovery", async () => {
const startAccount = vi.fn(async () => {});
installTestRegistry(createTestPlugin({ startAccount }));
const manager = createManager({
ambientAutostartSuppressedChannelIds: new Set(["discord"]),
tryRecoverAutostartSuppression: () => true,
});
manager.setAutostartSuppression({
reason: "crash-loop-breaker",
message: "safe mode",
});
await expect(manager.recoverAutostartSuppression()).resolves.toBe(true);
expect(manager.getAutostartSuppression()).toBeNull();
expect(startAccount).not.toHaveBeenCalled();
expect(manager.getRuntimeSnapshot().channelAccounts.discord?.default?.lastError).toBe(
"ambient channel credentials suppressed for dev gateway",
);
});
it("suppresses ambient dev channel autostart while allowing manual starts", async () => {
const startAccount = vi.fn(async (_ctx: ChannelGatewayContext<TestAccount>) => {});
installTestRegistry(createTestPlugin({ startAccount }));

27
src/gateway/server-channels.ts
View File

@@ -221,6 +221,7 @@ type ChannelManagerOptions = {
deferStartupAccountStartsUntil?: Promise<void>;
getNativeApprovalRuntime?: () => GatewayNativeApprovalRuntime | undefined;
ambientAutostartSuppressedChannelIds?: ReadonlySet<string>;
tryRecoverAutostartSuppression?: () => boolean;
};
type StopChannelOptions = {
@@ -259,6 +260,7 @@ export type ChannelManager = {
stopChannel: (channel: ChannelId, accountId?: string, opts?: StopChannelOptions) => Promise<void>;
setAutostartSuppression: (suppression: ChannelAutostartSuppression | null) => void;
getAutostartSuppression: () => ChannelAutostartSuppression | null;
recoverAutostartSuppression: () => Promise<boolean>;
setAmbientAutostartSuppressedChannelIds: (channelIds: ReadonlySet<string>) => void;
isAmbientAutostartSuppressed: (channelId: string) => boolean;
markChannelLoggedOut: (channelId: ChannelId, cleared: boolean, accountId?: string) => void;
@@ -1179,7 +1181,7 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage
}
};
const startChannels = async () => {
const startChannelsWithOptions = async (startOptions: StartChannelOptions = {}) => {
let releaseAccountStarts: (() => void) | undefined;
const deferAccountStartUntil =
opts.deferStartupAccountStartsUntil ??
@@ -1197,11 +1199,10 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage
tasks: [...listChannelPlugins()].map((plugin) => async () => {
try {
await measureStartup(`channels.${plugin.id}.start`, () =>
startChannelInternal(
plugin.id,
undefined,
deferAccountStartUntil ? { deferAccountStartUntil } : {},
),
startChannelInternal(plugin.id, undefined, {
...startOptions,
...(deferAccountStartUntil ? { deferAccountStartUntil } : {}),
}),
);
} catch (err) {
ensureChannelLog(plugin.id).error?.(
@@ -1215,6 +1216,19 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage
}
};
const startChannels = async () => await startChannelsWithOptions();
const recoverAutostartSuppression = async (): Promise<boolean> => {
if (!autostartSuppression || !opts.tryRecoverAutostartSuppression?.()) {
return false;
}
autostartSuppression = null;
// Recovery resumes the autostart attempt that safe mode deferred. Preserve
// explicit operator stops while still covering health-monitor opt-outs.
await startChannelsWithOptions({ preserveManualStop: true });
return true;
};
const markChannelLoggedOut = (channelId: ChannelId, cleared: boolean, accountId?: string) => {
const plugin = getChannelPlugin(channelId);
if (!plugin) {
@@ -1305,6 +1319,7 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage
autostartSuppression = suppression;
},
getAutostartSuppression: () => autostartSuppression,
recoverAutostartSuppression,
setAmbientAutostartSuppressedChannelIds: (channelIds) => {
ambientAutostartSuppressedChannelIds = new Set(channelIds);
},

2
src/gateway/server-public.ts
View File

@@ -54,6 +54,8 @@ export type GatewayServerOptions = {
channelWizardRunner?: import("./server-methods/wizard.js").ChannelSetupWizardRunner;
sidecarStartup?: GatewaySidecarStartupMode;
channelAutostartSuppression?: ChannelAutostartSuppression;
/** Internal lifecycle callback that re-proves and records crash-loop recovery. */
tryRecoverChannelAutostartSuppression?: () => boolean;
ambientEnvTriggers?: AmbientEnvTriggerPolicy;
/** Optional startup timestamp used for concise readiness logging. */
startupStartedAt?: number;

3
src/gateway/server-runtime-state-prepare.ts
View File

@@ -323,6 +323,9 @@ export async function prepareGatewayRuntimeState(params: {
deferStartupAccountStartsUntil: startupAccountStartsReady,
getNativeApprovalRuntime: () => gatewayInstanceRuntimeRef.current?.nativeApprovals,
ambientAutostartSuppressedChannelIds,
...(opts.tryRecoverChannelAutostartSuppression
? { tryRecoverAutostartSuppression: opts.tryRecoverChannelAutostartSuppression }
: {}),
});
channelManager.setAutostartSuppression(opts.channelAutostartSuppression ?? null);
const sidecarStartup = opts.sidecarStartup ?? "start";

1
src/gateway/server/readiness.test.ts
View File

@@ -35,6 +35,7 @@ function createManager(snapshot: ChannelRuntimeSnapshot): ChannelManager {
stopChannel: vi.fn(),
setAutostartSuppression: vi.fn(),
getAutostartSuppression: vi.fn(() => null),
recoverAutostartSuppression: vi.fn(async () => false),
setAmbientAutostartSuppressedChannelIds: vi.fn(),
isAmbientAutostartSuppressed: vi.fn(() => false),
markChannelLoggedOut: vi.fn(),

54
src/infra/gateway-boot-lifecycle.test.ts
View File

@@ -21,6 +21,7 @@ import {
formatGatewayCrashLoopManualChannelStartHint,
inspectGatewayCrashLoopBreaker,
recordGatewayBootStart,
recordGatewayCrashLoopRecovery,
repairGatewayAgentMediaMigrationStartupFailures,
} from "./gateway-boot-lifecycle.js";
import { executeSqliteQuerySync, getNodeSqliteKysely } from "./kysely-sync.js";
@@ -178,6 +179,59 @@ describe("gateway crash-loop breaker", () => {
expect(secondDecision).toMatchObject({ tripped: false, recovered: false });
});
it("records a fresh lifecycle segment before recovered channel startup", () => {
const db = createLifecycleDb();
const nowMs = 1_000_000;
const safeModeBootId = recordGatewayBootStart(
db.env,
nowMs - GATEWAY_BOOT_LOOP_WINDOW_MS - 1,
GATEWAY_CRASH_LOOP_BREAKER_REASON,
);
expect(inspectGatewayCrashLoopBreaker(db.env, nowMs)).toMatchObject({
recovered: true,
uncleanBoots: 0,
});
const recoveredBootId = recordGatewayCrashLoopRecovery(safeModeBootId, db.env, nowMs);
expect(recoveredBootId).toBeDefined();
expect(
executeSqliteQuerySync(
db.db,
db.kysely
.selectFrom("gateway_boot_lifecycle")
.select(["boot_id", "completed_at_ms", "outcome", "startup_reason"])
.orderBy("started_at_ms"),
).rows,
).toEqual([
{
boot_id: safeModeBootId,
completed_at_ms: nowMs,
outcome: "safe_mode_stable",
startup_reason: GATEWAY_CRASH_LOOP_BREAKER_REASON,
},
{
boot_id: recoveredBootId,
completed_at_ms: null,
outcome: null,
startup_reason: GATEWAY_CRASH_LOOP_RECOVERED_REASON,
},
]);
expect(inspectGatewayCrashLoopBreaker(db.env, nowMs + 1)).toMatchObject({
recovered: false,
uncleanBoots: 1,
});
insertBootRows(db, [
{ bootId: "post-recovery-crash-a", startedAtMs: nowMs + 2 },
{ bootId: "post-recovery-crash-b", startedAtMs: nowMs + 3 },
]);
expect(inspectGatewayCrashLoopBreaker(db.env, nowMs + 4)).toMatchObject({
tripped: true,
uncleanBoots: GATEWAY_BOOT_LOOP_UNCLEAN_THRESHOLD,
});
});
it("records forced stops without tripping the breaker", () => {
const db = createLifecycleDb();
const nowMs = 1_000_000;

62
src/infra/gateway-boot-lifecycle.ts
View File

@@ -23,17 +23,16 @@ import {
// can inspect a stable process instead of a flap.
const GATEWAY_BOOT_LOOP_UNCLEAN_THRESHOLD = 3;
const GATEWAY_BOOT_LOOP_WINDOW_MS = 5 * 60_000;
// Keep enough history for operator forensics while bounding one-row-per-boot
// Keep enough history for operator forensics while bounding lifecycle-segment
// growth. Retention must comfortably exceed GATEWAY_BOOT_LOOP_WINDOW_MS.
const GATEWAY_BOOT_LIFECYCLE_RETENTION_MS = 24 * 60 * 60_000;
export const GATEWAY_BOOT_REASON_MAX_UTF16_CODE_UNITS = 500;
export const GATEWAY_CRASH_LOOP_BREAKER_REASON = "gateway.crash_loop_breaker";
export const GATEWAY_CRASH_LOOP_RECOVERED_REASON = "gateway.crash_loop_recovered";
/**
* The breaker never self-clears within its window, so every operator-facing surface must name the
* manual override command instead of the internal RPC name. Account-scoped suppression must carry
* its accountId: `channels.start` resolves an omitted account to the channel default, so a hint
* without it would start a different account than the one the message named.
* The breaker only self-clears after the full window drains. Operator surfaces name the manual
* override command, not the internal RPC. Account hints carry accountId to avoid starting a
* different default account than the warning named.
*/
export function formatGatewayCrashLoopManualChannelStartHint(target?: {
channelId: string;
@@ -55,6 +54,7 @@ type GatewayBootLifecycleDatabase = Pick<OpenClawStateKyselyDatabase, "gateway_b
type GatewayBootLifecycleOutcome =
| "clean_stop"
| "planned_restart"
| "safe_mode_stable"
| "startup_failed"
| "startup_failure_repaired"
| "forced_stop";
@@ -189,6 +189,58 @@ export function recordGatewayBootStart(
}
}
/**
* Split a stable safe-mode lifetime before channel autostart resumes. A fresh
* open row makes a process death during recovered channel startup count toward
* the next breaker decision instead of aging out with the original boot.
*/
export function recordGatewayCrashLoopRecovery(
bootId: string | undefined,
env: NodeJS.ProcessEnv = process.env,
nowMs = Date.now(),
): string | undefined {
const recoveredBootId = randomUUID();
try {
runOpenClawStateWriteTransaction(
({ db }) => {
const kysely = getNodeSqliteKysely<GatewayBootLifecycleDatabase>(db);
if (bootId) {
executeSqliteQuerySync(
db,
kysely
.updateTable("gateway_boot_lifecycle")
.set({
completed_at_ms: nowMs,
outcome: "safe_mode_stable",
reason: null,
})
.where("boot_id", "=", bootId),
);
}
executeSqliteQuerySync(
db,
kysely.insertInto("gateway_boot_lifecycle").values({
boot_id: recoveredBootId,
pid: process.pid,
started_at_ms: nowMs,
completed_at_ms: null,
outcome: null,
startup_reason: GATEWAY_CRASH_LOOP_RECOVERED_REASON,
reason: null,
}),
);
},
{ env },
);
return recoveredBootId;
} catch (err) {
gatewayLifecycleLog.warn(
`failed to persist gateway crash-loop recovery; fail-safe: ${String(err)}`,
);
return undefined;
}
}
export function completeGatewayBootLifecycle(
bootId: string | undefined,
completion: GatewayBootLifecycleCompletion,