fix(qa-lab): keep lifecycle probe timeout trees tracked

This commit is contained in:
Vincent Koc
2026-06-20 11:58:10 +02:00
parent 2d603c90dc
commit 7cd58cca2a
2 changed files with 127 additions and 17 deletions

View File

@@ -234,13 +234,16 @@ async function runCommand(command: string, args: readonly string[], options: Com
try {
await new Promise<void>((resolve, reject) => {
const spawnImpl = options.spawnImpl ?? spawn;
const useProcessGroup = process.platform !== "win32";
const child = spawnImpl(command, args, {
cwd: process.cwd(),
detached: useProcessGroup,
env: options.env ?? process.env,
stdio: outputFd === undefined ? "inherit" : (["ignore", outputFd, outputFd] as const),
});
let settled = false;
let forceKillTimer: NodeJS.Timeout | undefined;
let forceSettleTimer: NodeJS.Timeout | undefined;
let timeoutTimer: NodeJS.Timeout | undefined;
let timeoutError: Error | undefined;
const clearTimers = () => {
@@ -250,6 +253,43 @@ async function runCommand(command: string, args: readonly string[], options: Com
if (forceKillTimer) {
clearTimeout(forceKillTimer);
}
if (forceSettleTimer) {
clearTimeout(forceSettleTimer);
}
};
const signalChild = (signal: NodeJS.Signals) => {
if (useProcessGroup && child.pid) {
try {
process.kill(-child.pid, signal);
return;
} catch {
// The process group may already be gone; fall back to the direct child.
}
}
child.kill(signal);
};
const isProcessGroupRunning = () => {
if (!useProcessGroup || !child.pid) {
return false;
}
try {
process.kill(-child.pid, 0);
return true;
} catch (error) {
return (error as NodeJS.ErrnoException).code === "EPERM";
}
};
const finish = (error?: Error) => {
if (settled) {
return;
}
settled = true;
clearTimers();
if (error) {
reject(error);
return;
}
resolve();
};
timeoutTimer =
options.timeoutMs === undefined
@@ -258,37 +298,38 @@ async function runCommand(command: string, args: readonly string[], options: Com
timeoutError = new Error(
`${command} ${args.join(" ")} timed out after ${options.timeoutMs}ms`,
);
child.kill("SIGTERM");
forceKillTimer = setTimeout(
() => child.kill("SIGKILL"),
options.timeoutKillGraceMs ?? 2_000,
);
signalChild("SIGTERM");
forceKillTimer = setTimeout(() => {
forceKillTimer = undefined;
signalChild("SIGKILL");
forceSettleTimer = setTimeout(
() => finish(timeoutError),
options.timeoutKillGraceMs ?? 2_000,
);
forceSettleTimer.unref();
}, options.timeoutKillGraceMs ?? 2_000);
forceKillTimer.unref();
}, options.timeoutMs);
timeoutTimer?.unref();
child.once("error", (error) => {
if (settled) {
return;
}
settled = true;
clearTimers();
reject(error);
finish(error);
});
child.once("exit", (code, signal) => {
if (settled) {
return;
}
settled = true;
clearTimers();
if (timeoutError) {
reject(timeoutError);
if (isProcessGroupRunning()) {
return;
}
finish(timeoutError);
return;
}
if (code === 0 && !signal) {
resolve();
finish();
return;
}
reject(new Error(`${command} ${args.join(" ")} failed with ${signal ?? `exit ${code}`}`));
finish(new Error(`${command} ${args.join(" ")} failed with ${signal ?? `exit ${code}`}`));
});
});
} catch (error) {

View File

@@ -1,6 +1,6 @@
// Plugin Lifecycle Probe tests cover QA Lab plugin lifecycle evidence.
import { EventEmitter } from "node:events";
import { mkdirSync, writeFileSync } from "node:fs";
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { createTempDirTracker } from "../../../helpers/temp-dir.js";
@@ -17,6 +17,32 @@ function makeTempDir(): string {
return tempDirs.make("openclaw-plugin-lifecycle-probe-");
}
function isProcessRunning(pid: number): boolean {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}
async function sleep(ms: number): Promise<void> {
await new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
async function waitForFile(pathToCheck: string, timeoutMs: number): Promise<void> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (existsSync(pathToCheck)) {
return;
}
await sleep(25);
}
throw new Error(`Timed out waiting for ${pathToCheck}`);
}
class FakeCommandChild extends EventEmitter {
readonly signals: string[] = [];
@@ -109,4 +135,47 @@ describe("plugin lifecycle matrix probe", () => {
vi.useRealTimers();
}
});
it("keeps fallback SIGKILL armed for ignored-stdio descendants", async () => {
if (process.platform === "win32") {
return;
}
const dir = makeTempDir();
const descendantPidPath = path.join(dir, "descendant.pid");
let descendantPid: number | undefined;
try {
const childScript = "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);";
const parentScript = [
"import { spawn } from 'node:child_process';",
"import { writeFileSync } from 'node:fs';",
`const child = spawn(process.execPath, ['-e', ${JSON.stringify(childScript)}], { stdio: 'ignore' });`,
"child.unref();",
"writeFileSync(process.env.OPENCLAW_TEST_DESCENDANT_PID, String(child.pid));",
"process.on('SIGTERM', () => process.exit(0));",
"setInterval(() => {}, 1000);",
].join("\n");
const run = probeTesting.runCommand(
process.execPath,
["--input-type=module", "-e", parentScript],
{
env: { ...process.env, OPENCLAW_TEST_DESCENDANT_PID: descendantPidPath },
timeoutKillGraceMs: 250,
timeoutMs: 500,
},
);
await waitForFile(descendantPidPath, 2_000);
await sleep(300);
await expect(run).rejects.toThrow(/timed out after 500ms/u);
descendantPid = Number(readFileSync(descendantPidPath, "utf8"));
expect(isProcessRunning(descendantPid)).toBe(false);
} finally {
if (descendantPid && isProcessRunning(descendantPid)) {
process.kill(descendantPid, "SIGKILL");
}
}
});
});