fix(qa-matrix): keep timed CLI process groups tracked

This commit is contained in:
Vincent Koc
2026-06-20 11:41:39 +02:00
parent dd29a6de52
commit cb394309fe
2 changed files with 107 additions and 21 deletions

View File

@@ -21,6 +21,19 @@ function isProcessRunning(pid: number): boolean {
}
}
async function waitForFile(pathToCheck: string, timeoutMs: number): Promise<void> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
try {
await readFile(pathToCheck, "utf8");
return;
} catch {
await sleep(25);
}
}
throw new Error(`Timed out waiting for ${pathToCheck}`);
}
describe("Matrix QA CLI runtime", () => {
it("redacts secret CLI arguments in diagnostic command text", () => {
expect(
@@ -314,4 +327,55 @@ describe("Matrix QA CLI runtime", () => {
await rm(root, { force: true, recursive: true });
}
});
it("kills ignored-stdio descendants after a timed-out CLI exits gracefully", async () => {
if (process.platform === "win32") {
return;
}
const root = await mkdtemp(
path.join(resolvePreferredOpenClawTmpDir(), "matrix-qa-cli-timeout-ignored-stdio-"),
);
const childPidPath = path.join(root, "child.pid");
const grandchildPidPath = path.join(root, "grandchild.pid");
let childPid: number | undefined;
let grandchildPid: number | undefined;
try {
await mkdir(path.join(root, "dist"));
await writeFile(
path.join(root, "dist", "index.mjs"),
[
"import { spawn } from 'node:child_process';",
"import { writeFileSync } from 'node:fs';",
`writeFileSync(${JSON.stringify(childPidPath)}, String(process.pid));`,
"const grandchild = spawn(process.execPath, ['-e', 'process.on(\\'SIGTERM\\', () => {}); setInterval(() => {}, 1000);'], { stdio: 'ignore' });",
"grandchild.unref();",
`writeFileSync(${JSON.stringify(grandchildPidPath)}, String(grandchild.pid));`,
"process.on('SIGTERM', () => process.exit(0));",
"setInterval(() => {}, 1000);",
].join("\n"),
);
const run = runMatrixQaOpenClawCli({
args: ["matrix", "verify", "self"],
cwd: root,
env: process.env,
timeoutMs: 500,
});
await waitForFile(grandchildPidPath, 2_000);
await expect(run).rejects.toThrow(/timed out after 500ms/u);
childPid = Number(await readFile(childPidPath, "utf8"));
grandchildPid = Number(await readFile(grandchildPidPath, "utf8"));
expect(isProcessRunning(childPid)).toBe(false);
expect(isProcessRunning(grandchildPid)).toBe(false);
} finally {
for (const pid of [grandchildPid, childPid]) {
if (pid && isProcessRunning(pid)) {
process.kill(pid, "SIGKILL");
}
}
await rm(root, { force: true, recursive: true });
}
});
});

View File

@@ -119,6 +119,20 @@ function killMatrixQaCliChild(
child.kill(signal);
}
function isMatrixQaCliChildProcessGroupRunning(
child: ReturnType<typeof startOpenClawCliProcess>,
): boolean {
if (process.platform === "win32" || !child.pid) {
return false;
}
try {
process.kill(-child.pid, 0);
return true;
} catch {
return false;
}
}
export function startMatrixQaOpenClawCli(params: {
allowNonZero?: boolean;
args: string[];
@@ -170,22 +184,36 @@ export function startMatrixQaOpenClawCli(params: {
settleWait.resolve(result);
}
};
const finishTimeout = (result: MatrixQaCliRunResult) => {
finish(result, new Error(formatMatrixQaCliTimeoutError(result, params.timeoutMs)));
};
const clearForcedTimeouts = () => {
if (forceKillTimeout) {
clearTimeout(forceKillTimeout);
forceKillTimeout = undefined;
}
if (forceSettleTimeout) {
clearTimeout(forceSettleTimeout);
forceSettleTimeout = undefined;
}
};
const timeout = setTimeout(() => {
timedOut = true;
killMatrixQaCliChild(child, "SIGTERM");
forceKillTimeout = setTimeout(() => {
if (!closed) {
killMatrixQaCliChild(child, "SIGKILL");
forceSettleTimeout = setTimeout(() => {
const result = buildMatrixQaCliResult({
forceKillTimeout = undefined;
killMatrixQaCliChild(child, "SIGKILL");
forceSettleTimeout = setTimeout(() => {
forceSettleTimeout = undefined;
finishTimeout(
buildMatrixQaCliResult({
args: params.args,
exitCode: 1,
output: readOutput(),
});
finish(result, new Error(formatMatrixQaCliTimeoutError(result, params.timeoutMs)));
}, MATRIX_QA_CLI_TIMEOUT_FORCE_SETTLE_MS);
}
}),
);
}, MATRIX_QA_CLI_TIMEOUT_FORCE_SETTLE_MS);
}, MATRIX_QA_CLI_TIMEOUT_KILL_GRACE_MS);
}, params.timeoutMs);
@@ -196,12 +224,7 @@ export function startMatrixQaOpenClawCli(params: {
}
child.on("error", (error) => {
clearTimeout(timeout);
if (forceKillTimeout) {
clearTimeout(forceKillTimeout);
}
if (forceSettleTimeout) {
clearTimeout(forceSettleTimeout);
}
clearForcedTimeouts();
finish(
buildMatrixQaCliResult({
args: params.args,
@@ -213,21 +236,20 @@ export function startMatrixQaOpenClawCli(params: {
});
child.on("close", (exitCode) => {
clearTimeout(timeout);
if (forceKillTimeout) {
clearTimeout(forceKillTimeout);
}
if (forceSettleTimeout) {
clearTimeout(forceSettleTimeout);
}
const result = buildMatrixQaCliResult({
args: params.args,
exitCode: exitCode ?? 1,
output: readOutput(),
});
if (timedOut) {
finish(result, new Error(formatMatrixQaCliTimeoutError(result, params.timeoutMs)));
if (isMatrixQaCliChildProcessGroupRunning(child)) {
return;
}
clearForcedTimeouts();
finishTimeout(result);
return;
}
clearForcedTimeouts();
if (result.exitCode !== 0 && params.allowNonZero !== true) {
finish(result, new Error(formatMatrixQaCliExitError(result)));
return;