diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-cli.test.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-cli.test.ts index 727d720720bf..38e89d967b7d 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-cli.test.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-cli.test.ts @@ -21,6 +21,19 @@ function isProcessRunning(pid: number): boolean { } } +async function waitForFile(pathToCheck: string, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + await readFile(pathToCheck, "utf8"); + return; + } catch { + await sleep(25); + } + } + throw new Error(`Timed out waiting for ${pathToCheck}`); +} + describe("Matrix QA CLI runtime", () => { it("redacts secret CLI arguments in diagnostic command text", () => { expect( @@ -314,4 +327,55 @@ describe("Matrix QA CLI runtime", () => { await rm(root, { force: true, recursive: true }); } }); + + it("kills ignored-stdio descendants after a timed-out CLI exits gracefully", async () => { + if (process.platform === "win32") { + return; + } + const root = await mkdtemp( + path.join(resolvePreferredOpenClawTmpDir(), "matrix-qa-cli-timeout-ignored-stdio-"), + ); + const childPidPath = path.join(root, "child.pid"); + const grandchildPidPath = path.join(root, "grandchild.pid"); + let childPid: number | undefined; + let grandchildPid: number | undefined; + try { + await mkdir(path.join(root, "dist")); + await writeFile( + path.join(root, "dist", "index.mjs"), + [ + "import { spawn } from 'node:child_process';", + "import { writeFileSync } from 'node:fs';", + `writeFileSync(${JSON.stringify(childPidPath)}, String(process.pid));`, + "const grandchild = spawn(process.execPath, ['-e', 'process.on(\\'SIGTERM\\', () => {}); setInterval(() => {}, 1000);'], { stdio: 'ignore' });", + "grandchild.unref();", + `writeFileSync(${JSON.stringify(grandchildPidPath)}, String(grandchild.pid));`, + "process.on('SIGTERM', () => process.exit(0));", + "setInterval(() => {}, 1000);", + ].join("\n"), + ); + + const run = runMatrixQaOpenClawCli({ + args: ["matrix", "verify", "self"], + cwd: root, + env: process.env, + timeoutMs: 500, + }); + await waitForFile(grandchildPidPath, 2_000); + + await expect(run).rejects.toThrow(/timed out after 500ms/u); + + childPid = Number(await readFile(childPidPath, "utf8")); + grandchildPid = Number(await readFile(grandchildPidPath, "utf8")); + expect(isProcessRunning(childPid)).toBe(false); + expect(isProcessRunning(grandchildPid)).toBe(false); + } finally { + for (const pid of [grandchildPid, childPid]) { + if (pid && isProcessRunning(pid)) { + process.kill(pid, "SIGKILL"); + } + } + await rm(root, { force: true, recursive: true }); + } + }); }); diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-cli.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-cli.ts index a6bc399963f3..929078e2ca21 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-cli.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-cli.ts @@ -119,6 +119,20 @@ function killMatrixQaCliChild( child.kill(signal); } +function isMatrixQaCliChildProcessGroupRunning( + child: ReturnType, +): boolean { + if (process.platform === "win32" || !child.pid) { + return false; + } + try { + process.kill(-child.pid, 0); + return true; + } catch { + return false; + } +} + export function startMatrixQaOpenClawCli(params: { allowNonZero?: boolean; args: string[]; @@ -170,22 +184,36 @@ export function startMatrixQaOpenClawCli(params: { settleWait.resolve(result); } }; + const finishTimeout = (result: MatrixQaCliRunResult) => { + finish(result, new Error(formatMatrixQaCliTimeoutError(result, params.timeoutMs))); + }; + const clearForcedTimeouts = () => { + if (forceKillTimeout) { + clearTimeout(forceKillTimeout); + forceKillTimeout = undefined; + } + if (forceSettleTimeout) { + clearTimeout(forceSettleTimeout); + forceSettleTimeout = undefined; + } + }; const timeout = setTimeout(() => { timedOut = true; killMatrixQaCliChild(child, "SIGTERM"); forceKillTimeout = setTimeout(() => { - if (!closed) { - killMatrixQaCliChild(child, "SIGKILL"); - forceSettleTimeout = setTimeout(() => { - const result = buildMatrixQaCliResult({ + forceKillTimeout = undefined; + killMatrixQaCliChild(child, "SIGKILL"); + forceSettleTimeout = setTimeout(() => { + forceSettleTimeout = undefined; + finishTimeout( + buildMatrixQaCliResult({ args: params.args, exitCode: 1, output: readOutput(), - }); - finish(result, new Error(formatMatrixQaCliTimeoutError(result, params.timeoutMs))); - }, MATRIX_QA_CLI_TIMEOUT_FORCE_SETTLE_MS); - } + }), + ); + }, MATRIX_QA_CLI_TIMEOUT_FORCE_SETTLE_MS); }, MATRIX_QA_CLI_TIMEOUT_KILL_GRACE_MS); }, params.timeoutMs); @@ -196,12 +224,7 @@ export function startMatrixQaOpenClawCli(params: { } child.on("error", (error) => { clearTimeout(timeout); - if (forceKillTimeout) { - clearTimeout(forceKillTimeout); - } - if (forceSettleTimeout) { - clearTimeout(forceSettleTimeout); - } + clearForcedTimeouts(); finish( buildMatrixQaCliResult({ args: params.args, @@ -213,21 +236,20 @@ export function startMatrixQaOpenClawCli(params: { }); child.on("close", (exitCode) => { clearTimeout(timeout); - if (forceKillTimeout) { - clearTimeout(forceKillTimeout); - } - if (forceSettleTimeout) { - clearTimeout(forceSettleTimeout); - } const result = buildMatrixQaCliResult({ args: params.args, exitCode: exitCode ?? 1, output: readOutput(), }); if (timedOut) { - finish(result, new Error(formatMatrixQaCliTimeoutError(result, params.timeoutMs))); + if (isMatrixQaCliChildProcessGroupRunning(child)) { + return; + } + clearForcedTimeouts(); + finishTimeout(result); return; } + clearForcedTimeouts(); if (result.exitCode !== 0 && params.allowNonZero !== true) { finish(result, new Error(formatMatrixQaCliExitError(result))); return;