mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-06 18:12:13 +00:00
fix(qa-lab): keep lifecycle probe timeout trees tracked
This commit is contained in:
@@ -234,13 +234,16 @@ async function runCommand(command: string, args: readonly string[], options: Com
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const spawnImpl = options.spawnImpl ?? spawn;
|
||||
const useProcessGroup = process.platform !== "win32";
|
||||
const child = spawnImpl(command, args, {
|
||||
cwd: process.cwd(),
|
||||
detached: useProcessGroup,
|
||||
env: options.env ?? process.env,
|
||||
stdio: outputFd === undefined ? "inherit" : (["ignore", outputFd, outputFd] as const),
|
||||
});
|
||||
let settled = false;
|
||||
let forceKillTimer: NodeJS.Timeout | undefined;
|
||||
let forceSettleTimer: NodeJS.Timeout | undefined;
|
||||
let timeoutTimer: NodeJS.Timeout | undefined;
|
||||
let timeoutError: Error | undefined;
|
||||
const clearTimers = () => {
|
||||
@@ -250,6 +253,43 @@ async function runCommand(command: string, args: readonly string[], options: Com
|
||||
if (forceKillTimer) {
|
||||
clearTimeout(forceKillTimer);
|
||||
}
|
||||
if (forceSettleTimer) {
|
||||
clearTimeout(forceSettleTimer);
|
||||
}
|
||||
};
|
||||
const signalChild = (signal: NodeJS.Signals) => {
|
||||
if (useProcessGroup && child.pid) {
|
||||
try {
|
||||
process.kill(-child.pid, signal);
|
||||
return;
|
||||
} catch {
|
||||
// The process group may already be gone; fall back to the direct child.
|
||||
}
|
||||
}
|
||||
child.kill(signal);
|
||||
};
|
||||
const isProcessGroupRunning = () => {
|
||||
if (!useProcessGroup || !child.pid) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
process.kill(-child.pid, 0);
|
||||
return true;
|
||||
} catch (error) {
|
||||
return (error as NodeJS.ErrnoException).code === "EPERM";
|
||||
}
|
||||
};
|
||||
const finish = (error?: Error) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
clearTimers();
|
||||
if (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
};
|
||||
timeoutTimer =
|
||||
options.timeoutMs === undefined
|
||||
@@ -258,37 +298,38 @@ async function runCommand(command: string, args: readonly string[], options: Com
|
||||
timeoutError = new Error(
|
||||
`${command} ${args.join(" ")} timed out after ${options.timeoutMs}ms`,
|
||||
);
|
||||
child.kill("SIGTERM");
|
||||
forceKillTimer = setTimeout(
|
||||
() => child.kill("SIGKILL"),
|
||||
options.timeoutKillGraceMs ?? 2_000,
|
||||
);
|
||||
signalChild("SIGTERM");
|
||||
forceKillTimer = setTimeout(() => {
|
||||
forceKillTimer = undefined;
|
||||
signalChild("SIGKILL");
|
||||
forceSettleTimer = setTimeout(
|
||||
() => finish(timeoutError),
|
||||
options.timeoutKillGraceMs ?? 2_000,
|
||||
);
|
||||
forceSettleTimer.unref();
|
||||
}, options.timeoutKillGraceMs ?? 2_000);
|
||||
forceKillTimer.unref();
|
||||
}, options.timeoutMs);
|
||||
timeoutTimer?.unref();
|
||||
child.once("error", (error) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
clearTimers();
|
||||
reject(error);
|
||||
finish(error);
|
||||
});
|
||||
child.once("exit", (code, signal) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
clearTimers();
|
||||
if (timeoutError) {
|
||||
reject(timeoutError);
|
||||
if (isProcessGroupRunning()) {
|
||||
return;
|
||||
}
|
||||
finish(timeoutError);
|
||||
return;
|
||||
}
|
||||
if (code === 0 && !signal) {
|
||||
resolve();
|
||||
finish();
|
||||
return;
|
||||
}
|
||||
reject(new Error(`${command} ${args.join(" ")} failed with ${signal ?? `exit ${code}`}`));
|
||||
finish(new Error(`${command} ${args.join(" ")} failed with ${signal ?? `exit ${code}`}`));
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Plugin Lifecycle Probe tests cover QA Lab plugin lifecycle evidence.
|
||||
import { EventEmitter } from "node:events";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { createTempDirTracker } from "../../../helpers/temp-dir.js";
|
||||
@@ -17,6 +17,32 @@ function makeTempDir(): string {
|
||||
return tempDirs.make("openclaw-plugin-lifecycle-probe-");
|
||||
}
|
||||
|
||||
function isProcessRunning(pid: number): boolean {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForFile(pathToCheck: string, timeoutMs: number): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (existsSync(pathToCheck)) {
|
||||
return;
|
||||
}
|
||||
await sleep(25);
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${pathToCheck}`);
|
||||
}
|
||||
|
||||
class FakeCommandChild extends EventEmitter {
|
||||
readonly signals: string[] = [];
|
||||
|
||||
@@ -109,4 +135,47 @@ describe("plugin lifecycle matrix probe", () => {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps fallback SIGKILL armed for ignored-stdio descendants", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
|
||||
const dir = makeTempDir();
|
||||
const descendantPidPath = path.join(dir, "descendant.pid");
|
||||
let descendantPid: number | undefined;
|
||||
try {
|
||||
const childScript = "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);";
|
||||
const parentScript = [
|
||||
"import { spawn } from 'node:child_process';",
|
||||
"import { writeFileSync } from 'node:fs';",
|
||||
`const child = spawn(process.execPath, ['-e', ${JSON.stringify(childScript)}], { stdio: 'ignore' });`,
|
||||
"child.unref();",
|
||||
"writeFileSync(process.env.OPENCLAW_TEST_DESCENDANT_PID, String(child.pid));",
|
||||
"process.on('SIGTERM', () => process.exit(0));",
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("\n");
|
||||
|
||||
const run = probeTesting.runCommand(
|
||||
process.execPath,
|
||||
["--input-type=module", "-e", parentScript],
|
||||
{
|
||||
env: { ...process.env, OPENCLAW_TEST_DESCENDANT_PID: descendantPidPath },
|
||||
timeoutKillGraceMs: 250,
|
||||
timeoutMs: 500,
|
||||
},
|
||||
);
|
||||
await waitForFile(descendantPidPath, 2_000);
|
||||
await sleep(300);
|
||||
|
||||
await expect(run).rejects.toThrow(/timed out after 500ms/u);
|
||||
|
||||
descendantPid = Number(readFileSync(descendantPidPath, "utf8"));
|
||||
expect(isProcessRunning(descendantPid)).toBe(false);
|
||||
} finally {
|
||||
if (descendantPid && isProcessRunning(descendantPid)) {
|
||||
process.kill(descendantPid, "SIGKILL");
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user