fix(scripts): clamp parallels host timeouts

This commit is contained in:
Vincent Koc
2026-06-22 02:18:48 +02:00
parent 4cb94cc2cf
commit 790dfb66a8
2 changed files with 54 additions and 12 deletions

View File

@@ -4,6 +4,10 @@ import { createWriteStream } from "node:fs";
import path from "node:path";
import { finished } from "node:stream/promises";
import { fileURLToPath } from "node:url";
import {
addTimerTimeoutGraceMs,
clampTimerTimeoutMs,
} from "@openclaw/normalization-core/number-coercion";
import { resolveNpmRunner } from "../../npm-runner.mjs";
import { resolvePnpmRunner } from "../../pnpm-runner.mjs";
import { buildCmdExeCommandLine, resolveWindowsCmdExePath } from "../../windows-cmd-helpers.mjs";
@@ -338,6 +342,14 @@ function isBareCommand(command: string, name: "npm" | "pnpm"): boolean {
return portableBasename(command) === command && command.toLowerCase() === name;
}
function resolveHostCommandTimeoutMs(timeoutMs: number): number {
return clampTimerTimeoutMs(timeoutMs) ?? 1;
}
function resolveOptionalHostCommandTimeoutMs(timeoutMs: number | undefined): number | undefined {
return timeoutMs === undefined ? undefined : resolveHostCommandTimeoutMs(timeoutMs);
}
export function resolveHostCommandInvocation(
command: string,
args: string[],
@@ -387,9 +399,10 @@ export function resolveHostCommandInvocation(
export function run(command: string, args: string[], options: RunOptions = {}): CommandResult {
const env = { ...process.env, ...options.env };
const invocation = resolveHostCommandInvocation(command, args, { env });
const usesPosixTimedWrapper = process.platform !== "win32" && options.timeoutMs !== undefined;
const timeoutMs = resolveOptionalHostCommandTimeoutMs(options.timeoutMs);
const usesPosixTimedWrapper = process.platform !== "win32" && timeoutMs !== undefined;
const result = usesPosixTimedWrapper
? runPosixTimedCommandSync(invocation, env, options)
? runPosixTimedCommandSync(invocation, env, options, timeoutMs)
: spawnSync(invocation.command, invocation.args, {
cwd: options.cwd ?? repoRoot,
encoding: "utf8",
@@ -399,7 +412,7 @@ export function run(command: string, args: string[], options: RunOptions = {}):
maxBuffer: HOST_COMMAND_MAX_BUFFER_BYTES,
stdio: options.quiet ? ["pipe", "pipe", "pipe"] : ["pipe", "pipe", "pipe"],
shell: invocation.shell,
timeout: options.timeoutMs,
timeout: timeoutMs,
windowsVerbatimArguments: invocation.windowsVerbatimArguments,
});
@@ -421,7 +434,7 @@ export function run(command: string, args: string[], options: RunOptions = {}):
wrapperTimedOut || (result.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT";
if (wrapperTimedOut && options.check !== false) {
const error = new Error(
`${command} ${args.join(" ")} timed out after ${options.timeoutMs}ms`,
`${command} ${args.join(" ")} timed out after ${timeoutMs}ms`,
) as NodeJS.ErrnoException;
error.code = "ETIMEDOUT";
throw error;
@@ -495,7 +508,9 @@ function runPosixTimedCommandSync(
invocation: HostCommandInvocation,
env: NodeJS.ProcessEnv,
options: RunOptions,
timeoutMs: number,
): SpawnSyncReturns<string> {
const wrapperTimeoutMs = addTimerTimeoutGraceMs(timeoutMs, HOST_COMMAND_WRAPPER_BACKSTOP_MS) ?? 1;
const payload = JSON.stringify({
args: invocation.args,
command: invocation.command,
@@ -505,7 +520,7 @@ function runPosixTimedCommandSync(
maxBufferBytes: HOST_COMMAND_MAX_BUFFER_BYTES,
shell: invocation.shell,
timeoutKillGraceMs: HOST_COMMAND_TIMEOUT_KILL_GRACE_MS,
timeoutMs: options.timeoutMs,
timeoutMs,
});
return spawnSync(process.execPath, ["-e", POSIX_TIMEOUT_WRAPPER], {
cwd: options.cwd ?? repoRoot,
@@ -515,7 +530,7 @@ function runPosixTimedCommandSync(
killSignal: "SIGKILL",
maxBuffer: HOST_COMMAND_MAX_BUFFER_BYTES * 2 + HOST_COMMAND_WRAPPER_EXTRA_BUFFER_BYTES,
stdio: ["pipe", "pipe", "pipe", "pipe"],
timeout: (options.timeoutMs ?? 0) + HOST_COMMAND_WRAPPER_BACKSTOP_MS,
timeout: wrapperTimeoutMs,
});
}
@@ -531,11 +546,12 @@ export async function runStreaming(
return await new Promise((resolve, reject) => {
const env = { ...process.env, ...options.env };
const invocation = resolveHostCommandInvocation(command, args, { env });
const timeoutMs = resolveOptionalHostCommandTimeoutMs(options.timeoutMs);
const logStream = options.logPath
? createWriteStream(options.logPath, { encoding: "utf8", flags: "w" })
: undefined;
let logStreamError: Error | undefined;
const detached = process.platform !== "win32" && options.timeoutMs != null;
const detached = process.platform !== "win32" && timeoutMs !== undefined;
const child = spawn(invocation.command, invocation.args, {
cwd: options.cwd ?? repoRoot,
detached,
@@ -570,8 +586,8 @@ export async function runStreaming(
return (error as NodeJS.ErrnoException).code === "EPERM";
}
};
const waitForStreamingProcessGroupExit = async (timeoutMs: number): Promise<boolean> => {
const deadlineAt = Date.now() + timeoutMs;
const waitForStreamingProcessGroupExit = async (timeoutBudgetMs: number): Promise<boolean> => {
const deadlineAt = Date.now() + timeoutBudgetMs;
while (Date.now() < deadlineAt) {
if (!streamingProcessGroupAlive()) {
return true;
@@ -637,7 +653,7 @@ export async function runStreaming(
}
}, HOST_COMMAND_TIMEOUT_KILL_GRACE_MS);
};
if (process.platform !== "win32" && options.timeoutMs != null) {
if (process.platform !== "win32" && timeoutMs !== undefined) {
for (const signal of ["SIGHUP", "SIGINT", "SIGTERM"] as const) {
const handler = (): void => {
forwardedParentSignal ??= signal;
@@ -702,7 +718,7 @@ export async function runStreaming(
}
};
const timer =
options.timeoutMs == null
timeoutMs === undefined
? undefined
: setTimeout(() => {
timedOut = true;
@@ -713,7 +729,7 @@ export async function runStreaming(
HOST_COMMAND_STREAMING_TIMEOUT_KILL_GRACE_MS,
);
killTimer.unref();
}, options.timeoutMs);
}, timeoutMs);
child.on("error", (error) => {
if (timer) {

View File

@@ -17,6 +17,7 @@ import { tmpdir } from "node:os";
import { basename, delimiter, join, win32 } from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import { pathToFileURL } from "node:url";
import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion";
import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
extractLastOpenClawVersionFromLog,
@@ -1466,6 +1467,16 @@ if (isPrlctl) {
expect(result.stdout).toBeTypeOf("string");
});
it("clamps oversized timed host command wrapper timeouts", () => {
const result = run(process.execPath, ["-e", "setTimeout(() => process.exit(0), 25);"], {
check: false,
quiet: true,
timeoutMs: MAX_TIMER_TIMEOUT_MS + 1,
});
expect(result.status).toBe(0);
});
it.runIf(process.platform !== "win32")(
"lets timed host command descendants drain before force kill",
() => {
@@ -1684,6 +1695,21 @@ setInterval(() => {}, 1000);
}
});
it("clamps oversized streaming host command timeouts before arming timers", async () => {
const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout");
try {
await expect(
runStreaming(process.execPath, ["-e", "setTimeout(() => process.exit(0), 25);"], {
quiet: true,
timeoutMs: MAX_TIMER_TIMEOUT_MS + 1,
}),
).resolves.toBe(0);
expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), MAX_TIMER_TIMEOUT_MS);
} finally {
setTimeoutSpy.mockRestore();
}
});
it.runIf(process.platform !== "win32")(
"lets timed streaming host command descendants drain before force kill",
async () => {