diff --git a/scripts/e2e/secret-provider-integrations.mjs b/scripts/e2e/secret-provider-integrations.mjs index 83d5dad4dfdb..3a28a180360e 100644 --- a/scripts/e2e/secret-provider-integrations.mjs +++ b/scripts/e2e/secret-provider-integrations.mjs @@ -1700,7 +1700,7 @@ async function p12OpenAiLiveProof() { return "OpenAI model auth probe consumed API key through plugin-managed auth-profile SecretRef"; } -async function runPtySecretsConfigurePreset(envCtx) { +async function runPtySecretsConfigurePreset(envCtx, options = {}) { const { spawn } = await import("@lydell/node-pty"); const command = await resolveOpenClawCommand( ["secrets", "configure", "--providers-only", "--apply", "--yes", "--allow-exec", "--json"], @@ -1715,16 +1715,39 @@ async function runPtySecretsConfigurePreset(envCtx) { }); const output = createOutputCapture("secrets configure stdout"); let phase = "providers-menu"; + const keyTimers = new Set(); + const clearKeyTimers = () => { + for (const keyTimer of keyTimers) { + clearTimeout(keyTimer); + } + keyTimers.clear(); + }; const sendKeys = (keys) => { keys.forEach((key, index) => { - setTimeout(() => child.write(key), index * 80); + const keyTimer = setTimeout(() => { + keyTimers.delete(keyTimer); + child.write(key); + }, index * 80); + keyTimers.add(keyTimer); }); }; return await new Promise((resolve, reject) => { + let timedOut = false; + let forceKillAt; + let forceKillTimer; + const timeoutMs = options.timeoutMs ?? 60000; + const timeoutKillGraceMs = options.timeoutKillGraceMs ?? COMMAND_TIMEOUT_KILL_GRACE_MS; const timer = setTimeout(() => { - child.kill(); - reject(new Error(`secrets configure preset timed out: ${scrub(output.text())}`)); - }, 60000); + timedOut = true; + signalPtyProcessTree(child, "SIGHUP"); + forceKillAt = Date.now() + timeoutKillGraceMs; + forceKillTimer = setTimeout(() => { + forceKillTimer = undefined; + forceKillAt = undefined; + signalPtyProcessTree(child, "SIGKILL"); + }, timeoutKillGraceMs); + forceKillTimer.unref?.(); + }, timeoutMs); child.onData((data) => { output.append(data); const outputText = output.text(); @@ -1746,6 +1769,20 @@ async function runPtySecretsConfigurePreset(envCtx) { }); child.onExit(({ exitCode }) => { clearTimeout(timer); + clearKeyTimers(); + if (timedOut) { + void finishTimedOutPtyProcessTree(child, { + forceKillAt, + forceKillTimer, + timeoutKillGraceMs, + }).finally(() => + reject(new Error(`secrets configure preset timed out: ${scrub(output.text())}`)), + ); + return; + } + if (forceKillTimer) { + clearTimeout(forceKillTimer); + } if (exitCode !== 0) { reject(new Error(`secrets configure preset failed (${exitCode}): ${scrub(output.text())}`)); return; @@ -1755,6 +1792,57 @@ async function runPtySecretsConfigurePreset(envCtx) { }); } +async function finishTimedOutPtyProcessTree( + child, + { forceKillAt, forceKillTimer, timeoutKillGraceMs }, +) { + const graceRemainingMs = + forceKillAt === undefined ? timeoutKillGraceMs : Math.max(0, forceKillAt - Date.now()); + if (graceRemainingMs > 0) { + await waitForPtyProcessTreeExit(child, graceRemainingMs); + } + if (forceKillTimer) { + clearTimeout(forceKillTimer); + } + if (ptyProcessTreeIsAlive(child)) { + signalPtyProcessTree(child, "SIGKILL"); + } + await waitForPtyProcessTreeExit(child, timeoutKillGraceMs); +} + +function ptyProcessTreeIsAlive(child) { + if (process.platform === "win32" || typeof child.pid !== "number") { + return false; + } + try { + process.kill(-child.pid, 0); + return true; + } catch (error) { + return error?.code === "EPERM"; + } +} + +async function waitForPtyProcessTreeExit(child, timeoutMs) { + const started = Date.now(); + while (Date.now() - started < timeoutMs) { + if (!ptyProcessTreeIsAlive(child)) { + return true; + } + await delay(50); + } + return !ptyProcessTreeIsAlive(child); +} + +function signalPtyProcessTree(child, signal) { + if (process.platform !== "win32" && typeof child.pid === "number") { + try { + process.kill(-child.pid, signal); + return; + } catch {} + } + child.kill(signal); +} + async function p13SecretsConfigurePreset() { await withProofEnv("p13", async (envCtx) => { const port = await allocatePort(); diff --git a/test/scripts/secret-provider-integrations.test.ts b/test/scripts/secret-provider-integrations.test.ts index b75d6c3494cf..5bbd8cc899fb 100644 --- a/test/scripts/secret-provider-integrations.test.ts +++ b/test/scripts/secret-provider-integrations.test.ts @@ -390,6 +390,74 @@ describe("secret provider integration proof harness", () => { } }); + it.runIf(process.platform !== "win32")( + "cleans PTY configure descendants before timeout failure", + async () => { + const root = makeTempDir(); + const fakeOpenClaw = path.join(root, "fake-openclaw-pty-timeout.mjs"); + const descendantPidPath = path.join(root, "descendant.pid"); + const readyPath = path.join(root, "ready"); + let descendantPid = 0; + const previousEntry = process.env.OPENCLAW_ENTRY; + const descendantScript = [ + "import fs from 'node:fs';", + "process.on('SIGHUP', () => {});", + "process.on('SIGTERM', () => {});", + `fs.writeFileSync(${JSON.stringify(readyPath)}, 'ready');`, + "setInterval(() => {}, 1000);", + ].join("\n"); + fs.writeFileSync( + fakeOpenClaw, + [ + "#!/usr/bin/env node", + "import childProcess from 'node:child_process';", + "import fs from 'node:fs';", + "const descendant = childProcess.spawn(process.execPath, [", + " '--input-type=module',", + ` '--eval', ${JSON.stringify(descendantScript)},`, + "], { stdio: 'ignore' });", + `fs.writeFileSync(${JSON.stringify(descendantPidPath)}, String(descendant.pid));`, + "setInterval(() => {}, 1000);", + "", + ].join("\n"), + { mode: 0o755 }, + ); + process.env.OPENCLAW_ENTRY = fakeOpenClaw; + const proof = await import( + `${pathToFileURL(proofScriptPath).href}?case=pty-timeout-${Date.now()}` + ); + + try { + const result = proof.runPtySecretsConfigurePreset( + { + env: { + ...process.env, + OPENCLAW_ENTRY: fakeOpenClaw, + }, + }, + { timeoutKillGraceMs: 50, timeoutMs: 2_000 }, + ); + result.catch(() => {}); + await waitFor(() => fs.existsSync(readyPath) && fs.existsSync(descendantPidPath)); + descendantPid = Number.parseInt(fs.readFileSync(descendantPidPath, "utf8"), 10); + expect(Number.isInteger(descendantPid)).toBe(true); + expect(isProcessAlive(descendantPid)).toBe(true); + + await expect(result).rejects.toThrow("secrets configure preset timed out"); + await waitFor(() => !isProcessAlive(descendantPid)); + } finally { + if (descendantPid && isProcessAlive(descendantPid)) { + process.kill(descendantPid, "SIGKILL"); + } + if (previousEntry === undefined) { + delete process.env.OPENCLAW_ENTRY; + } else { + process.env.OPENCLAW_ENTRY = previousEntry; + } + } + }, + ); + it.runIf(process.platform !== "win32")( "fails mandatory commands that exit by signal", async () => {