fix(e2e): preserve host command timeout grace

scripts/e2e/parallels/host-command.ts

@@ -14,6 +14,10 @@ export const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)
 const HOST_COMMAND_MAX_BUFFER_BYTES = 50 * 1024 * 1024;
 const HOST_COMMAND_WRAPPER_EXTRA_BUFFER_BYTES = 1024 * 1024;
 const HOST_COMMAND_WRAPPER_BACKSTOP_MS = 5_000;
+const HOST_COMMAND_TIMEOUT_KILL_GRACE_MS = 100;
+const HOST_COMMAND_STREAMING_TIMEOUT_KILL_GRACE_MS = 2_000;
+const HOST_COMMAND_PROCESS_GROUP_EXIT_POLL_MS = 25;
+const HOST_COMMAND_POST_FORCE_KILL_WAIT_MS = 100;
 const HOST_COMMAND_CHILD_PID_PREFIX = "__OPENCLAW_HOST_COMMAND_CHILD_PID__";
 const HOST_COMMAND_SPAWN_ERROR_PREFIX = "__OPENCLAW_HOST_COMMAND_SPAWN_ERROR__";
 const HOST_COMMAND_TIMEOUT_PREFIX = "__OPENCLAW_HOST_COMMAND_TIMEOUT__";
@@ -108,6 +112,7 @@ writeSync(
 
 let timedOut = false;
 let killTimer;
+let killDeadlineAt = 0;
 let outputExceeded = false;
 let stderrBytes = 0;
 let stdoutBytes = 0;
@@ -132,6 +137,58 @@ function signalGroup(signal) {
   }
 }
 
+function groupAlive() {
+  if (!child.pid) {
+    return false;
+  }
+  try {
+    process.kill(-child.pid, 0);
+    return true;
+  } catch (error) {
+    return Boolean(error && error.code === "EPERM");
+  }
+}
+
+function finishTimedOut() {
+  if (killTimer) {
+    clearTimeout(killTimer);
+  }
+  writeSync(3, ${JSON.stringify(HOST_COMMAND_TIMEOUT_PREFIX)} + "{}\n");
+  process.exit(124);
+}
+
+function finishTimedOutAfterCleanup() {
+  if (!groupAlive()) {
+    finishTimedOut();
+    return;
+  }
+  const pollMs = Math.max(1, Math.min(25, payload.timeoutKillGraceMs));
+  let pollTimer;
+  let forceFinishTimer;
+  let postForceFinishTimer;
+  const finish = () => {
+    if (pollTimer) {
+      clearInterval(pollTimer);
+    }
+    if (forceFinishTimer) {
+      clearTimeout(forceFinishTimer);
+    }
+    if (postForceFinishTimer) {
+      clearTimeout(postForceFinishTimer);
+    }
+    finishTimedOut();
+  };
+  pollTimer = setInterval(() => {
+    if (!groupAlive()) {
+      finish();
+    }
+  }, pollMs);
+  forceFinishTimer = setTimeout(() => {
+    signalGroup("SIGKILL");
+    postForceFinishTimer = setTimeout(finish, pollMs);
+  }, Math.max(0, killDeadlineAt - Date.now()));
+}
+
 function forwardBounded(stream, chunk) {
   const currentBytes = stream === "stdout" ? stdoutBytes : stderrBytes;
   const nextBytes = currentBytes + chunk.byteLength;
@@ -170,7 +227,8 @@ for (const signal of ["SIGHUP", "SIGINT", "SIGTERM"]) {
 const timeout = setTimeout(() => {
   timedOut = true;
   signalGroup("SIGTERM");
-  killTimer = setTimeout(() => signalGroup("SIGKILL"), 100);
+  killDeadlineAt = Date.now() + payload.timeoutKillGraceMs;
+  killTimer = setTimeout(() => signalGroup("SIGKILL"), payload.timeoutKillGraceMs);
   killTimer.unref();
 }, payload.timeoutMs);
 timeout.unref();
@@ -199,14 +257,13 @@ child.on("error", (error) => {
 });
 child.on("close", (code, signal) => {
   clearTimeout(timeout);
+  if (timedOut) {
+    finishTimedOutAfterCleanup();
+    return;
+  }
   if (killTimer) {
     clearTimeout(killTimer);
   }
-  if (timedOut) {
-    signalGroup("SIGKILL");
-    writeSync(3, ${JSON.stringify(HOST_COMMAND_TIMEOUT_PREFIX)} + "{}\n");
-    process.exit(124);
-  }
   if (outputExceeded) {
     process.exit(1);
   }
@@ -407,6 +464,7 @@ function runPosixTimedCommandSync(
     input: options.input,
     maxBufferBytes: HOST_COMMAND_MAX_BUFFER_BYTES,
     shell: invocation.shell,
+    timeoutKillGraceMs: HOST_COMMAND_TIMEOUT_KILL_GRACE_MS,
     timeoutMs: options.timeoutMs,
   });
   return spawnSync(process.execPath, ["-e", POSIX_TIMEOUT_WRAPPER], {
@@ -461,6 +519,29 @@ export async function runStreaming(
         }
       }
     };
+    const streamingProcessGroupAlive = (): boolean => {
+      if (!detached || !childPid) {
+        return false;
+      }
+      try {
+        process.kill(-childPid, 0);
+        return true;
+      } catch (error) {
+        return (error as NodeJS.ErrnoException).code === "EPERM";
+      }
+    };
+    const waitForStreamingProcessGroupExit = async (timeoutMs: number): Promise<boolean> => {
+      const deadlineAt = Date.now() + timeoutMs;
+      while (Date.now() < deadlineAt) {
+        if (!streamingProcessGroupAlive()) {
+          return true;
+        }
+        await new Promise((resolvePoll) => {
+          setTimeout(resolvePoll, HOST_COMMAND_PROCESS_GROUP_EXIT_POLL_MS);
+        });
+      }
+      return !streamingProcessGroupAlive();
+    };
     logStream?.on("error", (error) => {
       logStreamError = error;
       signalStreamingChild("SIGTERM");
@@ -520,13 +601,32 @@ export async function runStreaming(
 
     let timedOut = false;
     let killTimer: NodeJS.Timeout | undefined;
+    let killDeadlineAt = 0;
+    const waitForStreamingTimeoutCleanup = async (): Promise<void> => {
+      if (!detached) {
+        signalStreamingChild("SIGKILL");
+        return;
+      }
+      const remainingGraceMs = Math.max(0, killDeadlineAt - Date.now());
+      if (remainingGraceMs > 0) {
+        await waitForStreamingProcessGroupExit(remainingGraceMs);
+      }
+      if (streamingProcessGroupAlive()) {
+        signalStreamingChild("SIGKILL");
+        await waitForStreamingProcessGroupExit(HOST_COMMAND_POST_FORCE_KILL_WAIT_MS);
+      }
+    };
     const timer =
       options.timeoutMs == null
         ? undefined
         : setTimeout(() => {
             timedOut = true;
             signalHostCommandProcess(childPid, "SIGTERM");
-            killTimer = setTimeout(() => signalHostCommandProcess(childPid, "SIGKILL"), 2_000);
+            killDeadlineAt = Date.now() + HOST_COMMAND_STREAMING_TIMEOUT_KILL_GRACE_MS;
+            killTimer = setTimeout(
+              () => signalHostCommandProcess(childPid, "SIGKILL"),
+              HOST_COMMAND_STREAMING_TIMEOUT_KILL_GRACE_MS,
+            );
             killTimer.unref();
           }, options.timeoutMs);
 
@@ -546,12 +646,12 @@ export async function runStreaming(
         if (timer) {
           clearTimeout(timer);
         }
-        if (killTimer) {
-          clearTimeout(killTimer);
-        }
         removeParentSignalHandlers();
         if (timedOut) {
-          signalStreamingChild("SIGKILL");
+          await waitForStreamingTimeoutCleanup();
+        }
+        if (killTimer) {
+          clearTimeout(killTimer);
         }
         if (logStream) {
           logStream.end();