From fb0812c857ffe409947296eebd30677fef4ec211 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 7 Aug 2026 14:46:59 -0700 Subject: [PATCH] fix(qa): prevent false update-restart package failures (#120300) * test(qa): expose update restart process exits * test(qa): supervise update restart gateway * test(qa): isolate supervised gateway environment * test(qa): diagnose restart plugin convergence * test(qa): mount trusted upgrade harness * test(qa): include upgrade runtime companions * test(qa): remove temporary restart diagnostics * test(qa): enforce systemd restart budget * test(qa): honor systemd stop timeout * test(qa): tighten package fixture ownership * test(qa): preserve unrelated lane shapes * test(qa): complete service fixture boundaries --- scripts/e2e/lib/upgrade-survivor/run.sh | 161 +++++++- .../upgrade-survivor/update-restart-auth.sh | 158 +++++++- scripts/e2e/upgrade-survivor-docker.sh | 4 + scripts/lib/docker-e2e-package.sh | 13 +- scripts/lib/docker-e2e-plan.mjs | 12 +- scripts/lib/docker-e2e-scenarios.d.mts | 1 + scripts/lib/docker-e2e-scenarios.mjs | 5 + test/scripts/docker-build-helper.test.ts | 349 +++++++++++++++++- test/scripts/docker-e2e-plan.test.ts | 6 + 9 files changed, 681 insertions(+), 28 deletions(-) diff --git a/scripts/e2e/lib/upgrade-survivor/run.sh b/scripts/e2e/lib/upgrade-survivor/run.sh index e8dd9a7ded6c..6076112fa953 100644 --- a/scripts/e2e/lib/upgrade-survivor/run.sh +++ b/scripts/e2e/lib/upgrade-survivor/run.sh @@ -230,6 +230,9 @@ cleanup() { if [ -n "${plugin_registry_pid:-}" ]; then kill "$plugin_registry_pid" >/dev/null 2>&1 || true fi + if [ -s "$SYSTEMCTL_SHIM_PID_FILE" ]; then + systemctl --user stop openclaw-gateway.service >/dev/null 2>&1 || true + fi openclaw_e2e_terminate_gateways "${gateway_pid:-}" if [ -s "$SYSTEMCTL_SHIM_PID_FILE" ]; then local shim_pid @@ -784,6 +787,7 @@ set -euo pipefail log_file="${OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_LOG:-/tmp/openclaw-systemctl-shim.log}" pid_file="${OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_PID_FILE:-/tmp/openclaw-systemctl-shim.pid}" daemon_log="${OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_DAEMON_LOG:-/tmp/openclaw-systemctl-shim-gateway.log}" +supervisor_script="${pid_file}.supervisor.mjs" printf '%s\n' "$*" >>"$log_file" filtered=() @@ -815,24 +819,27 @@ command="${filtered[0]:-status}" is_running() { [ -s "$pid_file" ] || return 1 local pid + local process_state pid="$(cat "$pid_file" 2>/dev/null || true)" [ -n "$pid" ] || return 1 - kill -0 "$pid" >/dev/null 2>&1 + kill -0 "$pid" >/dev/null 2>&1 || return 1 + process_state="$(awk '{ print $3 }' "/proc/$pid/stat" 2>/dev/null || true)" + [ "$process_state" != "Z" ] } stop_gateway() { - [ -s "$pid_file" ] || return 0 - local pid + local pid="" pid="$(cat "$pid_file" 2>/dev/null || true)" if [[ "$pid" =~ ^[0-9]+$ ]] && [ "$pid" -gt 1 ] && kill -0 "$pid" >/dev/null 2>&1; then kill "$pid" >/dev/null 2>&1 || true - for _ in $(seq 1 100); do - kill -0 "$pid" >/dev/null 2>&1 || break + # The supervisor gives its child 30s, so keep this outer deadline comfortably longer. + for _ in $(seq 1 350); do + is_running || break sleep 0.1 done kill -9 "$pid" >/dev/null 2>&1 || true fi - rm -f "$pid_file" + rm -f "$pid_file" "$supervisor_script" } unit_path() { @@ -873,9 +880,149 @@ start_gateway() { echo "systemctl shim could not find ExecStart in $unit" >&2 return 1 } + rm -f "$pid_file" "$supervisor_script" + cat >"$supervisor_script" <<'SUPERVISOR' +import fs from "node:fs"; +import { spawn } from "node:child_process"; + +const command = process.env.OPENCLAW_SYSTEMCTL_SHIM_EXEC_START; +const daemonLog = process.env.OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG; +if (!command || !daemonLog) { + process.exit(2); +} + +const output = fs.openSync(daemonLog, "a"); +const childEnv = { ...process.env }; +delete childEnv.OPENCLAW_SYSTEMCTL_SHIM_EXEC_START; +delete childEnv.OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG; +// systemd does not pass transient systemctl-caller update state into the service. +for (const key of Object.keys(childEnv)) { + if (key.startsWith("OPENCLAW_UPDATE_")) { + delete childEnv[key]; + } +} +delete childEnv.OPENCLAW_COMPATIBILITY_HOST_VERSION; +const restartDelayMs = 5_000; +const restartWindowMs = 60_000; +const restartBurst = 5; +const stopTimeoutMs = 30_000; +const starts = []; +let child; +let activeGroupPid; +let drainingGroupPid; +let stopping = false; + +const finish = () => { + try { + fs.closeSync(output); + } catch {} + process.exit(0); +}; + +const signalProcessGroup = (pid, signal) => { + try { + process.kill(-pid, signal); + } catch (error) { + if (error?.code !== "ESRCH") { + fs.writeSync(output, `[systemctl-shim] gateway process group ${signal} failed: ${String(error)}\n`); + } + } +}; + +const isProcessGroupRunning = (pid) => { + try { + process.kill(-pid, 0); + return true; + } catch (error) { + return error?.code !== "ESRCH"; + } +}; + +const drainProcessGroup = (pid, onStopped) => { + if (!pid) return onStopped(); + if (drainingGroupPid === pid) return; + drainingGroupPid = pid; + let completed = false; + const complete = () => { + if (completed) return; + completed = true; + if (drainingGroupPid === pid) drainingGroupPid = undefined; + if (activeGroupPid === pid) activeGroupPid = undefined; + onStopped(); + }; + signalProcessGroup(pid, "SIGTERM"); + const forceKill = setTimeout(() => { + signalProcessGroup(pid, "SIGKILL"); + complete(); + }, stopTimeoutMs); + const finishWhenStopped = () => { + if (completed) return; + if (isProcessGroupRunning(pid)) { + setTimeout(finishWhenStopped, 25); + return; + } + clearTimeout(forceKill); + complete(); + }; + finishWhenStopped(); +}; + +const stop = () => { + if (stopping) return; + stopping = true; + if (drainingGroupPid) return; + if (activeGroupPid) { + drainProcessGroup(activeGroupPid, finish); + return; + } + if (child) { + child.kill("SIGTERM"); + return; + } + finish(); +}; + +const start = () => { + if (stopping) return finish(); + const now = Date.now(); + while (starts.length > 0 && starts[0] <= now - restartWindowMs) { + starts.shift(); + } + if (starts.length >= restartBurst) { + fs.writeSync(output, "[systemctl-shim] gateway restart limit reached\n"); + return finish(); + } + starts.push(now); + child = spawn("bash", ["-lc", `exec ${command}`], { + detached: true, + env: childEnv, + stdio: ["ignore", output, output], + }); + activeGroupPid = child.pid; + const childGroupPid = activeGroupPid; + child.on("error", (error) => { + fs.writeSync(output, `[systemctl-shim] gateway spawn failed: ${String(error)}\n`); + }); + child.once("close", (code) => { + child = undefined; + drainProcessGroup(childGroupPid, () => { + if (stopping) return finish(); + // Match the generated systemd unit's RestartPreventExitStatus contract. + if (code === 78) return finish(); + setTimeout(start, restartDelayMs); + }); + }); +}; + +process.on("SIGINT", stop); +process.on("SIGTERM", stop); +start(); +SUPERVISOR ( load_unit_environment "$unit" - nohup bash -lc "exec $exec_start" >>"$daemon_log" 2>&1 & + OPENCLAW_SYSTEMCTL_SHIM_EXEC_START="$exec_start" \ + OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG="$daemon_log" \ + nohup node "$supervisor_script" /dev/null 2>&1 & printf '%s\n' "$!" >"$pid_file" ) } diff --git a/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh b/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh index 342864776f79..22cd77d8ec18 100644 --- a/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh +++ b/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh @@ -10,6 +10,7 @@ set -euo pipefail log_file="${OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_LOG:-/tmp/openclaw-systemctl-shim.log}" pid_file="${OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_PID_FILE:-/tmp/openclaw-systemctl-shim.pid}" daemon_log="${OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_DAEMON_LOG:-/tmp/openclaw-systemctl-shim-gateway.log}" +supervisor_script="${pid_file}.supervisor.mjs" printf '%s\n' "$*" >>"$log_file" filtered=() @@ -41,24 +42,27 @@ command="${filtered[0]:-status}" is_running() { [ -s "$pid_file" ] || return 1 local pid + local process_state pid="$(cat "$pid_file" 2>/dev/null || true)" [ -n "$pid" ] || return 1 - kill -0 "$pid" >/dev/null 2>&1 + kill -0 "$pid" >/dev/null 2>&1 || return 1 + process_state="$(awk '{ print $3 }' "/proc/$pid/stat" 2>/dev/null || true)" + [ "$process_state" != "Z" ] } stop_gateway() { - [ -s "$pid_file" ] || return 0 - local pid + local pid="" pid="$(cat "$pid_file" 2>/dev/null || true)" if [[ "$pid" =~ ^[0-9]+$ ]] && [ "$pid" -gt 1 ] && kill -0 "$pid" >/dev/null 2>&1; then kill "$pid" >/dev/null 2>&1 || true - for _ in $(seq 1 100); do - kill -0 "$pid" >/dev/null 2>&1 || break + # The supervisor gives its child 30s, so keep this outer deadline comfortably longer. + for _ in $(seq 1 350); do + is_running || break sleep 0.1 done kill -9 "$pid" >/dev/null 2>&1 || true fi - rm -f "$pid_file" + rm -f "$pid_file" "$supervisor_script" } unit_path() { @@ -99,9 +103,149 @@ start_gateway() { echo "systemctl shim could not find ExecStart in $unit" >&2 return 1 } + rm -f "$pid_file" "$supervisor_script" + cat >"$supervisor_script" <<'SUPERVISOR' +import fs from "node:fs"; +import { spawn } from "node:child_process"; + +const command = process.env.OPENCLAW_SYSTEMCTL_SHIM_EXEC_START; +const daemonLog = process.env.OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG; +if (!command || !daemonLog) { + process.exit(2); +} + +const output = fs.openSync(daemonLog, "a"); +const childEnv = { ...process.env }; +delete childEnv.OPENCLAW_SYSTEMCTL_SHIM_EXEC_START; +delete childEnv.OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG; +// systemd does not pass transient systemctl-caller update state into the service. +for (const key of Object.keys(childEnv)) { + if (key.startsWith("OPENCLAW_UPDATE_")) { + delete childEnv[key]; + } +} +delete childEnv.OPENCLAW_COMPATIBILITY_HOST_VERSION; +const restartDelayMs = 5_000; +const restartWindowMs = 60_000; +const restartBurst = 5; +const stopTimeoutMs = 30_000; +const starts = []; +let child; +let activeGroupPid; +let drainingGroupPid; +let stopping = false; + +const finish = () => { + try { + fs.closeSync(output); + } catch {} + process.exit(0); +}; + +const signalProcessGroup = (pid, signal) => { + try { + process.kill(-pid, signal); + } catch (error) { + if (error?.code !== "ESRCH") { + fs.writeSync(output, `[systemctl-shim] gateway process group ${signal} failed: ${String(error)}\n`); + } + } +}; + +const isProcessGroupRunning = (pid) => { + try { + process.kill(-pid, 0); + return true; + } catch (error) { + return error?.code !== "ESRCH"; + } +}; + +const drainProcessGroup = (pid, onStopped) => { + if (!pid) return onStopped(); + if (drainingGroupPid === pid) return; + drainingGroupPid = pid; + let completed = false; + const complete = () => { + if (completed) return; + completed = true; + if (drainingGroupPid === pid) drainingGroupPid = undefined; + if (activeGroupPid === pid) activeGroupPid = undefined; + onStopped(); + }; + signalProcessGroup(pid, "SIGTERM"); + const forceKill = setTimeout(() => { + signalProcessGroup(pid, "SIGKILL"); + complete(); + }, stopTimeoutMs); + const finishWhenStopped = () => { + if (completed) return; + if (isProcessGroupRunning(pid)) { + setTimeout(finishWhenStopped, 25); + return; + } + clearTimeout(forceKill); + complete(); + }; + finishWhenStopped(); +}; + +const stop = () => { + if (stopping) return; + stopping = true; + if (drainingGroupPid) return; + if (activeGroupPid) { + drainProcessGroup(activeGroupPid, finish); + return; + } + if (child) { + child.kill("SIGTERM"); + return; + } + finish(); +}; + +const start = () => { + if (stopping) return finish(); + const now = Date.now(); + while (starts.length > 0 && starts[0] <= now - restartWindowMs) { + starts.shift(); + } + if (starts.length >= restartBurst) { + fs.writeSync(output, "[systemctl-shim] gateway restart limit reached\n"); + return finish(); + } + starts.push(now); + child = spawn("bash", ["-lc", `exec ${command}`], { + detached: true, + env: childEnv, + stdio: ["ignore", output, output], + }); + activeGroupPid = child.pid; + const childGroupPid = activeGroupPid; + child.on("error", (error) => { + fs.writeSync(output, `[systemctl-shim] gateway spawn failed: ${String(error)}\n`); + }); + child.once("close", (code) => { + child = undefined; + drainProcessGroup(childGroupPid, () => { + if (stopping) return finish(); + // Match the generated systemd unit's RestartPreventExitStatus contract. + if (code === 78) return finish(); + setTimeout(start, restartDelayMs); + }); + }); +}; + +process.on("SIGINT", stop); +process.on("SIGTERM", stop); +start(); +SUPERVISOR ( load_unit_environment "$unit" - nohup bash -lc "exec $exec_start" >>"$daemon_log" 2>&1 & + OPENCLAW_SYSTEMCTL_SHIM_EXEC_START="$exec_start" \ + OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG="$daemon_log" \ + nohup node "$supervisor_script" /dev/null 2>&1 & printf '%s\n' "$!" >"$pid_file" ) } diff --git a/scripts/e2e/upgrade-survivor-docker.sh b/scripts/e2e/upgrade-survivor-docker.sh index dbfddfa0783f..c43ccad960db 100755 --- a/scripts/e2e/upgrade-survivor-docker.sh +++ b/scripts/e2e/upgrade-survivor-docker.sh @@ -6,6 +6,7 @@ set -euo pipefail HARNESS_ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" ROOT_DIR="$(cd "${OPENCLAW_DOCKER_E2E_REPO_ROOT:-$HARNESS_ROOT_DIR}" && pwd)" +DOCKER_E2E_HARNESS_ROOT_DIR="$HARNESS_ROOT_DIR" source "$HARNESS_ROOT_DIR/scripts/lib/docker-e2e-image.sh" source "$HARNESS_ROOT_DIR/scripts/lib/docker-e2e-package.sh" source "$HARNESS_ROOT_DIR/scripts/lib/openclaw-e2e-instance.sh" @@ -268,6 +269,9 @@ cleanup() { if [ -n "${plugin_registry_pid:-}" ]; then kill "$plugin_registry_pid" >/dev/null 2>&1 || true fi + if [ -s "$SYSTEMCTL_SHIM_PID_FILE" ]; then + systemctl --user stop openclaw-gateway.service >/dev/null 2>&1 || true + fi openclaw_e2e_terminate_gateways "${gateway_pid:-}" if [ -s "$SYSTEMCTL_SHIM_PID_FILE" ]; then openclaw_e2e_terminate_gateways "$(cat "$SYSTEMCTL_SHIM_PID_FILE" 2>/dev/null || true)" diff --git a/scripts/lib/docker-e2e-package.sh b/scripts/lib/docker-e2e-package.sh index fe68ddeead9d..205c1fbe1bdf 100644 --- a/scripts/lib/docker-e2e-package.sh +++ b/scripts/lib/docker-e2e-package.sh @@ -264,13 +264,14 @@ docker_e2e_cleanup_container_cidfile() { } docker_e2e_harness_mount_args() { + local harness_root="${DOCKER_E2E_HARNESS_ROOT_DIR:-$ROOT_DIR}" DOCKER_E2E_HARNESS_ARGS=( - -v "$ROOT_DIR/scripts/e2e:/app/scripts/e2e:ro" - -v "$ROOT_DIR/scripts/lib:/app/scripts/lib:ro" - -v "$ROOT_DIR/packages/normalization-core/src:/app/packages/normalization-core/src:ro" - -v "$ROOT_DIR/test/e2e/qa-lab:/app/test/e2e/qa-lab:ro" - -v "$ROOT_DIR/test/helpers:/app/test/helpers:ro" - -v "$ROOT_DIR/scripts/windows-cmd-helpers.mjs:/app/scripts/windows-cmd-helpers.mjs:ro" + -v "$harness_root/scripts/e2e:/app/scripts/e2e:ro" + -v "$harness_root/scripts/lib:/app/scripts/lib:ro" + -v "$harness_root/packages/normalization-core/src:/app/packages/normalization-core/src:ro" + -v "$harness_root/test/e2e/qa-lab:/app/test/e2e/qa-lab:ro" + -v "$harness_root/test/helpers:/app/test/helpers:ro" + -v "$harness_root/scripts/windows-cmd-helpers.mjs:/app/scripts/windows-cmd-helpers.mjs:ro" ) } diff --git a/scripts/lib/docker-e2e-plan.mjs b/scripts/lib/docker-e2e-plan.mjs index a3fa3c84ee33..29176785ec26 100644 --- a/scripts/lib/docker-e2e-plan.mjs +++ b/scripts/lib/docker-e2e-plan.mjs @@ -674,7 +674,11 @@ function configuredChannelIdsForLane(poolLane, scenario) { export function requiredPrepublishPluginPackagesForLanes(poolLanes) { const configuredChannelIds = new Set(); + const requiredPackages = new Set(); for (const poolLane of poolLanes) { + for (const packageName of poolLane.prepublishPluginPackages ?? []) { + requiredPackages.add(packageName); + } const scenario = upgradeSurvivorScenarioForLane(poolLane); if (!scenario) { continue; @@ -683,7 +687,7 @@ export function requiredPrepublishPluginPackagesForLanes(poolLanes) { configuredChannelIds.add(channelId); } } - return (officialExternalChannelCatalog.entries ?? []) + for (const packageName of (officialExternalChannelCatalog.entries ?? []) .filter((entry) => { const channelId = entry.openclaw?.channel?.id; const install = entry.openclaw?.install; @@ -694,8 +698,10 @@ export function requiredPrepublishPluginPackagesForLanes(poolLanes) { install?.npmSpec === entry.name ); }) - .map((entry) => entry.name) - .toSorted((a, b) => a.localeCompare(b)); + .map((entry) => entry.name)) { + requiredPackages.add(packageName); + } + return [...requiredPackages].toSorted((a, b) => a.localeCompare(b)); } function buildPlanJson(params) { diff --git a/scripts/lib/docker-e2e-scenarios.d.mts b/scripts/lib/docker-e2e-scenarios.d.mts index 46ae9bc15c6f..90154fe968e8 100644 --- a/scripts/lib/docker-e2e-scenarios.d.mts +++ b/scripts/lib/docker-e2e-scenarios.d.mts @@ -11,6 +11,7 @@ export type DockerE2eLane = { name: string; needsLiveImage?: boolean; noOutputTimeoutMs?: number; + prepublishPluginPackages?: string[]; resources: string[]; retries: number; retryPatterns: RegExp[]; diff --git a/scripts/lib/docker-e2e-scenarios.mjs b/scripts/lib/docker-e2e-scenarios.mjs index 37652e29a98a..d8d0628cdd26 100644 --- a/scripts/lib/docker-e2e-scenarios.mjs +++ b/scripts/lib/docker-e2e-scenarios.mjs @@ -82,6 +82,9 @@ function lane(name, command, options = {}) { timeoutMs: options.timeoutMs, upgradeSurvivorScenario: options.upgradeSurvivorScenario, weight: options.weight ?? 1, + ...(options.prepublishPluginPackages + ? { prepublishPluginPackages: options.prepublishPluginPackages } + : {}), }; } @@ -198,6 +201,8 @@ function createPackageUpdateMaintenanceLanes() { weight: 3, }), npmLane("update-restart-auth", updateRestartAuthCommand, { + // Credential hydration auto-enables the candidate's Codex runtime during restart. + prepublishPluginPackages: ["@openclaw/codex"], stateScenario: "upgrade-survivor", timeoutMs: 25 * 60 * 1000, upgradeSurvivorScenario: "base", diff --git a/test/scripts/docker-build-helper.test.ts b/test/scripts/docker-build-helper.test.ts index b5af6c31ec7b..26c1cee7fccb 100644 --- a/test/scripts/docker-build-helper.test.ts +++ b/test/scripts/docker-build-helper.test.ts @@ -1,5 +1,5 @@ // Docker Build Helper tests cover docker build helper script behavior. -import { execFileSync, spawn, spawnSync } from "node:child_process"; +import { type ChildProcess, execFileSync, spawn, spawnSync } from "node:child_process"; import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; @@ -173,6 +173,55 @@ function expectTextToIncludeAll(text: string, snippets: readonly string[]): void } } +function extractUpgradeSurvivorSupervisor(script: string): string { + const match = script.match( + /cat >"\$supervisor_script" <<'SUPERVISOR'\n(?[\s\S]*?)\nSUPERVISOR/u, + ); + const source = match?.groups?.source; + if (!source) { + throw new Error("upgrade survivor supervisor source not found"); + } + return source; +} + +async function waitForProcessExit(child: ChildProcess, timeoutMs = 5_000): Promise { + if (child.exitCode !== null || child.signalCode !== null) { + return child.exitCode; + } + return await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error("process did not exit before its test deadline")); + }, timeoutMs); + child.once("exit", (code) => { + clearTimeout(timeout); + resolve(code); + }); + }); +} + +function isProcessRunning(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +function writeTermIgnoringDescendant(workDir: string): string { + const descendantPath = join(workDir, "descendant.mjs"); + writeFileSync( + descendantPath, + `import fs from "node:fs"; +process.on("SIGTERM", () => {}); +fs.writeFileSync(process.env.DESCENDANT_PID_FILE, String(process.pid)); +setInterval(() => {}, 1_000); +`, + ); + return descendantPath; +} + function cleanupSmokeLogTailHelpers(): string { const script = readFileSync(CLEANUP_SMOKE_RUN_PATH, "utf8"); const match = script.match( @@ -2423,6 +2472,7 @@ docker_e2e_docker_run_cmd run demo expect(upgradeSurvivor).toContain( 'ROOT_DIR="$(cd "${OPENCLAW_DOCKER_E2E_REPO_ROOT:-$HARNESS_ROOT_DIR}" && pwd)"', ); + expect(upgradeSurvivor).toContain('DOCKER_E2E_HARNESS_ROOT_DIR="$HARNESS_ROOT_DIR"'); expect(upgradeSurvivor).toContain( '-v "$HARNESS_ROOT_DIR/scripts/e2e/lib/upgrade-survivor/run.sh:/tmp/openclaw-upgrade-survivor-run.sh:ro"', ); @@ -2608,6 +2658,291 @@ fi expect(successIndex).toBeGreaterThan(manualSummaryIndex); }); + it("models systemd restart supervision in update-restart auth fixtures", () => { + const runner = readFileSync(UPGRADE_SURVIVOR_DOCKER_E2E_PATH, "utf8"); + const publishedRunner = readFileSync(UPGRADE_SURVIVOR_RUN_SCRIPT, "utf8"); + const updateRestartAuth = readFileSync(UPGRADE_SURVIVOR_UPDATE_RESTART_AUTH_PATH, "utf8"); + + for (const script of [publishedRunner, updateRestartAuth]) { + expectTextToIncludeAll(script, [ + 'supervisor_script="${pid_file}.supervisor.mjs"', + 'process_state="$(awk \'{ print $3 }\' "/proc/$pid/stat" 2>/dev/null || true)"', + 'OPENCLAW_SYSTEMCTL_SHIM_EXEC_START="$exec_start"', + 'nohup node "$supervisor_script"', + 'if (key.startsWith("OPENCLAW_UPDATE_")) {', + "delete childEnv.OPENCLAW_COMPATIBILITY_HOST_VERSION;", + 'process.on("SIGTERM", stop);', + "const stopTimeoutMs = 30_000;", + "process.kill(-pid, signal);", + 'signalProcessGroup(pid, "SIGTERM");', + 'signalProcessGroup(pid, "SIGKILL");', + "drainProcessGroup(childGroupPid, () => {", + "detached: true,", + "if (code === 78) return finish();", + "const restartDelayMs = 5_000;", + "const restartWindowMs = 60_000;", + "const restartBurst = 5;", + "if (starts.length >= restartBurst) {", + "setTimeout(start, restartDelayMs);", + "for _ in $(seq 1 350)", + ]); + } + for (const script of [runner, publishedRunner]) { + expect(script).toContain("systemctl --user stop openclaw-gateway.service"); + } + }); + + it("stops supervised gateway restarts after the systemd burst limit", async () => { + const workDir = tempDirs.make("openclaw-update-restart-supervisor-"); + const scripts = [ + readFileSync(UPGRADE_SURVIVOR_RUN_SCRIPT, "utf8"), + readFileSync(UPGRADE_SURVIVOR_UPDATE_RESTART_AUTH_PATH, "utf8"), + ]; + + for (const [index, script] of scripts.entries()) { + const supervisorPath = join(workDir, `supervisor-${index}.mjs`); + const countPath = join(workDir, `starts-${index}`); + const logPath = join(workDir, `daemon-${index}.log`); + const source = extractUpgradeSurvivorSupervisor(script) + .replace("const restartDelayMs = 5_000;", "const restartDelayMs = 5;") + .replace("const restartWindowMs = 60_000;", "const restartWindowMs = 5_000;"); + writeFileSync(supervisorPath, source); + + const command = + 'node -e \'require("node:fs").appendFileSync(process.env.COUNT_FILE, "x"); process.exit(1)\''; + const supervisor = spawn(process.execPath, [supervisorPath], { + env: { + ...process.env, + COUNT_FILE: countPath, + OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: logPath, + OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: command, + }, + stdio: "ignore", + }); + const exitCode = await waitForProcessExit(supervisor); + + expect(exitCode).toBe(0); + expect(readFileSync(countPath, "utf8")).toBe("xxxxx"); + expect(readFileSync(logPath, "utf8")).toContain( + "[systemctl-shim] gateway restart limit reached", + ); + } + }); + + it("allows a supervised gateway to drain within the systemd stop timeout", async () => { + const workDir = tempDirs.make("openclaw-update-restart-graceful-stop-"); + const scripts = [ + readFileSync(UPGRADE_SURVIVOR_RUN_SCRIPT, "utf8"), + readFileSync(UPGRADE_SURVIVOR_UPDATE_RESTART_AUTH_PATH, "utf8"), + ]; + + for (const [index, script] of scripts.entries()) { + const supervisorPath = join(workDir, `graceful-supervisor-${index}.mjs`); + const statePath = join(workDir, `graceful-state-${index}`); + const logPath = join(workDir, `graceful-daemon-${index}.log`); + const source = extractUpgradeSurvivorSupervisor(script).replace( + "const stopTimeoutMs = 30_000;", + "const stopTimeoutMs = 200;", + ); + writeFileSync(supervisorPath, source); + + const command = + 'node -e \'const fs=require("node:fs"); process.on("SIGTERM",()=>setTimeout(()=>{fs.appendFileSync(process.env.STATE_FILE, "-graceful"); process.exit(0)},50)); fs.writeFileSync(process.env.STATE_FILE, "ready"); setInterval(()=>{},1000)\''; + const supervisor = spawn(process.execPath, [supervisorPath], { + env: { + ...process.env, + OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: logPath, + OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: command, + STATE_FILE: statePath, + }, + stdio: "ignore", + }); + try { + for (let attempt = 0; attempt < 100 && !existsSync(statePath); attempt += 1) { + await delay(10); + } + expect(existsSync(statePath)).toBe(true); + + supervisor.kill("SIGTERM"); + expect(await waitForProcessExit(supervisor)).toBe(0); + expect(readFileSync(statePath, "utf8")).toBe("ready-graceful"); + } finally { + if (supervisor.exitCode === null && supervisor.signalCode === null) { + supervisor.kill("SIGTERM"); + await waitForProcessExit(supervisor).catch(() => undefined); + } + } + } + }); + + it.skipIf(process.platform === "win32")( + "terminates supervised gateway descendants at the systemd stop timeout", + async () => { + const workDir = tempDirs.make("openclaw-update-restart-process-group-"); + const descendantPath = writeTermIgnoringDescendant(workDir); + const gatewayPath = join(workDir, "gateway.mjs"); + writeFileSync( + gatewayPath, + `import fs from "node:fs"; +import { spawn } from "node:child_process"; +process.on("SIGTERM", () => { + setTimeout(() => { + fs.appendFileSync(process.env.STATE_FILE, "-graceful"); + process.exit(0); + }, 50); +}); +spawn(process.execPath, [process.env.DESCENDANT_SCRIPT], { stdio: "ignore" }); +const ready = setInterval(() => { + if (!fs.existsSync(process.env.DESCENDANT_PID_FILE)) return; + clearInterval(ready); + fs.writeFileSync(process.env.STATE_FILE, "ready"); +}, 5); +setInterval(() => {}, 1_000); +`, + ); + const scripts = [ + readFileSync(UPGRADE_SURVIVOR_RUN_SCRIPT, "utf8"), + readFileSync(UPGRADE_SURVIVOR_UPDATE_RESTART_AUTH_PATH, "utf8"), + ]; + + for (const [index, script] of scripts.entries()) { + const supervisorPath = join(workDir, `process-group-supervisor-${index}.mjs`); + const statePath = join(workDir, `process-group-state-${index}`); + const descendantPidPath = join(workDir, `process-group-descendant-${index}.pid`); + const logPath = join(workDir, `process-group-daemon-${index}.log`); + const source = extractUpgradeSurvivorSupervisor(script).replace( + "const stopTimeoutMs = 30_000;", + "const stopTimeoutMs = 200;", + ); + writeFileSync(supervisorPath, source); + + const supervisor = spawn(process.execPath, [supervisorPath], { + env: { + ...process.env, + DESCENDANT_PID_FILE: descendantPidPath, + DESCENDANT_SCRIPT: descendantPath, + OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: logPath, + OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: `${shellQuote(process.execPath)} ${shellQuote(gatewayPath)}`, + STATE_FILE: statePath, + }, + stdio: "ignore", + }); + let descendantPid: number | undefined; + try { + for (let attempt = 0; attempt < 100 && !existsSync(statePath); attempt += 1) { + await delay(10); + } + expect(existsSync(statePath)).toBe(true); + descendantPid = Number.parseInt(readFileSync(descendantPidPath, "utf8"), 10); + expect(descendantPid).toBeGreaterThan(1); + expect(isProcessRunning(descendantPid)).toBe(true); + + supervisor.kill("SIGTERM"); + expect(await waitForProcessExit(supervisor)).toBe(0); + expect(readFileSync(statePath, "utf8")).toBe("ready-graceful"); + for (let attempt = 0; attempt < 100 && isProcessRunning(descendantPid); attempt += 1) { + await delay(10); + } + expect(isProcessRunning(descendantPid)).toBe(false); + } finally { + if (supervisor.exitCode === null && supervisor.signalCode === null) { + supervisor.kill("SIGTERM"); + await waitForProcessExit(supervisor).catch(() => undefined); + } + if (descendantPid !== undefined && isProcessRunning(descendantPid)) { + try { + process.kill(descendantPid, "SIGKILL"); + } catch {} + } + } + } + }, + ); + + it.skipIf(process.platform === "win32")( + "drains the previous gateway process group before restarting", + async () => { + const workDir = tempDirs.make("openclaw-update-restart-process-group-restart-"); + const descendantPath = writeTermIgnoringDescendant(workDir); + const gatewayPath = join(workDir, "restart-gateway.mjs"); + writeFileSync( + gatewayPath, + `import fs from "node:fs"; +import { spawn } from "node:child_process"; +fs.appendFileSync(process.env.STARTS_FILE, "x"); +const starts = fs.readFileSync(process.env.STARTS_FILE, "utf8").length; +if (starts === 1) { + spawn(process.execPath, [process.env.DESCENDANT_SCRIPT], { stdio: "ignore" }); + const ready = setInterval(() => { + if (!fs.existsSync(process.env.DESCENDANT_PID_FILE)) return; + clearInterval(ready); + process.exit(1); + }, 5); + setInterval(() => {}, 1_000); +} else { + const pid = Number.parseInt(fs.readFileSync(process.env.DESCENDANT_PID_FILE, "utf8"), 10); + let running = false; + try { + process.kill(pid, 0); + running = true; + const statPath = "/proc/" + pid + "/stat"; + if (fs.existsSync(statPath)) running = fs.readFileSync(statPath, "utf8").split(" ")[2] !== "Z"; + } catch {} + fs.writeFileSync(process.env.REPLACEMENT_FILE, running ? "overlap" : "drained"); + process.exit(78); +} +`, + ); + const scripts = [ + readFileSync(UPGRADE_SURVIVOR_RUN_SCRIPT, "utf8"), + readFileSync(UPGRADE_SURVIVOR_UPDATE_RESTART_AUTH_PATH, "utf8"), + ]; + + for (const [index, script] of scripts.entries()) { + const supervisorPath = join(workDir, `restart-group-supervisor-${index}.mjs`); + const startsPath = join(workDir, `restart-group-starts-${index}`); + const descendantPidPath = join(workDir, `restart-group-descendant-${index}.pid`); + const replacementPath = join(workDir, `restart-group-replacement-${index}`); + const logPath = join(workDir, `restart-group-daemon-${index}.log`); + const source = extractUpgradeSurvivorSupervisor(script) + .replace("const restartDelayMs = 5_000;", "const restartDelayMs = 5;") + .replace("const stopTimeoutMs = 30_000;", "const stopTimeoutMs = 200;"); + writeFileSync(supervisorPath, source); + + const supervisor = spawn(process.execPath, [supervisorPath], { + env: { + ...process.env, + DESCENDANT_PID_FILE: descendantPidPath, + DESCENDANT_SCRIPT: descendantPath, + OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG: logPath, + OPENCLAW_SYSTEMCTL_SHIM_EXEC_START: `${shellQuote(process.execPath)} ${shellQuote(gatewayPath)}`, + REPLACEMENT_FILE: replacementPath, + STARTS_FILE: startsPath, + }, + stdio: "ignore", + }); + let descendantPid: number | undefined; + try { + expect(await waitForProcessExit(supervisor)).toBe(0); + descendantPid = Number.parseInt(readFileSync(descendantPidPath, "utf8"), 10); + expect(descendantPid).toBeGreaterThan(1); + expect(readFileSync(startsPath, "utf8")).toBe("xx"); + expect(readFileSync(replacementPath, "utf8")).toBe("drained"); + } finally { + if (supervisor.exitCode === null && supervisor.signalCode === null) { + supervisor.kill("SIGTERM"); + await waitForProcessExit(supervisor).catch(() => undefined); + } + if (descendantPid !== undefined && isProcessRunning(descendantPid)) { + try { + process.kill(descendantPid, "SIGKILL"); + } catch {} + } + } + } + }, + ); + it.each([ ["start budget", "OPENCLAW_UPGRADE_SURVIVOR_START_BUDGET_SECONDS", "90s"], ["status budget", "OPENCLAW_UPGRADE_SURVIVOR_STATUS_BUDGET_SECONDS", "30s"], @@ -2652,6 +2987,7 @@ fi expect(runner).not.toContain('cat "$GATEWAY_LOG"'); expect(runner).not.toContain('cat "$SYSTEMCTL_SHIM_DAEMON_LOG"'); expect(runner).not.toContain('cat "$log_file"'); + expect(runner).not.toContain('openclaw_e2e_print_log "$SYSTEMCTL_SHIM_LOG"'); expect(publishedRunner).toContain('openclaw_e2e_print_log "$BASELINE_INSTALL_LOG"'); expect(publishedRunner).toContain('openclaw_e2e_print_log "$BASELINE_CONFIG_VALIDATE_LOG"'); @@ -2675,6 +3011,8 @@ fi expect(publishedRunner).not.toContain('cat "$STATUS_ERR"'); expect(publishedRunner).not.toContain('cat "$STATUS_JSON"'); expect(publishedRunner).not.toContain('cat "$log_file"'); + expect(publishedRunner).not.toContain('openclaw_e2e_print_log "$SYSTEMCTL_SHIM_LOG"'); + expect(publishedRunner).not.toContain('openclaw_e2e_print_log "$SYSTEMCTL_SHIM_DAEMON_LOG"'); }); it("preserves caller-owned file descriptors around harness runs", () => { @@ -3723,10 +4061,11 @@ source "$ROOT_DIR/scripts/lib/docker-e2e-logs.sh" const helper = readFileSync(DOCKER_E2E_PACKAGE_HELPER_PATH, "utf8"); expectTextToIncludeAll(helper, [ "--allow-unreleased-changelog", - '-v "$ROOT_DIR/scripts/windows-cmd-helpers.mjs:/app/scripts/windows-cmd-helpers.mjs:ro"', - '-v "$ROOT_DIR/packages/normalization-core/src:/app/packages/normalization-core/src:ro"', - '-v "$ROOT_DIR/test/e2e/qa-lab:/app/test/e2e/qa-lab:ro"', - '-v "$ROOT_DIR/test/helpers:/app/test/helpers:ro"', + 'local harness_root="${DOCKER_E2E_HARNESS_ROOT_DIR:-$ROOT_DIR}"', + '-v "$harness_root/scripts/windows-cmd-helpers.mjs:/app/scripts/windows-cmd-helpers.mjs:ro"', + '-v "$harness_root/packages/normalization-core/src:/app/packages/normalization-core/src:ro"', + '-v "$harness_root/test/e2e/qa-lab:/app/test/e2e/qa-lab:ro"', + '-v "$harness_root/test/helpers:/app/test/helpers:ro"', ]); }); diff --git a/test/scripts/docker-e2e-plan.test.ts b/test/scripts/docker-e2e-plan.test.ts index f7afc546f1ee..fbdf2ee12120 100644 --- a/test/scripts/docker-e2e-plan.test.ts +++ b/test/scripts/docker-e2e-plan.test.ts @@ -1672,6 +1672,12 @@ describe("scripts/lib/docker-e2e-plan", () => { expect( planFor({ selectedLaneNames: ["update-migration"] }).requiredPrepublishPluginPackages, ).toEqual(["@openclaw/discord"]); + const updateRestartLane = findLaneByName("update-restart-auth"); + expect(updateRestartLane?.prepublishPluginPackages).toEqual(["@openclaw/codex"]); + expect(requiredPrepublishPluginPackagesForLanes([updateRestartLane!])).toEqual([ + "@openclaw/codex", + "@openclaw/discord", + ]); const legacyFeishuPlan = planFor({ selectedLaneNames: ["published-upgrade-survivor"], upgradeSurvivorBaselines: "2026.3.13",