diff --git a/CHANGELOG.md b/CHANGELOG.md index 94994bc341a6..a44414726e3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- Release/CI/E2E: keep temporary full-sync checkouts alive while slow Crabbox leases boot, so sparse worktree runs do not lose their sync source before file-list generation. - Release/CI/E2E: normalize inherited Linux `C.UTF-8` locale settings before raw AWS macOS Crabbox bootstrap commands, avoiding macOS locale warnings during package-manager hydration. - Agents/providers: keep streaming tool-call argument parsing record-shaped when providers emit valid non-object JSON such as `null` or arrays. - Release/CI/E2E: reset incremental log readers when watched log files rotate without shrinking, so same-size replacements do not hide new readiness or RPC lines. diff --git a/scripts/crabbox-wrapper.mjs b/scripts/crabbox-wrapper.mjs index 0aa7c9f6b199..da468306311f 100755 --- a/scripts/crabbox-wrapper.mjs +++ b/scripts/crabbox-wrapper.mjs @@ -11,6 +11,7 @@ import { readdirSync, rmSync, statSync, + utimesSync, writeFileSync, } from "node:fs"; import { homedir, tmpdir } from "node:os"; @@ -1939,30 +1940,58 @@ function fullCheckoutSyncRoot() { function prepareFullCheckoutForSync(options = {}) { const dir = mkdtempSync(resolve(fullCheckoutSyncRoot(), "openclaw-crabbox-sync-")); let active = false; - const add = gitOutput(["worktree", "add", "--detach", dir, "HEAD"]); - if (add.status !== 0) { - rmSync(dir, { recursive: true, force: true }); - throw new Error(`git worktree add failed: ${add.text}`); - } - active = true; - const disableSparse = gitOutput(["-C", dir, "sparse-checkout", "disable"]); - if (disableSparse.status !== 0) { - cleanupFullCheckout(dir, active); - throw new Error(`git sparse-checkout disable failed: ${disableSparse.text}`); - } + function create() { + const add = gitOutput(["worktree", "add", "--detach", dir, "HEAD"]); + if (add.status !== 0) { + rmSync(dir, { recursive: true, force: true }); + throw new Error(`git worktree add failed: ${add.text}`); + } + active = true; - if (options.changedGateBase) { - const reset = gitOutput(["-C", dir, "reset", "--mixed", "--quiet", options.changedGateBase]); - if (reset.status !== 0) { + const disableSparse = gitOutput(["-C", dir, "sparse-checkout", "disable"]); + if (disableSparse.status !== 0) { cleanupFullCheckout(dir, active); - throw new Error(`git reset for changed-gate sync failed: ${reset.text}`); + active = false; + throw new Error(`git sparse-checkout disable failed: ${disableSparse.text}`); + } + + if (options.changedGateBase) { + const reset = gitOutput(["-C", dir, "reset", "--mixed", "--quiet", options.changedGateBase]); + if (reset.status !== 0) { + cleanupFullCheckout(dir, active); + active = false; + throw new Error(`git reset for changed-gate sync failed: ${reset.text}`); + } } } + create(); + return { dir, changedGateBase: options.changedGateBase ?? "", + restoreIfMissing() { + try { + if (statSync(dir).isDirectory()) { + return false; + } + } catch { + // Recreate below. + } + + console.error(`[crabbox] temporary full checkout disappeared; recreating ${dir}`); + if (active) { + const remove = gitOutput(["worktree", "remove", "--force", dir]); + if (remove.status !== 0) { + console.error(`[crabbox] warning: git worktree remove failed for ${dir}: ${remove.text}`); + } + active = false; + } + rmSync(dir, { recursive: true, force: true }); + create(); + return true; + }, cleanup() { cleanupFullCheckout(dir, active); active = false; @@ -1970,6 +1999,33 @@ function prepareFullCheckoutForSync(options = {}) { }; } +function startFullCheckoutKeepalive(checkout) { + const refresh = () => { + try { + checkout.restoreIfMissing(); + const now = new Date(); + utimesSync(checkout.dir, now, now); + } catch (error) { + console.error( + `[crabbox] warning: failed to refresh temporary full checkout ${checkout.dir}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + }; + + refresh(); + const intervalMs = Number.parseInt( + process.env.OPENCLAW_CRABBOX_SYNC_KEEPALIVE_MS ?? "5000", + 10, + ); + if (!Number.isFinite(intervalMs) || intervalMs <= 0) { + return () => {}; + } + + const interval = setInterval(refresh, intervalMs); + interval.unref?.(); + return () => clearInterval(interval); +} + function cleanupFullCheckout(dir, active) { if (active) { const remove = gitOutput(["worktree", "remove", "--force", dir]); @@ -1981,6 +2037,20 @@ function cleanupFullCheckout(dir, active) { rmSync(dir, { recursive: true, force: true }); } +function assertFullCheckoutAvailableBeforeExit(dir) { + try { + if (statSync(dir).isDirectory()) { + return; + } + } catch { + // Report below. + } + + console.error( + `[crabbox] temporary full checkout vanished before Crabbox finished syncing: ${dir}`, + ); +} + const version = checkedOutput(binary, ["--version"]); const help = checkedOutput(binary, ["run", "--help"]); const providerAliases = new Map([ @@ -2138,6 +2208,8 @@ if (canonicalProvider === "blacksmith-testbox") { let childCwd = repoRoot; let cleanupChildCwd = () => {}; +let fullCheckout = null; +let stopFullCheckoutKeepalive = () => {}; let cleanupDone = false; let remoteChangedGateBase = ""; const scriptBootstrap = prepareAwsMacosScriptStdinBootstrap(normalizedArgs, provider); @@ -2148,6 +2220,7 @@ try { const runWords = runCommandArgs(normalizedArgs); const changedGateBase = isChangedGateCommand(runWords) ? mergeBaseForChangedGate() : ""; const checkout = prepareFullCheckoutForSync({ changedGateBase }); + fullCheckout = checkout; childCwd = checkout.dir; cleanupChildCwd = () => checkout.cleanup(); remoteChangedGateBase = checkout.changedGateBase; @@ -2170,6 +2243,7 @@ function cleanupOnce() { return; } cleanupDone = true; + stopFullCheckoutKeepalive(); scriptBootstrap.cleanup(); preserveTemporaryCrabboxRuns(); cleanupChildCwd(); @@ -2233,6 +2307,9 @@ const childArgs = ), remoteChangedGateBase, ); +if (fullCheckout) { + stopFullCheckoutKeepalive = startFullCheckoutKeepalive(fullCheckout); +} const childInvocation = spawnInvocation(binary, childArgs, childEnv, process.platform); const child = spawn(childInvocation.command, childInvocation.args, { cwd: childCwd, @@ -2258,6 +2335,9 @@ for (const signal of signalExitCodes.keys()) { process.once("exit", cleanupOnce); child.on("exit", (code, signal) => { + if (fullCheckout) { + assertFullCheckoutAvailableBeforeExit(fullCheckout.dir); + } cleanupOnce(); if (signal) { process.exit(signalExitCodes.get(signal) ?? 1); @@ -2267,6 +2347,9 @@ child.on("exit", (code, signal) => { }); child.on("error", (error) => { + if (fullCheckout) { + assertFullCheckoutAvailableBeforeExit(fullCheckout.dir); + } cleanupOnce(); console.error(`[crabbox] failed to execute ${displayBinary}: ${error.message}`); process.exit(2); diff --git a/test/scripts/crabbox-wrapper.test.ts b/test/scripts/crabbox-wrapper.test.ts index a1c7533e3928..7402ff28bbce 100644 --- a/test/scripts/crabbox-wrapper.test.ts +++ b/test/scripts/crabbox-wrapper.test.ts @@ -85,6 +85,21 @@ function writeFakeCrabbox(binDir: string, helpText: string): string { " fi", ' previous_arg="$arg"', "done", + 'if [ "${OPENCLAW_FAKE_CRABBOX_DELETE_CWD_ONCE:-}" = "1" ]; then', + ' deleted_cwd="$PWD"', + " cd / || exit 1", + ' rm -rf "$deleted_cwd"', + " deadline=100", + ' while [ "$deadline" -gt 0 ] && [ ! -d "$deleted_cwd" ]; do', + " deadline=$((deadline - 1))", + " sleep 0.01", + " done", + ' if [ ! -d "$deleted_cwd" ]; then', + ' printf "%s\\n" "cwd was not restored: $deleted_cwd" >&2', + " exit 66", + " fi", + ' cd "$deleted_cwd" || exit 1', + "fi", 'printf "%s\\0" "__OPENCLAW_FAKE_CRABBOX_V1__"', 'printf "%s\\0" "$PWD"', 'printf "%s\\0" "$#"', @@ -2477,6 +2492,31 @@ describe.concurrent("scripts/crabbox-wrapper", () => { } }); + (process.platform === "win32" ? it.skip : it)( + "recreates sparse-sync temporary full checkouts that disappear while Crabbox is running", + () => { + const result = runWrapper( + "provider: hetzner, aws, local-container, blacksmith-testbox, or cloudflare\n", + ["run", "--provider", "aws", "--", "echo ok"], + { + env: { + OPENCLAW_CRABBOX_SYNC_KEEPALIVE_MS: "10", + OPENCLAW_FAKE_CRABBOX_DELETE_CWD_ONCE: "1", + }, + gitResponses: { + [GIT_CONFIG_SPARSE_KEY]: { stdout: "true\n" }, + [GIT_STATUS_PORCELAIN_KEY]: { stdout: "" }, + }, + }, + ); + + const output = parseFakeCrabboxOutput(result); + expect(result.status).toBe(0); + expect(result.stderr).toContain("temporary full checkout disappeared; recreating"); + expect(output.cwd).toContain("openclaw-crabbox-sync-"); + }, + ); + it("uses a temporary full checkout when existing AWS leases sync clean sparse worktrees", () => { const result = runWrapper( "provider: hetzner, aws, local-container, blacksmith-testbox, or cloudflare\n",