fix(ci): keep crabbox sync checkouts alive

CHANGELOG.md

@@ -45,6 +45,7 @@ Docs: https://docs.openclaw.ai
 
 ### Fixes
 
+- Release/CI/E2E: keep temporary full-sync checkouts alive while slow Crabbox leases boot, so sparse worktree runs do not lose their sync source before file-list generation.
 - Release/CI/E2E: normalize inherited Linux `C.UTF-8` locale settings before raw AWS macOS Crabbox bootstrap commands, avoiding macOS locale warnings during package-manager hydration.
 - Agents/providers: keep streaming tool-call argument parsing record-shaped when providers emit valid non-object JSON such as `null` or arrays.
 - Release/CI/E2E: reset incremental log readers when watched log files rotate without shrinking, so same-size replacements do not hide new readiness or RPC lines.

scripts/crabbox-wrapper.mjs

@@ -11,6 +11,7 @@ import {
   readdirSync,
   rmSync,
   statSync,
+  utimesSync,
   writeFileSync,
 } from "node:fs";
 import { homedir, tmpdir } from "node:os";
@@ -1939,30 +1940,58 @@ function fullCheckoutSyncRoot() {
 function prepareFullCheckoutForSync(options = {}) {
   const dir = mkdtempSync(resolve(fullCheckoutSyncRoot(), "openclaw-crabbox-sync-"));
   let active = false;
-  const add = gitOutput(["worktree", "add", "--detach", dir, "HEAD"]);
-  if (add.status !== 0) {
-    rmSync(dir, { recursive: true, force: true });
-    throw new Error(`git worktree add failed: ${add.text}`);
-  }
-  active = true;
 
-  const disableSparse = gitOutput(["-C", dir, "sparse-checkout", "disable"]);
-  if (disableSparse.status !== 0) {
-    cleanupFullCheckout(dir, active);
-    throw new Error(`git sparse-checkout disable failed: ${disableSparse.text}`);
-  }
+  function create() {
+    const add = gitOutput(["worktree", "add", "--detach", dir, "HEAD"]);
+    if (add.status !== 0) {
+      rmSync(dir, { recursive: true, force: true });
+      throw new Error(`git worktree add failed: ${add.text}`);
+    }
+    active = true;
 
-  if (options.changedGateBase) {
-    const reset = gitOutput(["-C", dir, "reset", "--mixed", "--quiet", options.changedGateBase]);
-    if (reset.status !== 0) {
+    const disableSparse = gitOutput(["-C", dir, "sparse-checkout", "disable"]);
+    if (disableSparse.status !== 0) {
       cleanupFullCheckout(dir, active);
-      throw new Error(`git reset for changed-gate sync failed: ${reset.text}`);
+      active = false;
+      throw new Error(`git sparse-checkout disable failed: ${disableSparse.text}`);
+    }
+
+    if (options.changedGateBase) {
+      const reset = gitOutput(["-C", dir, "reset", "--mixed", "--quiet", options.changedGateBase]);
+      if (reset.status !== 0) {
+        cleanupFullCheckout(dir, active);
+        active = false;
+        throw new Error(`git reset for changed-gate sync failed: ${reset.text}`);
+      }
     }
   }
 
+  create();
+
   return {
     dir,
     changedGateBase: options.changedGateBase ?? "",
+    restoreIfMissing() {
+      try {
+        if (statSync(dir).isDirectory()) {
+          return false;
+        }
+      } catch {
+        // Recreate below.
+      }
+
+      console.error(`[crabbox] temporary full checkout disappeared; recreating ${dir}`);
+      if (active) {
+        const remove = gitOutput(["worktree", "remove", "--force", dir]);
+        if (remove.status !== 0) {
+          console.error(`[crabbox] warning: git worktree remove failed for ${dir}: ${remove.text}`);
+        }
+        active = false;
+      }
+      rmSync(dir, { recursive: true, force: true });
+      create();
+      return true;
+    },
     cleanup() {
       cleanupFullCheckout(dir, active);
       active = false;
@@ -1970,6 +1999,33 @@ function prepareFullCheckoutForSync(options = {}) {
   };
 }
 
+function startFullCheckoutKeepalive(checkout) {
+  const refresh = () => {
+    try {
+      checkout.restoreIfMissing();
+      const now = new Date();
+      utimesSync(checkout.dir, now, now);
+    } catch (error) {
+      console.error(
+        `[crabbox] warning: failed to refresh temporary full checkout ${checkout.dir}: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  };
+
+  refresh();
+  const intervalMs = Number.parseInt(
+    process.env.OPENCLAW_CRABBOX_SYNC_KEEPALIVE_MS ?? "5000",
+    10,
+  );
+  if (!Number.isFinite(intervalMs) || intervalMs <= 0) {
+    return () => {};
+  }
+
+  const interval = setInterval(refresh, intervalMs);
+  interval.unref?.();
+  return () => clearInterval(interval);
+}
+
 function cleanupFullCheckout(dir, active) {
   if (active) {
     const remove = gitOutput(["worktree", "remove", "--force", dir]);
@@ -1981,6 +2037,20 @@ function cleanupFullCheckout(dir, active) {
   rmSync(dir, { recursive: true, force: true });
 }
 
+function assertFullCheckoutAvailableBeforeExit(dir) {
+  try {
+    if (statSync(dir).isDirectory()) {
+      return;
+    }
+  } catch {
+    // Report below.
+  }
+
+  console.error(
+    `[crabbox] temporary full checkout vanished before Crabbox finished syncing: ${dir}`,
+  );
+}
+
 const version = checkedOutput(binary, ["--version"]);
 const help = checkedOutput(binary, ["run", "--help"]);
 const providerAliases = new Map([
@@ -2138,6 +2208,8 @@ if (canonicalProvider === "blacksmith-testbox") {
 
 let childCwd = repoRoot;
 let cleanupChildCwd = () => {};
+let fullCheckout = null;
+let stopFullCheckoutKeepalive = () => {};
 let cleanupDone = false;
 let remoteChangedGateBase = "";
 const scriptBootstrap = prepareAwsMacosScriptStdinBootstrap(normalizedArgs, provider);
@@ -2148,6 +2220,7 @@ try {
     const runWords = runCommandArgs(normalizedArgs);
     const changedGateBase = isChangedGateCommand(runWords) ? mergeBaseForChangedGate() : "";
     const checkout = prepareFullCheckoutForSync({ changedGateBase });
+    fullCheckout = checkout;
     childCwd = checkout.dir;
     cleanupChildCwd = () => checkout.cleanup();
     remoteChangedGateBase = checkout.changedGateBase;
@@ -2170,6 +2243,7 @@ function cleanupOnce() {
     return;
   }
   cleanupDone = true;
+  stopFullCheckoutKeepalive();
   scriptBootstrap.cleanup();
   preserveTemporaryCrabboxRuns();
   cleanupChildCwd();
@@ -2233,6 +2307,9 @@ const childArgs =
         ),
         remoteChangedGateBase,
       );
+if (fullCheckout) {
+  stopFullCheckoutKeepalive = startFullCheckoutKeepalive(fullCheckout);
+}
 const childInvocation = spawnInvocation(binary, childArgs, childEnv, process.platform);
 const child = spawn(childInvocation.command, childInvocation.args, {
   cwd: childCwd,
@@ -2258,6 +2335,9 @@ for (const signal of signalExitCodes.keys()) {
 process.once("exit", cleanupOnce);
 
 child.on("exit", (code, signal) => {
+  if (fullCheckout) {
+    assertFullCheckoutAvailableBeforeExit(fullCheckout.dir);
+  }
   cleanupOnce();
   if (signal) {
     process.exit(signalExitCodes.get(signal) ?? 1);
@@ -2267,6 +2347,9 @@ child.on("exit", (code, signal) => {
 });
 
 child.on("error", (error) => {
+  if (fullCheckout) {
+    assertFullCheckoutAvailableBeforeExit(fullCheckout.dir);
+  }
   cleanupOnce();
   console.error(`[crabbox] failed to execute ${displayBinary}: ${error.message}`);
   process.exit(2);

test/scripts/crabbox-wrapper.test.ts

@@ -85,6 +85,21 @@ function writeFakeCrabbox(binDir: string, helpText: string): string {
       "  fi",
       '  previous_arg="$arg"',
       "done",
+      'if [ "${OPENCLAW_FAKE_CRABBOX_DELETE_CWD_ONCE:-}" = "1" ]; then',
+      '  deleted_cwd="$PWD"',
+      "  cd / || exit 1",
+      '  rm -rf "$deleted_cwd"',
+      "  deadline=100",
+      '  while [ "$deadline" -gt 0 ] && [ ! -d "$deleted_cwd" ]; do',
+      "    deadline=$((deadline - 1))",
+      "    sleep 0.01",
+      "  done",
+      '  if [ ! -d "$deleted_cwd" ]; then',
+      '    printf "%s\\n" "cwd was not restored: $deleted_cwd" >&2',
+      "    exit 66",
+      "  fi",
+      '  cd "$deleted_cwd" || exit 1',
+      "fi",
       'printf "%s\\0" "__OPENCLAW_FAKE_CRABBOX_V1__"',
       'printf "%s\\0" "$PWD"',
       'printf "%s\\0" "$#"',
@@ -2477,6 +2492,31 @@ describe.concurrent("scripts/crabbox-wrapper", () => {
     }
   });
 
+  (process.platform === "win32" ? it.skip : it)(
+    "recreates sparse-sync temporary full checkouts that disappear while Crabbox is running",
+    () => {
+      const result = runWrapper(
+        "provider: hetzner, aws, local-container, blacksmith-testbox, or cloudflare\n",
+        ["run", "--provider", "aws", "--", "echo ok"],
+        {
+          env: {
+            OPENCLAW_CRABBOX_SYNC_KEEPALIVE_MS: "10",
+            OPENCLAW_FAKE_CRABBOX_DELETE_CWD_ONCE: "1",
+          },
+          gitResponses: {
+            [GIT_CONFIG_SPARSE_KEY]: { stdout: "true\n" },
+            [GIT_STATUS_PORCELAIN_KEY]: { stdout: "" },
+          },
+        },
+      );
+
+      const output = parseFakeCrabboxOutput(result);
+      expect(result.status).toBe(0);
+      expect(result.stderr).toContain("temporary full checkout disappeared; recreating");
+      expect(output.cwd).toContain("openclaw-crabbox-sync-");
+    },
+  );
+
   it("uses a temporary full checkout when existing AWS leases sync clean sparse worktrees", () => {
     const result = runWrapper(
       "provider: hetzner, aws, local-container, blacksmith-testbox, or cloudflare\n",