From 69a406118cbbd96364bc3ab20031870a7e538d62 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 6 Jun 2026 17:37:01 +0200 Subject: [PATCH] fix(e2e): preserve docker cleanup failure artifacts --- scripts/docker-e2e-rerun.mjs | 88 +++++++------ scripts/test-docker-all.mjs | 140 +++++++++++++++++++-- test/scripts/docker-all-scheduler.test.ts | 81 +++++++++++- test/scripts/docker-e2e-helper-cli.test.ts | 49 ++++++++ 4 files changed, 312 insertions(+), 46 deletions(-) diff --git a/scripts/docker-e2e-rerun.mjs b/scripts/docker-e2e-rerun.mjs index faa21e043400..bb4d92c147de 100644 --- a/scripts/docker-e2e-rerun.mjs +++ b/scripts/docker-e2e-rerun.mjs @@ -161,6 +161,27 @@ function ghWorkflowCommand(lanes, ref, workflow, reuseInputs = {}) { return fields.join(" "); } +function failureName(failure) { + return failure.name || failure.lane || ""; +} + +function failedEntryFromRecord(failure, file, ref, workflow, reuseInputs) { + const lane = failureName(failure); + const targetable = failure.targetable !== false; + return { + ghWorkflowCommand: targetable + ? failure.ghWorkflowCommand || ghWorkflowCommand([lane], ref, workflow, reuseInputs) + : "", + lane, + localRerunCommand: failure.rerunCommand, + logFile: failure.logFile, + reuseInputs, + source: file, + status: failure.status, + targetable, + }; +} + function detectRepo() { return run("gh", ["repo", "view", "--json", "nameWithOwner", "--jq", ".nameWithOwner"]).trim(); } @@ -183,31 +204,18 @@ function failedLaneEntriesFromJson(file, ref, workflow) { const source = path.basename(file); if (source === "failures.json" && Array.isArray(parsed.lanes)) { return parsed.lanes - .filter((lane) => lane.name) - .map((lane) => ({ - ghWorkflowCommand: - lane.ghWorkflowCommand || ghWorkflowCommand([lane.name], ref, workflow, reuseInputs), - lane: lane.name, - localRerunCommand: lane.rerunCommand, - logFile: lane.logFile, - reuseInputs, - source: file, - status: lane.status, - })); + .filter((lane) => failureName(lane)) + .map((lane) => failedEntryFromRecord(lane, file, ref, workflow, reuseInputs)); } const lanes = Array.isArray(parsed.lanes) ? parsed.lanes : []; - return lanes - .filter((lane) => lane.status !== 0 && lane.name) - .map((lane) => ({ - ghWorkflowCommand: ghWorkflowCommand([lane.name], ref, workflow, reuseInputs), - lane: lane.name, - localRerunCommand: lane.rerunCommand, - logFile: lane.logFile, - reuseInputs, - source: file, - status: lane.status, - })); + const failures = + Array.isArray(parsed.failures) && parsed.failures.length > 0 + ? parsed.failures + : lanes.filter((lane) => lane.status !== 0); + return failures + .filter((lane) => failureName(lane)) + .map((lane) => failedEntryFromRecord(lane, file, ref, workflow, reuseInputs)); } function mergeByLane(entries) { @@ -275,23 +283,29 @@ function printEntries(entries, ref, workflow, runValue) { console.log("No failed Docker E2E lanes found."); return; } - console.log(`Failed lanes: ${entries.map((entry) => entry.lane).join(", ")}`); - console.log(""); - console.log("Combined GitHub rerun:"); - console.log( - ghWorkflowCommand( - entries.map((entry) => entry.lane), - ref, - workflow, - commonReuseInputs(entries), - ), - ); - console.log(""); - console.log("Per-lane GitHub reruns:"); - for (const entry of entries) { + const workflowEntries = entries.filter((entry) => entry.targetable !== false); + console.log(`Failed Docker E2E entries: ${entries.map((entry) => entry.lane).join(", ")}`); + if (workflowEntries.length > 0) { + console.log(""); + console.log("Combined GitHub rerun:"); console.log( - `- ${entry.lane}: ${entry.ghWorkflowCommand || ghWorkflowCommand([entry.lane], ref, workflow)}`, + ghWorkflowCommand( + workflowEntries.map((entry) => entry.lane), + ref, + workflow, + commonReuseInputs(workflowEntries), + ), ); + console.log(""); + console.log("Per-lane GitHub reruns:"); + for (const entry of workflowEntries) { + console.log( + `- ${entry.lane}: ${entry.ghWorkflowCommand || ghWorkflowCommand([entry.lane], ref, workflow)}`, + ); + } + } else { + console.log(""); + console.log("No targetable failed Docker E2E lanes found."); } console.log(""); console.log("Local rerun starting points:"); diff --git a/scripts/test-docker-all.mjs b/scripts/test-docker-all.mjs index 5962cd770f29..77f3fa397483 100644 --- a/scripts/test-docker-all.mjs +++ b/scripts/test-docker-all.mjs @@ -40,6 +40,7 @@ const DEFAULT_LANE_TIMEOUT_MS = 120 * 60 * 1000; const DEFAULT_LANE_START_STAGGER_MS = 2_000; const DEFAULT_STATUS_INTERVAL_MS = 30_000; const DEFAULT_PREFLIGHT_RUN_TIMEOUT_MS = 60_000; +const CLEANUP_SMOKE_NAME = "cleanup-smoke"; export const SHELL_CAPTURE_MAX_CHARS = 1024 * 1024; export const LOG_TAIL_MAX_BYTES = 1024 * 1024; const DEFAULT_TIMINGS_FILE = path.join(ROOT_DIR, ".artifacts/docker-tests/lane-timings.json"); @@ -456,8 +457,10 @@ async function writeFailureIndex(logDir, summary) { const failures = Array.isArray(summary.failures) ? summary.failures : (summary.lanes ?? []).filter((lane) => lane.status !== 0); + const workflowRerunFailures = failures.filter((failure) => failure.targetable !== false); const lanes = failures.map((failure) => ({ - ghWorkflowCommand: githubWorkflowRerunCommand([failure.name], ref), + ghWorkflowCommand: + failure.targetable === false ? undefined : githubWorkflowRerunCommand([failure.name], ref), image: failure.image, imageKind: failure.imageKind, lane: failure.name, @@ -466,13 +469,14 @@ async function writeFailureIndex(logDir, summary) { noOutputTimedOut: failure.noOutputTimedOut, rerunCommand: failure.rerunCommand, status: failure.status, + targetable: failure.targetable, timedOut: failure.timedOut, })); const failureIndex = { combinedGhWorkflowCommand: - lanes.length > 0 + workflowRerunFailures.length > 0 ? githubWorkflowRerunCommand( - lanes.map((lane) => lane.lane), + workflowRerunFailures.map((failure) => failure.name), ref, ) : undefined, @@ -712,6 +716,85 @@ async function runForeground(label, command, env) { } } +async function recordCleanupSmokeFailure(error, baseEnv, logDir, command, startedAtMs) { + const status = 1; + const logFile = path.join(logDir, `${CLEANUP_SMOKE_NAME}.log`); + const message = error instanceof Error ? error.message : String(error); + await fs.promises.writeFile( + logFile, + [ + `==> [${CLEANUP_SMOKE_NAME}] command: ${command}`, + `==> [${CLEANUP_SMOKE_NAME}] status: ${status}`, + `==> [${CLEANUP_SMOKE_NAME}] error: ${message}`, + ] + .filter(Boolean) + .join("\n"), + ); + return { + command, + attempts: [ + { + attempt: 1, + elapsedSeconds: phaseElapsedSeconds(startedAtMs), + finishedAt: new Date().toISOString(), + noOutputTimedOut: false, + startedAt: new Date(startedAtMs).toISOString(), + status, + timedOut: false, + }, + ], + elapsedSeconds: phaseElapsedSeconds(startedAtMs), + finishedAt: new Date().toISOString(), + image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE, + logFile, + name: CLEANUP_SMOKE_NAME, + noOutputTimedOut: false, + rerunCommand: command, + startedAt: new Date(startedAtMs).toISOString(), + status, + targetable: false, + timedOut: false, + }; +} + +async function runCleanupSmoke(baseEnv, logDir, command, startedAtMs) { + const logFile = path.join(logDir, `${CLEANUP_SMOKE_NAME}.log`); + const result = await runShellCommand({ + command, + env: baseEnv, + label: CLEANUP_SMOKE_NAME, + logFile, + }); + if (result.status === 0) { + return undefined; + } + return { + command, + attempts: [ + { + attempt: 1, + elapsedSeconds: phaseElapsedSeconds(startedAtMs), + finishedAt: new Date().toISOString(), + noOutputTimedOut: result.noOutputTimedOut, + startedAt: new Date(startedAtMs).toISOString(), + status: result.status, + timedOut: result.timedOut, + }, + ], + elapsedSeconds: phaseElapsedSeconds(startedAtMs), + finishedAt: new Date().toISOString(), + image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE, + logFile, + name: CLEANUP_SMOKE_NAME, + noOutputTimedOut: result.noOutputTimedOut, + rerunCommand: command, + startedAt: new Date(startedAtMs).toISOString(), + status: result.status, + targetable: false, + timedOut: result.timedOut, + }; +} + async function runForegroundGroup(entries, env) { const failures = []; for (const entry of entries) { @@ -1469,17 +1552,58 @@ async function main() { } if (profile === DEFAULT_PROFILE && selectedLaneNames.length === 0) { - await runPhase(phases, "cleanup-smoke", {}, async () => { - await runForeground( - "Run cleanup smoke after parallel lanes", - "pnpm test:docker:cleanup", + const cleanupSmokeCommand = "pnpm test:docker:cleanup"; + const cleanupStartedAtMs = Date.now(); + let cleanupFailure; + try { + await runPhase(phases, CLEANUP_SMOKE_NAME, {}, async () => { + cleanupFailure = await runCleanupSmoke( + baseEnv, + logDir, + cleanupSmokeCommand, + cleanupStartedAtMs, + ); + if (cleanupFailure) { + throw new Error( + `Run cleanup smoke after parallel lanes failed with status ${cleanupFailure.status}`, + ); + } + }); + } catch (error) { + cleanupFailure ??= await recordCleanupSmokeFailure( + error, baseEnv, + logDir, + cleanupSmokeCommand, + cleanupStartedAtMs, ); - }); + } + if (cleanupFailure) { + failures.push(cleanupFailure); + } } else { console.log("==> Cleanup smoke after parallel lanes: skipped for selected/release lanes"); } await writeTimingStore(timingStore, allResults); + if (failures.length > 0) { + await writeRunSummary(logDir, { + chunk: releaseChunk || undefined, + failures, + image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE, + images: { + bare: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE, + functional: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE, + }, + lanes: allResults, + phases, + profile, + selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined, + startedAt: runStartedAt, + status: "failed", + }); + await printFailureSummary(failures, tailLines); + process.exit(1); + } await writeRunSummary(logDir, { chunk: releaseChunk || undefined, failures, diff --git a/test/scripts/docker-all-scheduler.test.ts b/test/scripts/docker-all-scheduler.test.ts index 59d8b97d9131..9147487580f0 100644 --- a/test/scripts/docker-all-scheduler.test.ts +++ b/test/scripts/docker-all-scheduler.test.ts @@ -1,6 +1,6 @@ // Docker All Scheduler tests cover docker all scheduler script behavior. import { spawnSync } from "node:child_process"; -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; @@ -177,6 +177,85 @@ describe("scripts/test-docker-all scheduler", () => { } }); + posixIt("writes Docker run artifacts when cleanup smoke fails", () => { + const root = mkdtempSync(`${tmpdir()}/openclaw-docker-all-cleanup-`); + const logDir = path.join(root, "logs"); + const packageTgz = path.join(root, "openclaw-current.tgz"); + const fakePnpm = path.join(root, "pnpm"); + writeFileSync(packageTgz, "fake package\n", "utf8"); + writeFileSync( + fakePnpm, + `#!/usr/bin/env node +const command = process.argv.slice(2).join(" "); +if (command === "test:docker:cleanup") { + console.error("cleanup smoke failed intentionally"); + process.exit(42); +} +process.exit(0); +`, + "utf8", + ); + chmodSync(fakePnpm, 0o755); + + try { + const result = spawnSync(process.execPath, ["scripts/test-docker-all.mjs"], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + OPENCLAW_CURRENT_PACKAGE_TGZ: packageTgz, + OPENCLAW_DOCKER_ALL_BUILD: "0", + OPENCLAW_DOCKER_ALL_LIVE_MODE: "skip", + OPENCLAW_DOCKER_ALL_LOG_DIR: logDir, + OPENCLAW_DOCKER_ALL_PARALLELISM: "16", + OPENCLAW_DOCKER_ALL_PREFLIGHT: "0", + OPENCLAW_DOCKER_ALL_START_STAGGER_MS: "0", + OPENCLAW_DOCKER_ALL_STATUS_INTERVAL_MS: "0", + OPENCLAW_DOCKER_ALL_TAIL_PARALLELISM: "16", + OPENCLAW_DOCKER_ALL_TIMINGS: "0", + PATH: `${root}${path.delimiter}${process.env.PATH ?? ""}`, + }, + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("cleanup smoke failed intentionally"); + + const summary = JSON.parse(readFileSync(path.join(logDir, "summary.json"), "utf8")); + expect(summary.status).toBe("failed"); + expect(summary.failures).toHaveLength(1); + expect(summary.failures[0]).toMatchObject({ + name: "cleanup-smoke", + rerunCommand: "pnpm test:docker:cleanup", + status: 42, + targetable: false, + }); + expect(summary.lanes.some((lane: { name?: string }) => lane.name === "cleanup-smoke")).toBe( + false, + ); + expect(summary.phases.at(-1)).toMatchObject({ + name: "cleanup-smoke", + status: "failed", + }); + + const failureIndex = JSON.parse(readFileSync(path.join(logDir, "failures.json"), "utf8")); + expect(failureIndex.status).toBe("failed"); + expect(failureIndex.combinedGhWorkflowCommand).toBeUndefined(); + expect(failureIndex.lanes[0]?.ghWorkflowCommand).toBeUndefined(); + expect(failureIndex.lanes).toEqual([ + expect.objectContaining({ + lane: "cleanup-smoke", + rerunCommand: "pnpm test:docker:cleanup", + status: 42, + targetable: false, + }), + ]); + const cleanupLog = readFileSync(path.join(logDir, "cleanup-smoke.log"), "utf8"); + expect(cleanupLog).toContain("cleanup smoke failed intentionally"); + } finally { + rmSync(root, { force: true, recursive: true }); + } + }); + it("allows an overweight lane to start alone under low parallelism", () => { expect( canStartSchedulerLane( diff --git a/test/scripts/docker-e2e-helper-cli.test.ts b/test/scripts/docker-e2e-helper-cli.test.ts index 5b3456258f9c..c65738f78992 100644 --- a/test/scripts/docker-e2e-helper-cli.test.ts +++ b/test/scripts/docker-e2e-helper-cli.test.ts @@ -1,5 +1,8 @@ // Docker E2E Helper Cli tests cover docker e2e helper cli script behavior. import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { describe, expect, it } from "vitest"; function runHelper(script: string, ...args: string[]) { @@ -62,4 +65,50 @@ describe("Docker E2E helper CLIs", () => { "node scripts/docker-e2e-rerun.mjs ", ); }); + + it.each(["summary.json", "failures.json"])( + "prints local cleanup reruns without synthesizing Docker lane reruns from %s", + (fileName) => { + const root = mkdtempSync(`${tmpdir()}/openclaw-docker-e2e-rerun-`); + try { + const cleanupFailure = { + lane: "cleanup-smoke", + logFile: "cleanup-smoke.log", + name: "cleanup-smoke", + rerunCommand: "pnpm test:docker:cleanup", + status: 42, + targetable: false, + }; + const payload = + fileName === "summary.json" + ? { + failures: [cleanupFailure], + lanes: [ + { + name: "gateway-network", + status: 0, + }, + ], + status: "failed", + } + : { + lanes: [cleanupFailure], + status: "failed", + }; + const file = path.join(root, fileName); + writeFileSync(file, `${JSON.stringify(payload, null, 2)}\n`, "utf8"); + + const result = runHelper("scripts/docker-e2e-rerun.mjs", file, "--ref", "abc123"); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(result.stdout).toContain("Failed Docker E2E entries: cleanup-smoke"); + expect(result.stdout).toContain("No targetable failed Docker E2E lanes found."); + expect(result.stdout).toContain("- cleanup-smoke: pnpm test:docker:cleanup"); + expect(result.stdout).not.toContain("docker_lanes='cleanup-smoke'"); + } finally { + rmSync(root, { force: true, recursive: true }); + } + }, + ); });