From 06b5f0b2ffee9afe421e436ce85c4d66faff506b Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 4 Aug 2026 06:54:42 +0800 Subject: [PATCH] test(qa): cover CLI onboarding and plugin install sources (#118882) * test(qa): cover CLI onboarding and plugin sources * test(qa): assert npm file plugin source * test(qa): redact onboarding password assertion * test(qa): retain onboarding evidence log --- qa/scenarios/cli/cli-onboarding.yaml | 39 +++++ .../cli/cli-plugin-install-sources.yaml | 33 ++++ scripts/e2e/lib/onboard/assert-config.mjs | 14 ++ scripts/e2e/lib/onboard/scenario.sh | 82 +++++++++- scripts/e2e/onboard-docker.sh | 6 + scripts/e2e/qa-cli-onboarding.mjs | 141 ++++++++++++++++++ scripts/e2e/qa-plugin-install-sources.mjs | 104 +++++++++++++ .../qa-lab/runtime/script-evidence.test.ts | 10 ++ test/e2e/qa-lab/runtime/script-evidence.ts | 10 +- test/scripts/onboard-config-fixtures.test.ts | 77 ++++++++++ test/scripts/qa-cli-onboarding.test.ts | 13 ++ .../scripts/qa-plugin-install-sources.test.ts | 14 ++ 12 files changed, 535 insertions(+), 8 deletions(-) create mode 100644 qa/scenarios/cli/cli-onboarding.yaml create mode 100644 qa/scenarios/cli/cli-plugin-install-sources.yaml create mode 100644 scripts/e2e/qa-cli-onboarding.mjs create mode 100644 scripts/e2e/qa-plugin-install-sources.mjs create mode 100644 test/scripts/qa-cli-onboarding.test.ts create mode 100644 test/scripts/qa-plugin-install-sources.test.ts diff --git a/qa/scenarios/cli/cli-onboarding.yaml b/qa/scenarios/cli/cli-onboarding.yaml new file mode 100644 index 000000000000..0224eb0bbb20 --- /dev/null +++ b/qa/scenarios/cli/cli-onboarding.yaml @@ -0,0 +1,39 @@ +title: CLI onboarding contracts + +scenario: + id: cli-onboarding + surface: cli + category: cli.onboarding-and-auth-setup + coverage: + primary: + - cli.gateway-auth-storage + - cli.guided-onboarding + - cli.remote-onboarding + - cli.targeted-reconfiguration + objective: Verify guided and non-interactive CLI onboarding across Gateway storage, remote setup, reset, and section-scoped reconfiguration. + successCriteria: + - Guided onboarding runs under a real PTY, verifies the configured model, reaches completion, and performs no UI or terminal handoff. + - Gateway auth proves token SecretRef storage and password storage through executable non-interactive setup. + - Remote onboarding persists the requested remote URL and token, and reset removes stale remote configuration. + - Section-scoped skills configuration records configure metadata without selecting a Gateway run mode. + - The selected onboarding cases exclude channel setup. + - QA evidence records a separate executable assertion entry for every primary coverage boundary. + docsRefs: + - docs/cli/onboard.md + - docs/cli/configure.md + codeRefs: + - src/commands/onboard.ts + - src/commands/onboard-non-interactive.ts + - src/commands/onboard-non-interactive/remote.ts + - src/commands/onboard-non-interactive/local/auth-choice.ts + - src/commands/onboard-non-interactive/local/gateway-config.ts + - src/commands/configure.commands.ts + - scripts/e2e/lib/onboard/scenario.sh + execution: + kind: script + path: scripts/e2e/qa-cli-onboarding.mjs + summary: Runs the executable onboarding Docker lane with only the guided, Gateway-storage, remote, reset, and skills cases selected. + timeoutMs: 1800000 + args: + - --artifact-base + - ${outputDir} diff --git a/qa/scenarios/cli/cli-plugin-install-sources.yaml b/qa/scenarios/cli/cli-plugin-install-sources.yaml new file mode 100644 index 000000000000..1a795377e866 --- /dev/null +++ b/qa/scenarios/cli/cli-plugin-install-sources.yaml @@ -0,0 +1,33 @@ +title: CLI plugin install sources + +scenario: + id: cli-plugin-install-sources + surface: cli + category: cli.plugin-and-channel-setup + coverage: + primary: + - cli.plugin-install-sources + objective: Verify the plugin CLI installs supported bundled, local, archive, npm, ClawHub, marketplace, file, and Git sources through executable commands. + successCriteria: + - The plugin CLI installs a bundled plugin, tarball archive, local directory, file package, npm package, and Git source. + - Local directory installation preserves vendored dependencies. + - Invalid npm metadata fails without creating an install record. + - Moving and exact Git references update and resolve to the expected commits. + - Hermetic ClawHub and marketplace fixtures install, update, inspect, and uninstall plugins. + - QA evidence fails when any required source-class assertion marker is absent. + docsRefs: + - docs/cli/plugins.md + codeRefs: + - src/cli/plugins-cli.ts + - src/plugins/install.ts + - src/plugins/install-source-info.ts + - scripts/e2e/lib/plugins/sweep.sh + - scripts/e2e/lib/bundled-plugin-install-uninstall/sweep.sh + execution: + kind: script + path: scripts/e2e/qa-plugin-install-sources.mjs + summary: Runs the hermetic plugin source sweep plus the packaged bundled-plugin install/uninstall lane and emits bounded QA evidence. + timeoutMs: 1800000 + args: + - --artifact-base + - ${outputDir} diff --git a/scripts/e2e/lib/onboard/assert-config.mjs b/scripts/e2e/lib/onboard/assert-config.mjs index 442948c4bbbc..b01825eb1d2b 100644 --- a/scripts/e2e/lib/onboard/assert-config.mjs +++ b/scripts/e2e/lib/onboard/assert-config.mjs @@ -44,6 +44,20 @@ switch (scenario) { expectEqual("wizard.lastRunCommand", cfg?.wizard?.lastRunCommand, "onboard"); break; } + case "local-auth-refs": + assertLocalWizard(); + expectEqual("gateway.auth.mode", cfg?.gateway?.auth?.mode, "token"); + expectEqual("gateway.auth.token.source", cfg?.gateway?.auth?.token?.source, "env"); + expectEqual("gateway.auth.token.provider", cfg?.gateway?.auth?.token?.provider, "default"); + expectEqual("gateway.auth.token.id", cfg?.gateway?.auth?.token?.id, "OPENCLAW_GATEWAY_TOKEN"); + break; + case "local-password": + assertLocalWizard(); + expectEqual("gateway.auth.mode", cfg?.gateway?.auth?.mode, "password"); + if (cfg?.gateway?.auth?.password !== "openclaw-onboard-password-e2e") { + errors.push("gateway.auth.password mismatch"); + } + break; case "remote-non-interactive": expectEqual("gateway.mode", cfg?.gateway?.mode, "remote"); expectEqual("gateway.remote.url", cfg?.gateway?.remote?.url, "ws://gateway.local:18789"); diff --git a/scripts/e2e/lib/onboard/scenario.sh b/scripts/e2e/lib/onboard/scenario.sh index 443b09603104..7576dfb8cbed 100644 --- a/scripts/e2e/lib/onboard/scenario.sh +++ b/scripts/e2e/lib/onboard/scenario.sh @@ -304,6 +304,7 @@ run_case_guided_skip_ui() { send_guided_skip_ui_flow validate_guided_skip_ui_log "$WIZARD_LOG_PATH" "$mock_request_log" + echo "QA_ASSERT cli.guided-onboarding pass" openclaw_e2e_stop_process "$mock_openai_pid" mock_openai_pid="" } @@ -336,6 +337,54 @@ run_case_local_basic() { } +run_case_local_auth_refs() { + set_isolated_openclaw_env local-auth-refs + export OPENAI_API_KEY="sk-openclaw-onboard-auth-ref-e2e" + export OPENCLAW_GATEWAY_TOKEN="openclaw-onboard-gateway-ref-e2e" + + openclaw_e2e_run_logged local-auth-refs node "$OPENCLAW_ENTRY" onboard \ + --non-interactive \ + --accept-risk \ + --flow quickstart \ + --mode local \ + --auth-choice openai-api-key \ + --secret-input-mode ref \ + --gateway-auth token \ + --gateway-token-ref-env OPENCLAW_GATEWAY_TOKEN \ + --skip-channels \ + --skip-skills \ + --skip-daemon \ + --skip-ui \ + --skip-health + + node scripts/e2e/lib/release-scenarios/assertions.mjs \ + assert-openai-env-ref \ + "$OPENAI_API_KEY" + assert_onboard_config local-auth-refs + echo "QA_ASSERT cli.gateway-auth-storage.token-ref pass" +} + +run_case_local_password() { + set_isolated_openclaw_env local-password + + openclaw_e2e_run_logged local-password node "$OPENCLAW_ENTRY" onboard \ + --non-interactive \ + --accept-risk \ + --flow quickstart \ + --mode local \ + --auth-choice skip \ + --gateway-auth password \ + --gateway-password "openclaw-onboard-password-e2e" \ + --skip-channels \ + --skip-skills \ + --skip-daemon \ + --skip-ui \ + --skip-health + + assert_onboard_config local-password + echo "QA_ASSERT cli.gateway-auth-storage.password pass" +} + run_case_remote_non_interactive() { set_isolated_openclaw_env remote-non-interactive # Smoke test non-interactive remote config write. @@ -347,6 +396,7 @@ run_case_remote_non_interactive() { --skip-health assert_onboard_config remote-non-interactive + echo "QA_ASSERT cli.remote-onboarding pass" } run_case_reset() { @@ -366,6 +416,7 @@ run_case_reset() { --skip-health assert_onboard_config reset + echo "QA_ASSERT cli.targeted-reconfiguration.reset pass" } run_case_channels() { @@ -384,6 +435,7 @@ run_case_skills() { run_wizard_cmd skills "$home_dir" "node \"$OPENCLAW_ENTRY\" configure --section skills" send_skills_flow assert_onboard_config skills + echo "QA_ASSERT cli.targeted-reconfiguration.skills pass" } validate_local_basic_log() { @@ -391,11 +443,29 @@ validate_local_basic_log() { openclaw_e2e_assert_log_not_contains "$log_path" "systemctl --user unavailable" } +run_selected_cases() { + local selected_cases="${OPENCLAW_ONBOARD_E2E_CASES:-guided-skip-ui,local-basic,remote-non-interactive,reset,channels,skills}" + local case_name + local -a cases=() + IFS="," read -r -a cases <<<"$selected_cases" + for case_name in "${cases[@]}"; do + case "$case_name" in + guided-skip-ui) run_case_guided_skip_ui ;; + local-basic) run_case_local_basic ;; + local-auth-refs) run_case_local_auth_refs ;; + local-password) run_case_local_password ;; + remote-non-interactive) run_case_remote_non_interactive ;; + reset) run_case_reset ;; + channels) run_case_channels ;; + skills) run_case_skills ;; + *) + echo "Unknown onboarding E2E case: $case_name" >&2 + return 2 + ;; + esac + done +} + if [ "$OPENCLAW_ONBOARD_SCENARIO_SOURCE_ONLY" != "1" ]; then - run_case_guided_skip_ui - run_case_local_basic - run_case_remote_non_interactive - run_case_reset - run_case_channels - run_case_skills + run_selected_cases fi diff --git a/scripts/e2e/onboard-docker.sh b/scripts/e2e/onboard-docker.sh index b44ae0a90d47..e62cb0feea93 100755 --- a/scripts/e2e/onboard-docker.sh +++ b/scripts/e2e/onboard-docker.sh @@ -12,6 +12,7 @@ DOCKER_RUN_TIMEOUT="${OPENCLAW_ONBOARD_DOCKER_RUN_TIMEOUT:-1200s}" COMMAND_TIMEOUT="${OPENCLAW_ONBOARD_COMMAND_TIMEOUT:-${OPENCLAW_E2E_COMMAND_TIMEOUT:-300s}}" GATEWAY_WAIT_ATTEMPTS="$(openclaw_e2e_read_positive_int_env OPENCLAW_ONBOARD_GATEWAY_WAIT_ATTEMPTS 20)" GATEWAY_WAIT_INTERVAL_S="$(docker_e2e_read_nonnegative_decimal_env OPENCLAW_ONBOARD_GATEWAY_WAIT_INTERVAL_S 1)" +ONBOARD_CASES="${OPENCLAW_ONBOARD_E2E_CASES:-}" CONTAINER_NAME="openclaw-onboard-e2e-$$" RUN_LOG="$(mktemp "${TMPDIR:-/tmp}/openclaw-onboard.XXXXXX")" STATS_LOG="$(mktemp "${TMPDIR:-/tmp}/openclaw-onboard-stats.XXXXXX")" @@ -27,11 +28,16 @@ docker_e2e_build_or_reuse "$IMAGE_NAME" onboard echo "Running onboarding E2E..." docker_e2e_docker_cmd rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true docker_e2e_harness_mount_args +ONBOARD_CASE_ENV_ARGS=() +if [ -n "$ONBOARD_CASES" ]; then + ONBOARD_CASE_ENV_ARGS+=(-e "OPENCLAW_ONBOARD_E2E_CASES=$ONBOARD_CASES") +fi DOCKER_COMMAND_TIMEOUT="$DOCKER_RUN_TIMEOUT" docker_e2e_docker_run_cmd run --name "$CONTAINER_NAME" "${DOCKER_E2E_HARNESS_ARGS[@]}" -t \ -e "OPENCLAW_TEST_STATE_FUNCTION_B64=$OPENCLAW_TEST_STATE_FUNCTION_B64" \ -e "OPENCLAW_E2E_COMMAND_TIMEOUT=$COMMAND_TIMEOUT" \ -e "OPENCLAW_ONBOARD_GATEWAY_WAIT_ATTEMPTS=$GATEWAY_WAIT_ATTEMPTS" \ -e "OPENCLAW_ONBOARD_GATEWAY_WAIT_INTERVAL_S=$GATEWAY_WAIT_INTERVAL_S" \ + "${ONBOARD_CASE_ENV_ARGS[@]}" \ "$IMAGE_NAME" bash scripts/e2e/lib/onboard/scenario.sh >"$RUN_LOG" 2>&1 & docker_pid="$!" diff --git a/scripts/e2e/qa-cli-onboarding.mjs b/scripts/e2e/qa-cli-onboarding.mjs new file mode 100644 index 000000000000..e1161c3cc60c --- /dev/null +++ b/scripts/e2e/qa-cli-onboarding.mjs @@ -0,0 +1,141 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { buildScriptEvidenceSummary, QA_EVIDENCE_FILENAME } from "../../extensions/qa-lab/api.js"; +import { createQaScriptEvidenceWriter } from "../../test/e2e/qa-lab/runtime/script-evidence.js"; + +const args = process.argv.slice(2); +const artifactBaseIndex = args.indexOf("--artifact-base"); +const artifactBase = args[artifactBaseIndex + 1]; +if (artifactBaseIndex === -1 || !artifactBase || artifactBase.startsWith("-")) { + throw new Error("--artifact-base is required"); +} + +const boundaries = [ + { + id: "cli-gateway-auth-storage", + title: "CLI Gateway auth storage", + markers: [ + "QA_ASSERT cli.gateway-auth-storage.token-ref pass", + "QA_ASSERT cli.gateway-auth-storage.password pass", + ], + }, + { + id: "cli-guided-onboarding", + title: "CLI guided onboarding", + marker: "QA_ASSERT cli.guided-onboarding pass", + }, + { + id: "cli-remote-onboarding", + title: "CLI remote onboarding", + marker: "QA_ASSERT cli.remote-onboarding pass", + }, + { + id: "cli-targeted-reconfiguration", + title: "CLI targeted reconfiguration", + markers: [ + "QA_ASSERT cli.targeted-reconfiguration.reset pass", + "QA_ASSERT cli.targeted-reconfiguration.skills pass", + ], + }, +]; +const writer = createQaScriptEvidenceWriter({ + artifactBase, + logFileName: "cli-onboarding.log", + primaryModel: "openai/gpt-5.6-luna", + providerMode: "mock-openai", + repoRoot: process.cwd(), + target: { + id: "cli-onboarding", + title: "CLI onboarding boundaries", + sourcePath: "scripts/e2e/qa-cli-onboarding.mjs", + }, +}); + +const seenMarkers = new Set(); +let markerCarry = ""; +const collectMarkers = (chunk) => { + const text = `${markerCarry}${chunk}`; + for (const boundary of boundaries) { + for (const marker of boundary.markers ?? [boundary.marker]) { + if (text.includes(marker)) { + seenMarkers.add(marker); + } + } + } + markerCarry = text.slice(-256); +}; + +const startedAt = Date.now(); +const child = spawn("bash", ["scripts/e2e/onboard-docker.sh"], { + env: { + ...process.env, + OPENCLAW_ONBOARD_E2E_CASES: + "guided-skip-ui,local-auth-refs,local-password,remote-non-interactive,reset,skills", + }, + stdio: ["inherit", "pipe", "pipe"], +}); + +child.stdout.on("data", (chunk) => { + writer.appendLog(chunk); + collectMarkers(chunk); + process.stdout.write(chunk); +}); +child.stderr.on("data", (chunk) => { + writer.appendLog(chunk); + collectMarkers(chunk); + process.stderr.write(chunk); +}); + +const { code, signal } = await new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", (closeCode, closeSignal) => + resolve({ code: closeCode, signal: closeSignal }), + ); +}); +const durationMs = Date.now() - startedAt; +const results = boundaries.map((boundary) => { + const markers = boundary.markers ?? [boundary.marker]; + const missingMarkers = markers.filter((marker) => !seenMarkers.has(marker)); + return { + id: boundary.id, + status: missingMarkers.length === 0 ? "pass" : "fail", + durationMs, + ...(missingMarkers.length > 0 + ? { failureMessage: `missing executable assertion marker(s): ${missingMarkers.join(", ")}` } + : {}), + }; +}); +const logArtifact = await writer.writeLog(); +const evidence = buildScriptEvidenceSummary({ + artifactPaths: [logArtifact], + env: process.env, + generatedAt: new Date().toISOString(), + primaryModel: "openai/gpt-5.6-luna", + providerMode: "mock-openai", + repoRoot: process.cwd(), + runner: "script", + targets: boundaries.map((boundary) => ({ + id: boundary.id, + title: boundary.title, + sourcePath: "scripts/e2e/qa-cli-onboarding.mjs", + })), + results, +}); +await fs.mkdir(artifactBase, { recursive: true }); +await fs.writeFile( + path.join(artifactBase, QA_EVIDENCE_FILENAME), + `${JSON.stringify(evidence, null, 2)}\n`, + "utf8", +); +await fs.writeFile( + path.join(artifactBase, "latest-run.json"), + `${JSON.stringify({ qaEvidence: QA_EVIDENCE_FILENAME }, null, 2)}\n`, + "utf8", +); + +if (signal) { + process.kill(process.pid, signal); +} +const missingBoundary = results.some((result) => result.status !== "pass"); +process.exit(code === 0 && !missingBoundary ? 0 : (code ?? 1)); diff --git a/scripts/e2e/qa-plugin-install-sources.mjs b/scripts/e2e/qa-plugin-install-sources.mjs new file mode 100644 index 000000000000..eca81e140a41 --- /dev/null +++ b/scripts/e2e/qa-plugin-install-sources.mjs @@ -0,0 +1,104 @@ +import { spawn } from "node:child_process"; +import { createQaScriptEvidenceWriter } from "../../test/e2e/qa-lab/runtime/script-evidence.js"; + +const args = process.argv.slice(2); +const artifactBaseIndex = args.indexOf("--artifact-base"); +const artifactBase = args[artifactBaseIndex + 1]; +if (artifactBaseIndex === -1 || !artifactBase || artifactBase.startsWith("-")) { + throw new Error("--artifact-base is required"); +} + +const writer = createQaScriptEvidenceWriter({ + artifactBase, + logFileName: "cli-plugin-install-sources.log", + primaryModel: "openai/gpt-5.6-luna", + providerMode: "mock-openai", + repoRoot: process.cwd(), + target: { + id: "cli-plugin-install-sources", + title: "CLI plugin install sources", + sourcePath: "scripts/e2e/qa-plugin-install-sources.mjs", + docsRefs: ["docs/cli/plugins.md"], + codeRefs: [ + "src/cli/plugins-cli.ts", + "src/plugins/install.ts", + "src/plugins/install-source-info.ts", + "scripts/e2e/lib/plugins/sweep.sh", + "scripts/e2e/lib/bundled-plugin-install-uninstall/sweep.sh", + ], + }, +}); +const requiredMarkers = [ + "Testing tgz install flow...", + "Testing install from local folder (plugins.load.paths)...", + "Testing install from npm spec (file:)...", + "Testing install and update from npm registry...", + "Testing install from git repo and plugin CLI execution...", + "Testing marketplace install and update flows...", + "Testing ClawHub plugin install and uninstall...", +]; +const seenMarkers = new Set(); +let markerCarry = ""; +let receivedSignal = null; +let bundledSelectionProven = false; + +const collectMarkers = (chunk) => { + const text = `${markerCarry}${chunk}`; + for (const marker of requiredMarkers) { + if (text.includes(marker)) { + seenMarkers.add(marker); + } + } + if (/bundled plugin install\/uninstall sweep passed \(([1-9]\d*) plugin\(s\)\)/u.test(text)) { + bundledSelectionProven = true; + } + markerCarry = text.slice(-256); +}; + +const runScript = (script, env = {}) => + new Promise((resolve, reject) => { + const child = spawn("bash", [script], { + env: { ...process.env, ...env }, + stdio: ["inherit", "pipe", "pipe"], + }); + const onChunk = (stream, chunk) => { + writer.appendLog(chunk); + collectMarkers(chunk); + stream.write(chunk); + }; + child.stdout.on("data", (chunk) => onChunk(process.stdout, chunk)); + child.stderr.on("data", (chunk) => onChunk(process.stderr, chunk)); + child.once("error", reject); + child.once("close", (code, signal) => { + receivedSignal ??= signal; + resolve(code ?? 1); + }); + }); + +const startedAt = Date.now(); +let status = await runScript("scripts/e2e/plugins-docker.sh", { + OPENCLAW_PLUGINS_E2E_CLAWHUB: "1", + OPENCLAW_PLUGINS_E2E_LIVE_CLAWHUB: "0", +}); +if (status === 0) { + status = await runScript("scripts/e2e/bundled-plugin-install-uninstall-docker.sh"); +} +const missingMarkers = requiredMarkers.filter((marker) => !seenMarkers.has(marker)); +if (!bundledSelectionProven) { + missingMarkers.push("bundled plugin install/uninstall sweep passed (>0 plugin(s))"); +} +if (missingMarkers.length > 0) { + status = 1; +} +await writer.write({ + status: status === 0 ? "pass" : "fail", + durationMs: Date.now() - startedAt, + ...(missingMarkers.length > 0 + ? { details: `missing executable assertion marker(s): ${missingMarkers.join(", ")}` } + : {}), +}); + +if (receivedSignal) { + process.kill(process.pid, receivedSignal); +} +process.exit(status); diff --git a/test/e2e/qa-lab/runtime/script-evidence.test.ts b/test/e2e/qa-lab/runtime/script-evidence.test.ts index e3e7fcdf6b3e..32b5dc09bf40 100644 --- a/test/e2e/qa-lab/runtime/script-evidence.test.ts +++ b/test/e2e/qa-lab/runtime/script-evidence.test.ts @@ -92,6 +92,16 @@ describe("QA script evidence writer", () => { expect(Buffer.byteLength(log, "utf8")).toBeLessThanOrEqual(24); }); + it("writes the bounded log independently for multi-target summaries", async () => { + const { artifactBase, writer } = await makeWriter(); + writer.appendLog("producer output\n"); + + await expect(writer.writeLog()).resolves.toEqual({ kind: "log", path: "producer.log" }); + await expect(fs.readFile(path.join(artifactBase, "producer.log"), "utf8")).resolves.toBe( + "producer output\n", + ); + }); + it("keeps only the bounded failure detail tail", async () => { const { writer } = await makeWriter({ maxDetailsBytes: 24 }); diff --git a/test/e2e/qa-lab/runtime/script-evidence.ts b/test/e2e/qa-lab/runtime/script-evidence.ts index 507859a5b744..4f0f192a4176 100644 --- a/test/e2e/qa-lab/runtime/script-evidence.ts +++ b/test/e2e/qa-lab/runtime/script-evidence.ts @@ -212,6 +212,12 @@ export function createQaScriptEvidenceWriter(options: QaScriptEvidenceWriterOpti ], }); + const writeLog = async () => { + await fs.mkdir(options.artifactBase, { recursive: true }); + await fs.writeFile(logFile.absoluteFilePath, boundedLogText(), "utf8"); + return { kind: "log", path: logFile.relativePath }; + }; + return { appendLog(chunk: unknown) { log.append(String(chunk)); @@ -220,10 +226,10 @@ export function createQaScriptEvidenceWriter(options: QaScriptEvidenceWriterOpti logText() { return boundedLogText(); }, + writeLog, async write(result: QaScriptEvidenceResult) { const evidence = build(result); - await fs.mkdir(options.artifactBase, { recursive: true }); - await fs.writeFile(logFile.absoluteFilePath, boundedLogText(), "utf8"); + await writeLog(); await writeJson(path.join(options.artifactBase, QA_EVIDENCE_FILENAME), evidence); await writeJson(path.join(options.artifactBase, "latest-run.json"), { qaEvidence: QA_EVIDENCE_FILENAME, diff --git a/test/scripts/onboard-config-fixtures.test.ts b/test/scripts/onboard-config-fixtures.test.ts index d9958d5be883..11d428430932 100644 --- a/test/scripts/onboard-config-fixtures.test.ts +++ b/test/scripts/onboard-config-fixtures.test.ts @@ -177,6 +177,83 @@ describe("onboard config fixture helpers", () => { expect(remoteResult.stderr).toBe(""); }); + it("accepts provider and gateway environment references from non-interactive onboarding", () => { + const root = makeTempDir(tempDirs, "openclaw-onboard-config-auth-refs-"); + const configPath = path.join(root, "openclaw.json"); + writeFileSync( + configPath, + `${JSON.stringify( + { + gateway: { + mode: "local", + auth: { + mode: "token", + token: { + source: "env", + provider: "default", + id: "OPENCLAW_GATEWAY_TOKEN", + }, + }, + }, + wizard: { lastRunMode: "local" }, + }, + null, + 2, + )}\n`, + "utf8", + ); + + const result = runScript(ASSERT_CONFIG_SCRIPT, ["local-auth-refs", configPath]); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + }); + + it("accepts password Gateway fixtures", () => { + const root = makeTempDir(tempDirs, "openclaw-onboard-config-password-"); + const passwordConfigPath = path.join(root, "password.json"); + writeFileSync( + passwordConfigPath, + `${JSON.stringify( + { + gateway: { + mode: "local", + auth: { mode: "password", password: "openclaw-onboard-password-e2e" }, + }, + wizard: { lastRunMode: "local" }, + }, + null, + 2, + )}\n`, + "utf8", + ); + + const passwordResult = runScript(ASSERT_CONFIG_SCRIPT, ["local-password", passwordConfigPath]); + + expect(passwordResult.status).toBe(0); + expect(passwordResult.stderr).toBe(""); + + const secretValue = "must-not-appear-in-assertion-output"; + writeFileSync( + passwordConfigPath, + `${JSON.stringify( + { + gateway: { mode: "local", auth: { mode: "password", password: secretValue } }, + wizard: { lastRunMode: "local" }, + }, + null, + 2, + )}\n`, + "utf8", + ); + + const mismatchResult = runScript(ASSERT_CONFIG_SCRIPT, ["local-password", passwordConfigPath]); + + expect(mismatchResult.status).toBe(1); + expect(mismatchResult.stderr).toContain("gateway.auth.password mismatch"); + expect(mismatchResult.stderr).not.toContain(secretValue); + }); + it("accepts channel configuration assertions for scrubbed channel secrets", () => { const root = makeTempDir(tempDirs, "openclaw-onboard-config-channels-"); const configPath = path.join(root, "channels.json"); diff --git a/test/scripts/qa-cli-onboarding.test.ts b/test/scripts/qa-cli-onboarding.test.ts new file mode 100644 index 000000000000..ce636c3c97bf --- /dev/null +++ b/test/scripts/qa-cli-onboarding.test.ts @@ -0,0 +1,13 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +describe("QA CLI onboarding evidence", () => { + it("retains the bounded redacted child log", () => { + const source = readFileSync("scripts/e2e/qa-cli-onboarding.mjs", "utf8"); + + expect(source).toContain("writer.appendLog(chunk)"); + expect(source).toContain("const logArtifact = await writer.writeLog()"); + expect(source).toContain("artifactPaths: [logArtifact]"); + expect(source).not.toContain("artifactPaths: []"); + }); +}); diff --git a/test/scripts/qa-plugin-install-sources.test.ts b/test/scripts/qa-plugin-install-sources.test.ts new file mode 100644 index 000000000000..1db3d258f2ed --- /dev/null +++ b/test/scripts/qa-plugin-install-sources.test.ts @@ -0,0 +1,14 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +const FILE_SPEC_MARKER = "Testing install from npm spec (file:)..."; + +describe("QA plugin install source coverage", () => { + it("requires the executable npm file-spec assertion marker", () => { + const qaWrapper = readFileSync("scripts/e2e/qa-plugin-install-sources.mjs", "utf8"); + const pluginSweep = readFileSync("scripts/e2e/lib/plugins/sweep.sh", "utf8"); + + expect(pluginSweep).toContain(`echo "${FILE_SPEC_MARKER}"`); + expect(qaWrapper).toContain(`"${FILE_SPEC_MARKER}",`); + }); +});