mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-08 02:52:15 +00:00
test(qa): cover CLI onboarding and plugin install sources (#118882)
* test(qa): cover CLI onboarding and plugin sources * test(qa): assert npm file plugin source * test(qa): redact onboarding password assertion * test(qa): retain onboarding evidence log
This commit is contained in:
39
qa/scenarios/cli/cli-onboarding.yaml
Normal file
39
qa/scenarios/cli/cli-onboarding.yaml
Normal file
@@ -0,0 +1,39 @@
|
||||
title: CLI onboarding contracts
|
||||
|
||||
scenario:
|
||||
id: cli-onboarding
|
||||
surface: cli
|
||||
category: cli.onboarding-and-auth-setup
|
||||
coverage:
|
||||
primary:
|
||||
- cli.gateway-auth-storage
|
||||
- cli.guided-onboarding
|
||||
- cli.remote-onboarding
|
||||
- cli.targeted-reconfiguration
|
||||
objective: Verify guided and non-interactive CLI onboarding across Gateway storage, remote setup, reset, and section-scoped reconfiguration.
|
||||
successCriteria:
|
||||
- Guided onboarding runs under a real PTY, verifies the configured model, reaches completion, and performs no UI or terminal handoff.
|
||||
- Gateway auth proves token SecretRef storage and password storage through executable non-interactive setup.
|
||||
- Remote onboarding persists the requested remote URL and token, and reset removes stale remote configuration.
|
||||
- Section-scoped skills configuration records configure metadata without selecting a Gateway run mode.
|
||||
- The selected onboarding cases exclude channel setup.
|
||||
- QA evidence records a separate executable assertion entry for every primary coverage boundary.
|
||||
docsRefs:
|
||||
- docs/cli/onboard.md
|
||||
- docs/cli/configure.md
|
||||
codeRefs:
|
||||
- src/commands/onboard.ts
|
||||
- src/commands/onboard-non-interactive.ts
|
||||
- src/commands/onboard-non-interactive/remote.ts
|
||||
- src/commands/onboard-non-interactive/local/auth-choice.ts
|
||||
- src/commands/onboard-non-interactive/local/gateway-config.ts
|
||||
- src/commands/configure.commands.ts
|
||||
- scripts/e2e/lib/onboard/scenario.sh
|
||||
execution:
|
||||
kind: script
|
||||
path: scripts/e2e/qa-cli-onboarding.mjs
|
||||
summary: Runs the executable onboarding Docker lane with only the guided, Gateway-storage, remote, reset, and skills cases selected.
|
||||
timeoutMs: 1800000
|
||||
args:
|
||||
- --artifact-base
|
||||
- ${outputDir}
|
||||
33
qa/scenarios/cli/cli-plugin-install-sources.yaml
Normal file
33
qa/scenarios/cli/cli-plugin-install-sources.yaml
Normal file
@@ -0,0 +1,33 @@
|
||||
title: CLI plugin install sources
|
||||
|
||||
scenario:
|
||||
id: cli-plugin-install-sources
|
||||
surface: cli
|
||||
category: cli.plugin-and-channel-setup
|
||||
coverage:
|
||||
primary:
|
||||
- cli.plugin-install-sources
|
||||
objective: Verify the plugin CLI installs supported bundled, local, archive, npm, ClawHub, marketplace, file, and Git sources through executable commands.
|
||||
successCriteria:
|
||||
- The plugin CLI installs a bundled plugin, tarball archive, local directory, file package, npm package, and Git source.
|
||||
- Local directory installation preserves vendored dependencies.
|
||||
- Invalid npm metadata fails without creating an install record.
|
||||
- Moving and exact Git references update and resolve to the expected commits.
|
||||
- Hermetic ClawHub and marketplace fixtures install, update, inspect, and uninstall plugins.
|
||||
- QA evidence fails when any required source-class assertion marker is absent.
|
||||
docsRefs:
|
||||
- docs/cli/plugins.md
|
||||
codeRefs:
|
||||
- src/cli/plugins-cli.ts
|
||||
- src/plugins/install.ts
|
||||
- src/plugins/install-source-info.ts
|
||||
- scripts/e2e/lib/plugins/sweep.sh
|
||||
- scripts/e2e/lib/bundled-plugin-install-uninstall/sweep.sh
|
||||
execution:
|
||||
kind: script
|
||||
path: scripts/e2e/qa-plugin-install-sources.mjs
|
||||
summary: Runs the hermetic plugin source sweep plus the packaged bundled-plugin install/uninstall lane and emits bounded QA evidence.
|
||||
timeoutMs: 1800000
|
||||
args:
|
||||
- --artifact-base
|
||||
- ${outputDir}
|
||||
@@ -44,6 +44,20 @@ switch (scenario) {
|
||||
expectEqual("wizard.lastRunCommand", cfg?.wizard?.lastRunCommand, "onboard");
|
||||
break;
|
||||
}
|
||||
case "local-auth-refs":
|
||||
assertLocalWizard();
|
||||
expectEqual("gateway.auth.mode", cfg?.gateway?.auth?.mode, "token");
|
||||
expectEqual("gateway.auth.token.source", cfg?.gateway?.auth?.token?.source, "env");
|
||||
expectEqual("gateway.auth.token.provider", cfg?.gateway?.auth?.token?.provider, "default");
|
||||
expectEqual("gateway.auth.token.id", cfg?.gateway?.auth?.token?.id, "OPENCLAW_GATEWAY_TOKEN");
|
||||
break;
|
||||
case "local-password":
|
||||
assertLocalWizard();
|
||||
expectEqual("gateway.auth.mode", cfg?.gateway?.auth?.mode, "password");
|
||||
if (cfg?.gateway?.auth?.password !== "openclaw-onboard-password-e2e") {
|
||||
errors.push("gateway.auth.password mismatch");
|
||||
}
|
||||
break;
|
||||
case "remote-non-interactive":
|
||||
expectEqual("gateway.mode", cfg?.gateway?.mode, "remote");
|
||||
expectEqual("gateway.remote.url", cfg?.gateway?.remote?.url, "ws://gateway.local:18789");
|
||||
|
||||
@@ -304,6 +304,7 @@ run_case_guided_skip_ui() {
|
||||
send_guided_skip_ui_flow
|
||||
|
||||
validate_guided_skip_ui_log "$WIZARD_LOG_PATH" "$mock_request_log"
|
||||
echo "QA_ASSERT cli.guided-onboarding pass"
|
||||
openclaw_e2e_stop_process "$mock_openai_pid"
|
||||
mock_openai_pid=""
|
||||
}
|
||||
@@ -336,6 +337,54 @@ run_case_local_basic() {
|
||||
|
||||
}
|
||||
|
||||
run_case_local_auth_refs() {
|
||||
set_isolated_openclaw_env local-auth-refs
|
||||
export OPENAI_API_KEY="sk-openclaw-onboard-auth-ref-e2e"
|
||||
export OPENCLAW_GATEWAY_TOKEN="openclaw-onboard-gateway-ref-e2e"
|
||||
|
||||
openclaw_e2e_run_logged local-auth-refs node "$OPENCLAW_ENTRY" onboard \
|
||||
--non-interactive \
|
||||
--accept-risk \
|
||||
--flow quickstart \
|
||||
--mode local \
|
||||
--auth-choice openai-api-key \
|
||||
--secret-input-mode ref \
|
||||
--gateway-auth token \
|
||||
--gateway-token-ref-env OPENCLAW_GATEWAY_TOKEN \
|
||||
--skip-channels \
|
||||
--skip-skills \
|
||||
--skip-daemon \
|
||||
--skip-ui \
|
||||
--skip-health
|
||||
|
||||
node scripts/e2e/lib/release-scenarios/assertions.mjs \
|
||||
assert-openai-env-ref \
|
||||
"$OPENAI_API_KEY"
|
||||
assert_onboard_config local-auth-refs
|
||||
echo "QA_ASSERT cli.gateway-auth-storage.token-ref pass"
|
||||
}
|
||||
|
||||
run_case_local_password() {
|
||||
set_isolated_openclaw_env local-password
|
||||
|
||||
openclaw_e2e_run_logged local-password node "$OPENCLAW_ENTRY" onboard \
|
||||
--non-interactive \
|
||||
--accept-risk \
|
||||
--flow quickstart \
|
||||
--mode local \
|
||||
--auth-choice skip \
|
||||
--gateway-auth password \
|
||||
--gateway-password "openclaw-onboard-password-e2e" \
|
||||
--skip-channels \
|
||||
--skip-skills \
|
||||
--skip-daemon \
|
||||
--skip-ui \
|
||||
--skip-health
|
||||
|
||||
assert_onboard_config local-password
|
||||
echo "QA_ASSERT cli.gateway-auth-storage.password pass"
|
||||
}
|
||||
|
||||
run_case_remote_non_interactive() {
|
||||
set_isolated_openclaw_env remote-non-interactive
|
||||
# Smoke test non-interactive remote config write.
|
||||
@@ -347,6 +396,7 @@ run_case_remote_non_interactive() {
|
||||
--skip-health
|
||||
|
||||
assert_onboard_config remote-non-interactive
|
||||
echo "QA_ASSERT cli.remote-onboarding pass"
|
||||
}
|
||||
|
||||
run_case_reset() {
|
||||
@@ -366,6 +416,7 @@ run_case_reset() {
|
||||
--skip-health
|
||||
|
||||
assert_onboard_config reset
|
||||
echo "QA_ASSERT cli.targeted-reconfiguration.reset pass"
|
||||
}
|
||||
|
||||
run_case_channels() {
|
||||
@@ -384,6 +435,7 @@ run_case_skills() {
|
||||
run_wizard_cmd skills "$home_dir" "node \"$OPENCLAW_ENTRY\" configure --section skills" send_skills_flow
|
||||
|
||||
assert_onboard_config skills
|
||||
echo "QA_ASSERT cli.targeted-reconfiguration.skills pass"
|
||||
}
|
||||
|
||||
validate_local_basic_log() {
|
||||
@@ -391,11 +443,29 @@ validate_local_basic_log() {
|
||||
openclaw_e2e_assert_log_not_contains "$log_path" "systemctl --user unavailable"
|
||||
}
|
||||
|
||||
run_selected_cases() {
|
||||
local selected_cases="${OPENCLAW_ONBOARD_E2E_CASES:-guided-skip-ui,local-basic,remote-non-interactive,reset,channels,skills}"
|
||||
local case_name
|
||||
local -a cases=()
|
||||
IFS="," read -r -a cases <<<"$selected_cases"
|
||||
for case_name in "${cases[@]}"; do
|
||||
case "$case_name" in
|
||||
guided-skip-ui) run_case_guided_skip_ui ;;
|
||||
local-basic) run_case_local_basic ;;
|
||||
local-auth-refs) run_case_local_auth_refs ;;
|
||||
local-password) run_case_local_password ;;
|
||||
remote-non-interactive) run_case_remote_non_interactive ;;
|
||||
reset) run_case_reset ;;
|
||||
channels) run_case_channels ;;
|
||||
skills) run_case_skills ;;
|
||||
*)
|
||||
echo "Unknown onboarding E2E case: $case_name" >&2
|
||||
return 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
if [ "$OPENCLAW_ONBOARD_SCENARIO_SOURCE_ONLY" != "1" ]; then
|
||||
run_case_guided_skip_ui
|
||||
run_case_local_basic
|
||||
run_case_remote_non_interactive
|
||||
run_case_reset
|
||||
run_case_channels
|
||||
run_case_skills
|
||||
run_selected_cases
|
||||
fi
|
||||
|
||||
@@ -12,6 +12,7 @@ DOCKER_RUN_TIMEOUT="${OPENCLAW_ONBOARD_DOCKER_RUN_TIMEOUT:-1200s}"
|
||||
COMMAND_TIMEOUT="${OPENCLAW_ONBOARD_COMMAND_TIMEOUT:-${OPENCLAW_E2E_COMMAND_TIMEOUT:-300s}}"
|
||||
GATEWAY_WAIT_ATTEMPTS="$(openclaw_e2e_read_positive_int_env OPENCLAW_ONBOARD_GATEWAY_WAIT_ATTEMPTS 20)"
|
||||
GATEWAY_WAIT_INTERVAL_S="$(docker_e2e_read_nonnegative_decimal_env OPENCLAW_ONBOARD_GATEWAY_WAIT_INTERVAL_S 1)"
|
||||
ONBOARD_CASES="${OPENCLAW_ONBOARD_E2E_CASES:-}"
|
||||
CONTAINER_NAME="openclaw-onboard-e2e-$$"
|
||||
RUN_LOG="$(mktemp "${TMPDIR:-/tmp}/openclaw-onboard.XXXXXX")"
|
||||
STATS_LOG="$(mktemp "${TMPDIR:-/tmp}/openclaw-onboard-stats.XXXXXX")"
|
||||
@@ -27,11 +28,16 @@ docker_e2e_build_or_reuse "$IMAGE_NAME" onboard
|
||||
echo "Running onboarding E2E..."
|
||||
docker_e2e_docker_cmd rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
|
||||
docker_e2e_harness_mount_args
|
||||
ONBOARD_CASE_ENV_ARGS=()
|
||||
if [ -n "$ONBOARD_CASES" ]; then
|
||||
ONBOARD_CASE_ENV_ARGS+=(-e "OPENCLAW_ONBOARD_E2E_CASES=$ONBOARD_CASES")
|
||||
fi
|
||||
DOCKER_COMMAND_TIMEOUT="$DOCKER_RUN_TIMEOUT" docker_e2e_docker_run_cmd run --name "$CONTAINER_NAME" "${DOCKER_E2E_HARNESS_ARGS[@]}" -t \
|
||||
-e "OPENCLAW_TEST_STATE_FUNCTION_B64=$OPENCLAW_TEST_STATE_FUNCTION_B64" \
|
||||
-e "OPENCLAW_E2E_COMMAND_TIMEOUT=$COMMAND_TIMEOUT" \
|
||||
-e "OPENCLAW_ONBOARD_GATEWAY_WAIT_ATTEMPTS=$GATEWAY_WAIT_ATTEMPTS" \
|
||||
-e "OPENCLAW_ONBOARD_GATEWAY_WAIT_INTERVAL_S=$GATEWAY_WAIT_INTERVAL_S" \
|
||||
"${ONBOARD_CASE_ENV_ARGS[@]}" \
|
||||
"$IMAGE_NAME" bash scripts/e2e/lib/onboard/scenario.sh >"$RUN_LOG" 2>&1 &
|
||||
docker_pid="$!"
|
||||
|
||||
|
||||
141
scripts/e2e/qa-cli-onboarding.mjs
Normal file
141
scripts/e2e/qa-cli-onboarding.mjs
Normal file
@@ -0,0 +1,141 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { buildScriptEvidenceSummary, QA_EVIDENCE_FILENAME } from "../../extensions/qa-lab/api.js";
|
||||
import { createQaScriptEvidenceWriter } from "../../test/e2e/qa-lab/runtime/script-evidence.js";
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const artifactBaseIndex = args.indexOf("--artifact-base");
|
||||
const artifactBase = args[artifactBaseIndex + 1];
|
||||
if (artifactBaseIndex === -1 || !artifactBase || artifactBase.startsWith("-")) {
|
||||
throw new Error("--artifact-base is required");
|
||||
}
|
||||
|
||||
const boundaries = [
|
||||
{
|
||||
id: "cli-gateway-auth-storage",
|
||||
title: "CLI Gateway auth storage",
|
||||
markers: [
|
||||
"QA_ASSERT cli.gateway-auth-storage.token-ref pass",
|
||||
"QA_ASSERT cli.gateway-auth-storage.password pass",
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "cli-guided-onboarding",
|
||||
title: "CLI guided onboarding",
|
||||
marker: "QA_ASSERT cli.guided-onboarding pass",
|
||||
},
|
||||
{
|
||||
id: "cli-remote-onboarding",
|
||||
title: "CLI remote onboarding",
|
||||
marker: "QA_ASSERT cli.remote-onboarding pass",
|
||||
},
|
||||
{
|
||||
id: "cli-targeted-reconfiguration",
|
||||
title: "CLI targeted reconfiguration",
|
||||
markers: [
|
||||
"QA_ASSERT cli.targeted-reconfiguration.reset pass",
|
||||
"QA_ASSERT cli.targeted-reconfiguration.skills pass",
|
||||
],
|
||||
},
|
||||
];
|
||||
const writer = createQaScriptEvidenceWriter({
|
||||
artifactBase,
|
||||
logFileName: "cli-onboarding.log",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
providerMode: "mock-openai",
|
||||
repoRoot: process.cwd(),
|
||||
target: {
|
||||
id: "cli-onboarding",
|
||||
title: "CLI onboarding boundaries",
|
||||
sourcePath: "scripts/e2e/qa-cli-onboarding.mjs",
|
||||
},
|
||||
});
|
||||
|
||||
const seenMarkers = new Set();
|
||||
let markerCarry = "";
|
||||
const collectMarkers = (chunk) => {
|
||||
const text = `${markerCarry}${chunk}`;
|
||||
for (const boundary of boundaries) {
|
||||
for (const marker of boundary.markers ?? [boundary.marker]) {
|
||||
if (text.includes(marker)) {
|
||||
seenMarkers.add(marker);
|
||||
}
|
||||
}
|
||||
}
|
||||
markerCarry = text.slice(-256);
|
||||
};
|
||||
|
||||
const startedAt = Date.now();
|
||||
const child = spawn("bash", ["scripts/e2e/onboard-docker.sh"], {
|
||||
env: {
|
||||
...process.env,
|
||||
OPENCLAW_ONBOARD_E2E_CASES:
|
||||
"guided-skip-ui,local-auth-refs,local-password,remote-non-interactive,reset,skills",
|
||||
},
|
||||
stdio: ["inherit", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
child.stdout.on("data", (chunk) => {
|
||||
writer.appendLog(chunk);
|
||||
collectMarkers(chunk);
|
||||
process.stdout.write(chunk);
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
writer.appendLog(chunk);
|
||||
collectMarkers(chunk);
|
||||
process.stderr.write(chunk);
|
||||
});
|
||||
|
||||
const { code, signal } = await new Promise((resolve, reject) => {
|
||||
child.once("error", reject);
|
||||
child.once("close", (closeCode, closeSignal) =>
|
||||
resolve({ code: closeCode, signal: closeSignal }),
|
||||
);
|
||||
});
|
||||
const durationMs = Date.now() - startedAt;
|
||||
const results = boundaries.map((boundary) => {
|
||||
const markers = boundary.markers ?? [boundary.marker];
|
||||
const missingMarkers = markers.filter((marker) => !seenMarkers.has(marker));
|
||||
return {
|
||||
id: boundary.id,
|
||||
status: missingMarkers.length === 0 ? "pass" : "fail",
|
||||
durationMs,
|
||||
...(missingMarkers.length > 0
|
||||
? { failureMessage: `missing executable assertion marker(s): ${missingMarkers.join(", ")}` }
|
||||
: {}),
|
||||
};
|
||||
});
|
||||
const logArtifact = await writer.writeLog();
|
||||
const evidence = buildScriptEvidenceSummary({
|
||||
artifactPaths: [logArtifact],
|
||||
env: process.env,
|
||||
generatedAt: new Date().toISOString(),
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
providerMode: "mock-openai",
|
||||
repoRoot: process.cwd(),
|
||||
runner: "script",
|
||||
targets: boundaries.map((boundary) => ({
|
||||
id: boundary.id,
|
||||
title: boundary.title,
|
||||
sourcePath: "scripts/e2e/qa-cli-onboarding.mjs",
|
||||
})),
|
||||
results,
|
||||
});
|
||||
await fs.mkdir(artifactBase, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(artifactBase, QA_EVIDENCE_FILENAME),
|
||||
`${JSON.stringify(evidence, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(artifactBase, "latest-run.json"),
|
||||
`${JSON.stringify({ qaEvidence: QA_EVIDENCE_FILENAME }, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
if (signal) {
|
||||
process.kill(process.pid, signal);
|
||||
}
|
||||
const missingBoundary = results.some((result) => result.status !== "pass");
|
||||
process.exit(code === 0 && !missingBoundary ? 0 : (code ?? 1));
|
||||
104
scripts/e2e/qa-plugin-install-sources.mjs
Normal file
104
scripts/e2e/qa-plugin-install-sources.mjs
Normal file
@@ -0,0 +1,104 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { createQaScriptEvidenceWriter } from "../../test/e2e/qa-lab/runtime/script-evidence.js";
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const artifactBaseIndex = args.indexOf("--artifact-base");
|
||||
const artifactBase = args[artifactBaseIndex + 1];
|
||||
if (artifactBaseIndex === -1 || !artifactBase || artifactBase.startsWith("-")) {
|
||||
throw new Error("--artifact-base is required");
|
||||
}
|
||||
|
||||
const writer = createQaScriptEvidenceWriter({
|
||||
artifactBase,
|
||||
logFileName: "cli-plugin-install-sources.log",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
providerMode: "mock-openai",
|
||||
repoRoot: process.cwd(),
|
||||
target: {
|
||||
id: "cli-plugin-install-sources",
|
||||
title: "CLI plugin install sources",
|
||||
sourcePath: "scripts/e2e/qa-plugin-install-sources.mjs",
|
||||
docsRefs: ["docs/cli/plugins.md"],
|
||||
codeRefs: [
|
||||
"src/cli/plugins-cli.ts",
|
||||
"src/plugins/install.ts",
|
||||
"src/plugins/install-source-info.ts",
|
||||
"scripts/e2e/lib/plugins/sweep.sh",
|
||||
"scripts/e2e/lib/bundled-plugin-install-uninstall/sweep.sh",
|
||||
],
|
||||
},
|
||||
});
|
||||
const requiredMarkers = [
|
||||
"Testing tgz install flow...",
|
||||
"Testing install from local folder (plugins.load.paths)...",
|
||||
"Testing install from npm spec (file:)...",
|
||||
"Testing install and update from npm registry...",
|
||||
"Testing install from git repo and plugin CLI execution...",
|
||||
"Testing marketplace install and update flows...",
|
||||
"Testing ClawHub plugin install and uninstall...",
|
||||
];
|
||||
const seenMarkers = new Set();
|
||||
let markerCarry = "";
|
||||
let receivedSignal = null;
|
||||
let bundledSelectionProven = false;
|
||||
|
||||
const collectMarkers = (chunk) => {
|
||||
const text = `${markerCarry}${chunk}`;
|
||||
for (const marker of requiredMarkers) {
|
||||
if (text.includes(marker)) {
|
||||
seenMarkers.add(marker);
|
||||
}
|
||||
}
|
||||
if (/bundled plugin install\/uninstall sweep passed \(([1-9]\d*) plugin\(s\)\)/u.test(text)) {
|
||||
bundledSelectionProven = true;
|
||||
}
|
||||
markerCarry = text.slice(-256);
|
||||
};
|
||||
|
||||
const runScript = (script, env = {}) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const child = spawn("bash", [script], {
|
||||
env: { ...process.env, ...env },
|
||||
stdio: ["inherit", "pipe", "pipe"],
|
||||
});
|
||||
const onChunk = (stream, chunk) => {
|
||||
writer.appendLog(chunk);
|
||||
collectMarkers(chunk);
|
||||
stream.write(chunk);
|
||||
};
|
||||
child.stdout.on("data", (chunk) => onChunk(process.stdout, chunk));
|
||||
child.stderr.on("data", (chunk) => onChunk(process.stderr, chunk));
|
||||
child.once("error", reject);
|
||||
child.once("close", (code, signal) => {
|
||||
receivedSignal ??= signal;
|
||||
resolve(code ?? 1);
|
||||
});
|
||||
});
|
||||
|
||||
const startedAt = Date.now();
|
||||
let status = await runScript("scripts/e2e/plugins-docker.sh", {
|
||||
OPENCLAW_PLUGINS_E2E_CLAWHUB: "1",
|
||||
OPENCLAW_PLUGINS_E2E_LIVE_CLAWHUB: "0",
|
||||
});
|
||||
if (status === 0) {
|
||||
status = await runScript("scripts/e2e/bundled-plugin-install-uninstall-docker.sh");
|
||||
}
|
||||
const missingMarkers = requiredMarkers.filter((marker) => !seenMarkers.has(marker));
|
||||
if (!bundledSelectionProven) {
|
||||
missingMarkers.push("bundled plugin install/uninstall sweep passed (>0 plugin(s))");
|
||||
}
|
||||
if (missingMarkers.length > 0) {
|
||||
status = 1;
|
||||
}
|
||||
await writer.write({
|
||||
status: status === 0 ? "pass" : "fail",
|
||||
durationMs: Date.now() - startedAt,
|
||||
...(missingMarkers.length > 0
|
||||
? { details: `missing executable assertion marker(s): ${missingMarkers.join(", ")}` }
|
||||
: {}),
|
||||
});
|
||||
|
||||
if (receivedSignal) {
|
||||
process.kill(process.pid, receivedSignal);
|
||||
}
|
||||
process.exit(status);
|
||||
@@ -92,6 +92,16 @@ describe("QA script evidence writer", () => {
|
||||
expect(Buffer.byteLength(log, "utf8")).toBeLessThanOrEqual(24);
|
||||
});
|
||||
|
||||
it("writes the bounded log independently for multi-target summaries", async () => {
|
||||
const { artifactBase, writer } = await makeWriter();
|
||||
writer.appendLog("producer output\n");
|
||||
|
||||
await expect(writer.writeLog()).resolves.toEqual({ kind: "log", path: "producer.log" });
|
||||
await expect(fs.readFile(path.join(artifactBase, "producer.log"), "utf8")).resolves.toBe(
|
||||
"producer output\n",
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps only the bounded failure detail tail", async () => {
|
||||
const { writer } = await makeWriter({ maxDetailsBytes: 24 });
|
||||
|
||||
|
||||
@@ -212,6 +212,12 @@ export function createQaScriptEvidenceWriter(options: QaScriptEvidenceWriterOpti
|
||||
],
|
||||
});
|
||||
|
||||
const writeLog = async () => {
|
||||
await fs.mkdir(options.artifactBase, { recursive: true });
|
||||
await fs.writeFile(logFile.absoluteFilePath, boundedLogText(), "utf8");
|
||||
return { kind: "log", path: logFile.relativePath };
|
||||
};
|
||||
|
||||
return {
|
||||
appendLog(chunk: unknown) {
|
||||
log.append(String(chunk));
|
||||
@@ -220,10 +226,10 @@ export function createQaScriptEvidenceWriter(options: QaScriptEvidenceWriterOpti
|
||||
logText() {
|
||||
return boundedLogText();
|
||||
},
|
||||
writeLog,
|
||||
async write(result: QaScriptEvidenceResult) {
|
||||
const evidence = build(result);
|
||||
await fs.mkdir(options.artifactBase, { recursive: true });
|
||||
await fs.writeFile(logFile.absoluteFilePath, boundedLogText(), "utf8");
|
||||
await writeLog();
|
||||
await writeJson(path.join(options.artifactBase, QA_EVIDENCE_FILENAME), evidence);
|
||||
await writeJson(path.join(options.artifactBase, "latest-run.json"), {
|
||||
qaEvidence: QA_EVIDENCE_FILENAME,
|
||||
|
||||
@@ -177,6 +177,83 @@ describe("onboard config fixture helpers", () => {
|
||||
expect(remoteResult.stderr).toBe("");
|
||||
});
|
||||
|
||||
it("accepts provider and gateway environment references from non-interactive onboarding", () => {
|
||||
const root = makeTempDir(tempDirs, "openclaw-onboard-config-auth-refs-");
|
||||
const configPath = path.join(root, "openclaw.json");
|
||||
writeFileSync(
|
||||
configPath,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
gateway: {
|
||||
mode: "local",
|
||||
auth: {
|
||||
mode: "token",
|
||||
token: {
|
||||
source: "env",
|
||||
provider: "default",
|
||||
id: "OPENCLAW_GATEWAY_TOKEN",
|
||||
},
|
||||
},
|
||||
},
|
||||
wizard: { lastRunMode: "local" },
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = runScript(ASSERT_CONFIG_SCRIPT, ["local-auth-refs", configPath]);
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stderr).toBe("");
|
||||
});
|
||||
|
||||
it("accepts password Gateway fixtures", () => {
|
||||
const root = makeTempDir(tempDirs, "openclaw-onboard-config-password-");
|
||||
const passwordConfigPath = path.join(root, "password.json");
|
||||
writeFileSync(
|
||||
passwordConfigPath,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
gateway: {
|
||||
mode: "local",
|
||||
auth: { mode: "password", password: "openclaw-onboard-password-e2e" },
|
||||
},
|
||||
wizard: { lastRunMode: "local" },
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const passwordResult = runScript(ASSERT_CONFIG_SCRIPT, ["local-password", passwordConfigPath]);
|
||||
|
||||
expect(passwordResult.status).toBe(0);
|
||||
expect(passwordResult.stderr).toBe("");
|
||||
|
||||
const secretValue = "must-not-appear-in-assertion-output";
|
||||
writeFileSync(
|
||||
passwordConfigPath,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
gateway: { mode: "local", auth: { mode: "password", password: secretValue } },
|
||||
wizard: { lastRunMode: "local" },
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const mismatchResult = runScript(ASSERT_CONFIG_SCRIPT, ["local-password", passwordConfigPath]);
|
||||
|
||||
expect(mismatchResult.status).toBe(1);
|
||||
expect(mismatchResult.stderr).toContain("gateway.auth.password mismatch");
|
||||
expect(mismatchResult.stderr).not.toContain(secretValue);
|
||||
});
|
||||
|
||||
it("accepts channel configuration assertions for scrubbed channel secrets", () => {
|
||||
const root = makeTempDir(tempDirs, "openclaw-onboard-config-channels-");
|
||||
const configPath = path.join(root, "channels.json");
|
||||
|
||||
13
test/scripts/qa-cli-onboarding.test.ts
Normal file
13
test/scripts/qa-cli-onboarding.test.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
describe("QA CLI onboarding evidence", () => {
|
||||
it("retains the bounded redacted child log", () => {
|
||||
const source = readFileSync("scripts/e2e/qa-cli-onboarding.mjs", "utf8");
|
||||
|
||||
expect(source).toContain("writer.appendLog(chunk)");
|
||||
expect(source).toContain("const logArtifact = await writer.writeLog()");
|
||||
expect(source).toContain("artifactPaths: [logArtifact]");
|
||||
expect(source).not.toContain("artifactPaths: []");
|
||||
});
|
||||
});
|
||||
14
test/scripts/qa-plugin-install-sources.test.ts
Normal file
14
test/scripts/qa-plugin-install-sources.test.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const FILE_SPEC_MARKER = "Testing install from npm spec (file:)...";
|
||||
|
||||
describe("QA plugin install source coverage", () => {
|
||||
it("requires the executable npm file-spec assertion marker", () => {
|
||||
const qaWrapper = readFileSync("scripts/e2e/qa-plugin-install-sources.mjs", "utf8");
|
||||
const pluginSweep = readFileSync("scripts/e2e/lib/plugins/sweep.sh", "utf8");
|
||||
|
||||
expect(pluginSweep).toContain(`echo "${FILE_SPEC_MARKER}"`);
|
||||
expect(qaWrapper).toContain(`"${FILE_SPEC_MARKER}",`);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user