diff --git a/qa/scenarios/runtime/compose-setup.yaml b/qa/scenarios/runtime/compose-setup.yaml index 577bf64ede63..317a79faab2f 100644 --- a/qa/scenarios/runtime/compose-setup.yaml +++ b/qa/scenarios/runtime/compose-setup.yaml @@ -5,20 +5,32 @@ scenario: surface: containers category: containers.container-setup coverage: + primary: + - containers.docker-compose-gateway + - containers.container-health-endpoints secondary: - containers.docker-compose-mounts-and-secrets - objective: Launch the documented Docker Compose topology from the package-backed functional image and record the exact package, image, and service container identities. + objective: Launch the documented Docker Compose topology from the package-backed functional image and prove its liveness, readiness, authenticated health CLI, Docker health state, recovery, and operator diagnostics. successCriteria: - The scheduler builds the real package artifact and package-backed functional image through the canonical Docker planner. - "`docker compose up` starts the documented `openclaw-gateway` service with isolated state, config, workspace, and auth-profile mounts." - - The Gateway reaches healthy state and the documented container-side health command passes. - - "The `openclaw-cli` Compose service runs successfully in the Gateway network namespace." - - Evidence records the package digest and version, functional image ID, and Gateway and CLI container IDs. + - The Dockerfile definition, resolved Compose configuration, and running container all use `node dist/docker-healthcheck.js`. + - "`/healthz` reports live and `/readyz` reports ready before and after a bounded forced-unhealthy cycle." + - Wrong tokens fail and the configured token passes for `gateway health --json` from both the Gateway service and `openclaw-cli` sidecar. + - Docker records the forced failure as `unhealthy`, records a successful recovery to `healthy`, and exposes non-empty `State.Health` and Compose log diagnostics. + - Evidence records the package digest and version, functional image ID, Gateway and CLI container IDs, and explicit pass markers for each health boundary. docsRefs: - docs/install/docker.md + - docs/cli/gateway.md + - docs/gateway/logging.md + - docs/gateway/opentelemetry.md + - docs/gateway/prometheus.md - docs/help/testing.md codeRefs: + - Dockerfile - docker-compose.yml + - src/docker-healthcheck.ts + - src/gateway/server-http.ts - scripts/e2e/compose-setup.sh - scripts/lib/docker-e2e-scenarios.mjs - scripts/package-openclaw-for-docker.mjs @@ -26,7 +38,7 @@ scenario: execution: kind: script path: test/e2e/qa-lab/runtime/docker-artifact-proof.ts - summary: Builds the package-backed functional image, launches the documented Compose services, validates health, and records artifact identities. + summary: Builds the package-backed functional image, launches the documented Compose services, validates endpoint and authenticated CLI health, forces and recovers Docker health state, captures diagnostics, and records artifact identities. timeoutMs: 1800000 args: - --artifact-base diff --git a/scripts/e2e/compose-setup.sh b/scripts/e2e/compose-setup.sh index 94a9dba268c5..c76f7350bf93 100755 --- a/scripts/e2e/compose-setup.sh +++ b/scripts/e2e/compose-setup.sh @@ -11,7 +11,15 @@ PROJECT_DIR="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-compose-proof.XXXXXX")" PROJECT_NAME="openclaw-compose-proof-$$" CLI_NAME="$PROJECT_NAME-cli-proof" TOKEN="compose-proof-$$-$(date +%s)" -COMPOSE=(docker compose --project-name "$PROJECT_NAME" --project-directory "$PROJECT_DIR" -f "$ROOT_DIR/docker-compose.yml") +WRONG_TOKEN="$TOKEN-wrong" +HEALTH_OVERRIDE_PATH="$PROJECT_DIR/compose-health-override.yaml" +COMPOSE=( + docker compose + --project-name "$PROJECT_NAME" + --project-directory "$PROJECT_DIR" + -f "$ROOT_DIR/docker-compose.yml" + -f "$HEALTH_OVERRIDE_PATH" +) cleanup() { docker_e2e_docker_cmd rm -f "$CLI_NAME" >/dev/null 2>&1 || true @@ -43,6 +51,141 @@ if ( ) { throw new Error(`${label} gateway health JSON is incomplete`); } +NODE + echo "$label accepted the configured token and returned a complete health envelope." +} + +assert_dockerfile_healthcheck() { + node - "$ROOT_DIR/Dockerfile" <<'NODE' +const fs = require("node:fs"); +const dockerfilePath = process.argv[2]; +const dockerfile = fs.readFileSync(dockerfilePath, "utf8").replace(/\\\r?\n[ \t]*/g, " "); +if (!/HEALTHCHECK\b[^\n]*CMD \["node", "dist\/docker-healthcheck\.js"\]/u.test(dockerfile)) { + throw new Error("Dockerfile does not install the built Gateway healthcheck"); +} +NODE + echo "Dockerfile healthcheck definition uses dist/docker-healthcheck.js." +} + +assert_effective_healthcheck() { + local label="$1" + local kind="$2" + local inspect_path="$3" + node - "$label" "$kind" "$inspect_path" <<'NODE' +const fs = require("node:fs"); +const label = process.argv[2]; +const kind = process.argv[3]; +const inspectPath = process.argv[4]; +const payload = JSON.parse(fs.readFileSync(inspectPath, "utf8")); +const actual = + kind === "compose" + ? payload?.services?.["openclaw-gateway"]?.healthcheck?.test + : payload?.[0]?.Config?.Healthcheck?.Test; +const expected = ["CMD", "node", "dist/docker-healthcheck.js"]; +if (JSON.stringify(actual) !== JSON.stringify(expected)) { + throw new Error(`${label} healthcheck mismatch: ${JSON.stringify(actual)}`); +} +NODE + echo "$label uses dist/docker-healthcheck.js." +} + +wait_for_gateway_health() { + local expected="$1" + local attempts="$2" + local health="" + for _ in $(seq 1 "$attempts"); do + health="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$GATEWAY_ID")" + if [ "$health" = "$expected" ]; then + return 0 + fi + if [ "$health" = "exited" ] || [ "$health" = "dead" ]; then + break + fi + sleep 1 + done + echo "Gateway health did not reach $expected (last state: $health)" >&2 + docker inspect --format '{{json .State.Health}}' "$GATEWAY_ID" >&2 || true + "${COMPOSE[@]}" logs --no-color openclaw-gateway >&2 || true + return 1 +} + +assert_probe_endpoints() { + local label="$1" + local output_path="$2" + "${COMPOSE[@]}" exec -T openclaw-gateway node - "$label" >"$output_path" <<'NODE' +const label = process.argv[2]; +const readJson = async (path) => { + const response = await fetch(`http://127.0.0.1:18789${path}`); + const body = await response.json(); + return { body, status: response.status }; +}; +const health = await readJson("/healthz"); +const ready = await readJson("/readyz"); +if ( + health.status !== 200 || + health.body?.ok !== true || + health.body?.status !== "live" +) { + throw new Error(`${label} /healthz did not report live: ${JSON.stringify(health)}`); +} +if ( + ready.status !== 200 || + ready.body?.ready !== true || + !Array.isArray(ready.body?.failing) || + ready.body.failing.length !== 0 +) { + throw new Error(`${label} /readyz did not report ready: ${JSON.stringify(ready)}`); +} +console.log(JSON.stringify({ healthz: health, label, readyz: ready })); +NODE + echo "$label: /healthz is live and /readyz is ready." +} + +assert_auth_rejected() { + local label="$1" + local stdout_path="$2" + local stderr_path="$3" + shift 3 + local exit_code + set +e + "$@" >"$stdout_path" 2>"$stderr_path" + exit_code=$? + set -e + if [ "$exit_code" -eq 0 ]; then + echo "$label accepted an invalid Gateway token" >&2 + return 1 + fi + node - "$label" "$stdout_path" "$stderr_path" <<'NODE' +const fs = require("node:fs"); +const label = process.argv[2]; +const output = [process.argv[3], process.argv[4]] + .map((path) => fs.readFileSync(path, "utf8")) + .join("\n"); +if (!/(unauthorized|token mismatch|authentication)/iu.test(output)) { + throw new Error(`${label} failed without an authentication diagnostic`); +} +NODE + echo "$label rejected an invalid Gateway token (exit $exit_code)." +} + +assert_health_state() { + local label="$1" + local expected="$2" + local health_path="$3" + node - "$label" "$expected" "$health_path" <<'NODE' +const fs = require("node:fs"); +const label = process.argv[2]; +const expected = process.argv[3]; +const healthPath = process.argv[4]; +const health = JSON.parse(fs.readFileSync(healthPath, "utf8")); +if (health?.Status !== expected || !Array.isArray(health.Log) || health.Log.length === 0) { + throw new Error(`${label} Docker health state is incomplete: ${JSON.stringify(health)}`); +} +const expectedExit = expected === "unhealthy" ? (code) => code !== 0 : (code) => code === 0; +if (!health.Log.some((entry) => expectedExit(Number(entry?.ExitCode)))) { + throw new Error(`${label} Docker health log lacks the expected exit status`); +} +console.log(`${label}: ${JSON.stringify({ log: health.Log.slice(-3), status: health.Status })}`); NODE } @@ -57,6 +200,15 @@ cat >"$PROJECT_DIR/config/openclaw.json" <"$HEALTH_OVERRIDE_PATH" <<'EOF' +services: + openclaw-gateway: + healthcheck: + interval: 1s + timeout: 5s + retries: 3 + start_period: 5s +EOF export OPENCLAW_IMAGE="$IMAGE_NAME" export OPENCLAW_CONFIG_DIR="$PROJECT_DIR/config" @@ -71,6 +223,10 @@ export OPENCLAW_CURRENT_PACKAGE_TGZ="$PACKAGE_TGZ" docker_e2e_build_or_reuse "$IMAGE_NAME" compose-setup "$ROOT_DIR/scripts/e2e/Dockerfile" "$ROOT_DIR" functional +assert_dockerfile_healthcheck +"${COMPOSE[@]}" config --format json >"$PROJECT_DIR/compose-config.json" +assert_effective_healthcheck "resolved Compose service" compose "$PROJECT_DIR/compose-config.json" + echo "Launching documented Docker Compose gateway topology..." "${COMPOSE[@]}" up -d --no-build openclaw-gateway GATEWAY_ID="$("${COMPOSE[@]}" ps -q openclaw-gateway)" @@ -78,29 +234,53 @@ if [ -z "$GATEWAY_ID" ]; then echo "Compose did not create openclaw-gateway" >&2 exit 1 fi +docker inspect "$GATEWAY_ID" >"$PROJECT_DIR/gateway-container.json" +assert_effective_healthcheck "running Gateway container" container "$PROJECT_DIR/gateway-container.json" -for _ in $(seq 1 180); do - health="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$GATEWAY_ID")" - if [ "$health" = "healthy" ]; then - break - fi - if [ "$health" = "unhealthy" ] || [ "$health" = "exited" ] || [ "$health" = "dead" ]; then - "${COMPOSE[@]}" logs --no-color openclaw-gateway >&2 - exit 1 - fi - sleep 1 -done -if [ "$(docker inspect --format '{{.State.Health.Status}}' "$GATEWAY_ID")" != "healthy" ]; then - "${COMPOSE[@]}" logs --no-color openclaw-gateway >&2 - echo "Compose gateway did not become healthy" >&2 - exit 1 -fi +wait_for_gateway_health healthy 180 +docker inspect --format '{{json .State.Health}}' "$GATEWAY_ID" >"$PROJECT_DIR/health-initial.json" +assert_health_state "initial healthy state" healthy "$PROJECT_DIR/health-initial.json" +assert_probe_endpoints "initial probes" "$PROJECT_DIR/probes-initial.json" "${COMPOSE[@]}" exec -T openclaw-gateway sh -lc 'node dist/index.js gateway health --token "$OPENCLAW_GATEWAY_TOKEN"' +assert_auth_rejected \ + "gateway service" \ + "$PROJECT_DIR/gateway-wrong-token.json" \ + "$PROJECT_DIR/gateway-wrong-token.err" \ + "${COMPOSE[@]}" exec -T openclaw-gateway \ + node dist/index.js gateway health --token "$WRONG_TOKEN" --json "${COMPOSE[@]}" exec -T openclaw-gateway node dist/index.js gateway health --token "$TOKEN" --json >"$PROJECT_DIR/gateway-health.json" assert_gateway_health_json "gateway service" "$PROJECT_DIR/gateway-health.json" +assert_auth_rejected \ + "CLI sidecar" \ + "$PROJECT_DIR/cli-wrong-token.json" \ + "$PROJECT_DIR/cli-wrong-token.err" \ + "${COMPOSE[@]}" run -T --rm --no-deps \ + openclaw-cli gateway health --token "$WRONG_TOKEN" --json "${COMPOSE[@]}" run -T --no-deps --name "$CLI_NAME" openclaw-cli gateway health --token "$TOKEN" --json >"$PROJECT_DIR/cli-health.json" assert_gateway_health_json "CLI sidecar" "$PROJECT_DIR/cli-health.json" + +echo "Forcing the configured Docker healthcheck to fail..." +docker exec --user 0:0 "$GATEWAY_ID" \ + sh -c 'test -f /app/dist/docker-healthcheck.js && mv /app/dist/docker-healthcheck.js /app/dist/docker-healthcheck.js.c3-disabled' +wait_for_gateway_health unhealthy 30 +docker inspect --format '{{json .State.Health}}' "$GATEWAY_ID" >"$PROJECT_DIR/health-unhealthy.json" +assert_health_state "forced unhealthy state" unhealthy "$PROJECT_DIR/health-unhealthy.json" +docker exec --user 0:0 "$GATEWAY_ID" \ + mv /app/dist/docker-healthcheck.js.c3-disabled /app/dist/docker-healthcheck.js +wait_for_gateway_health healthy 60 +docker inspect --format '{{json .State.Health}}' "$GATEWAY_ID" >"$PROJECT_DIR/health-recovered.json" +assert_health_state "recovered healthy state" healthy "$PROJECT_DIR/health-recovered.json" +assert_probe_endpoints "recovered probes" "$PROJECT_DIR/probes-recovered.json" + +"${COMPOSE[@]}" logs --no-color openclaw-gateway >"$PROJECT_DIR/gateway-compose.log" +if [ ! -s "$PROJECT_DIR/gateway-compose.log" ]; then + echo "Compose gateway logs were empty" >&2 + exit 1 +fi +echo "Compose gateway log tail:" +tail -n 40 "$PROJECT_DIR/gateway-compose.log" + GATEWAY_VERSION="$("${COMPOSE[@]}" exec -T openclaw-gateway node -p "require('./package.json').version")" node --import tsx "$ROOT_DIR/scripts/e2e/lib/docker-artifact-proof/write-identities.ts" \ @@ -112,8 +292,21 @@ node --import tsx "$ROOT_DIR/scripts/e2e/lib/docker-artifact-proof/write-identit --container "cli=$CLI_NAME" \ --detail "gateway:openclawVersion=$GATEWAY_VERSION" \ --detail "gateway:health=healthy" \ + --detail "gateway:dockerfileHealthcheckDefinition=passed" \ + --detail "gateway:composeHealthcheckResolved=passed" \ + --detail "gateway:runtimeHealthcheckEffective=passed" \ + --detail "gateway:healthz=live" \ + --detail "gateway:readyz=ready" \ + --detail "gateway:wrongTokenRejected=passed" \ + --detail "gateway:correctTokenAccepted=passed" \ + --detail "gateway:dockerUnhealthy=observed" \ + --detail "gateway:dockerRecovery=passed" \ + --detail "gateway:composeLogs=observed" \ + --detail "gateway:healthStateDiagnostics=observed" \ --detail "gateway:documentedHealthCommand=passed" \ --detail "gateway:healthJsonEnvelope=passed" \ + --detail "cli:wrongTokenRejected=passed" \ + --detail "cli:correctTokenAccepted=passed" \ --detail "cli:healthJsonEnvelope=passed" echo "Docker Compose setup proof passed."