refactor(verification): own browser fixture lifecycle

This commit is contained in:
morluto committed 2026-10-27 13:36:57 +08:00
1 parent 7d6bccb86d
commit c9df40f810
5 files changed
+322 -211

No files matched your search

+39 -20
View File
@@ -53,6 +53,7 @@ export const BROWSER_DEVTOOLS_STARTUP_TIMEOUT_MS = 60_000;
/** Owned child process and startup policy for Chrome CDP discovery. */
export interface BrowserStartupOptions {
/** Caller retains process error handling through its complete resource lifetime. */
readonly child: ChildProcess;
readonly executable: string;
readonly activePortPath: string;
@@ -71,23 +72,29 @@ export const waitForBrowserDevtoolsPort = async (
const terminal = childTermination(options, startedAt, timeoutMs);
const deadline = startedAt + timeoutMs;
while (true) {
const terminalState = currentTerminalState(options.child);
if (terminalState !== null)
throw startupError(options, startedAt, timeoutMs, terminalState);
try {
while (true) {
const terminalState = currentTerminalState(options.child);
if (terminalState !== null)
throw startupError(options, startedAt, timeoutMs, terminalState);
const port = await readDevtoolsPort(options.activePortPath);
if (port !== null) return port;
const port = await readDevtoolsPort(options.activePortPath);
if (port !== null) return port;
const remainingMs = deadline - performance.now();
if (remainingMs <= 0)
throw startupError(options, startedAt, timeoutMs, { failure: "timeout" });
const remainingMs = deadline - performance.now();
if (remainingMs <= 0)
throw startupError(options, startedAt, timeoutMs, {
failure: "timeout",
});
const outcome = await Promise.race([
terminal,
delay(Math.min(pollIntervalMs, remainingMs)).then(() => null),
]);
if (outcome !== null) throw outcome;
const outcome = await Promise.race([
terminal.outcome,
delay(Math.min(pollIntervalMs, remainingMs)).then(() => null),
]);
if (outcome !== null) throw outcome;
}
} finally {
terminal.close();
}
};
@@ -101,17 +108,19 @@ const childTermination = (
options: BrowserStartupOptions,
startedAt: number,
startupTimeoutMs: number,
): Promise<BrowserStartupError> =>
new Promise((resolve) => {
options.child.once("error", (cause: unknown) => {
): { readonly outcome: Promise<BrowserStartupError>; close(): void } => {
let onError: (cause: unknown) => void;
let onExit: () => void;
const outcome = new Promise<BrowserStartupError>((resolve) => {
onError = (cause: unknown) => {
resolve(
startupError(options, startedAt, startupTimeoutMs, {
failure: "spawn-error",
cause,
}),
);
});
options.child.once("exit", () => {
};
onExit = () => {
resolve(
startupError(
options,
@@ -122,8 +131,18 @@ const childTermination = (
: { failure: "signalled" },
),
);
});
};
options.child.once("error", onError);
options.child.once("exit", onExit);
});
return {
outcome,
close: () => {
options.child.off("error", onError);
options.child.off("exit", onExit);
},
};
};
const currentTerminalState = (child: ChildProcess): TerminalState | null => {
if (child.signalCode !== null) return { failure: "signalled" };
+97 -103
View File
@@ -1,12 +1,16 @@
import assert from "node:assert/strict";
import { randomUUID } from "node:crypto";
import { once } from "node:events";
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { spawnOwnedProviderProcess } from "../../dist/process/ProviderProcess.js";
import { createSystemProcessOwnershipHost } from "../../dist/process/ProcessOwnershipObservation.js";
import { liveProcesses } from "../../dist/process/ProcessOwnershipProcessTree.js";
import {
ProviderProcessSupervisor,
spawnOwnedProviderProcess,
} from "../../dist/process/ProviderProcess.js";
import { cleanupOwnedProcessGroup } from "../../dist/process/ProcessOwnership.js";
import { ArtifactResourceScope } from "../../dist/artifacts/ArtifactResourceScope.js";
import { AnalysisCleanupLease } from "../../dist/artifacts/AnalysisCleanupLease.js";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
import { CdpConnection } from "../../dist/browser/CdpConnection.js";
import { importVerifierSource } from "./verifier-source-loader.mjs";
@@ -18,13 +22,30 @@ const { BROWSER_DEVTOOLS_STARTUP_TIMEOUT_MS, waitForBrowserDevtoolsPort } =
import.meta.url,
);
/** Launch the synthetic fixture browser and await CDP in its own verified process group. */
export async function startRuntimeFixtureBrowser(executable, origin) {
/** Launch the verifier browser and await CDP in its own verified process group. */
export async function startRuntimeFixtureBrowser(executable, initialUrl) {
await requireOrdinaryWindowsVerifierCaller();
const profile = await mkdtemp(join(tmpdir(), "rea-real-web-runtime-"));
let owned;
let supervisor;
let stopOwnedBrowser;
const resources = new ArtifactResourceScope();
const cleanup = async () => {
if (stopOwnedBrowser !== undefined) await stopOwnedBrowser();
await rm(profile, {
recursive: true,
force: true,
maxRetries: 5,
retryDelay: 100,
});
};
resources.retain({
kind: "lease",
resource: profile,
lease: AnalysisCleanupLease.forCloseOwner({ close: cleanup }, profile),
});
const close = () => resources.close();
try {
owned = await spawnOwnedProviderProcess({
const owned = await spawnOwnedProviderProcess({
command: executable,
arguments: [
"--headless=new",
@@ -41,119 +62,92 @@ export async function startRuntimeFixtureBrowser(executable, origin) {
"--disable-component-update",
"--disable-dev-shm-usage",
"--disable-sync",
origin,
"--metrics-recording-only",
initialUrl,
],
runId: randomUUID(),
// Installed Chrome launch scripts exec another binary; do not require an argv prefix.
expectedCommand: null,
});
owned.process.stdout?.resume();
let stderr = "";
owned.process.stderr?.on("data", (chunk) => {
if (stderr.length < 65536) stderr += chunk;
});
supervisor = new ProviderProcessSupervisor(
{
...owned,
ownsProcessLifetime: true,
cleanup:
owned.cleanup ?? (() => cleanupOwnedProcessGroup(owned.ownership)),
},
{
maxDiagnosticBytes: { stdout: 0, stderr: 65_536 },
onDiagnostic: (event) => {
if (event.type === "error")
process.stderr.write(
`Browser verifier process error: ${event.message}\n`,
);
},
},
);
stopOwnedBrowser = createOwnedBrowserStop(supervisor, profile);
const port = await waitForBrowserDevtoolsPort({
child: owned.process,
executable,
activePortPath: join(profile, "DevToolsActivePort"),
stderr: () => stderr,
stderr: () => supervisor.snapshot().stderr.text,
// Match the broader real-browser verifier's allowance for cold Chrome
// startup on shared runners; early process failures still reject at once.
timeoutMs: BROWSER_DEVTOOLS_STARTUP_TIMEOUT_MS,
});
return {
port,
close: async () => {
await stopOwnedBrowser(owned, profile);
await rm(profile, {
recursive: true,
force: true,
maxRetries: 5,
retryDelay: 100,
});
},
};
return { port, profile, process: owned.process, close };
} catch (cause) {
if (owned !== undefined) await stopOwnedBrowser(owned, profile);
await rm(profile, {
recursive: true,
force: true,
maxRetries: 5,
retryDelay: 100,
});
throw cause;
return withVerifierCleanup(async () => {
throw cause;
}, [close]);
}
}
async function stopOwnedBrowser(owned, profile) {
const child = owned.process;
const exited =
child.exitCode === null && child.signalCode === null
? once(child, "exit")
: Promise.resolve();
void exited.catch(() => undefined);
function createOwnedBrowserStop(supervisor, profile) {
let connection;
let timer;
const escalation = setTimeout(() => {
if (child.exitCode === null && child.signalCode === null)
child.kill("SIGKILL");
}, 2_000);
try {
if (child.exitCode === null && child.signalCode === null) {
try {
const [port, browserPath] = (
await readFile(join(profile, "DevToolsActivePort"), "utf8")
)
.trim()
.split("\n");
assert.match(port, /^\d+$/);
assert.match(browserPath, /^\/devtools\/browser\/[^\s]+$/);
const signal = AbortSignal.timeout(2_000);
connection = await CdpConnection.connect(
`ws://127.0.0.1:${port}${browserPath}`,
"inspect_web_page",
signal,
);
// This browser belongs to the verifier; public REA operations never close it.
await connection.send("Browser.close", {}, undefined, signal);
} catch {
if (child.exitCode === null && child.signalCode === null)
child.kill("SIGTERM");
return async () => {
const child = supervisor.launch.process;
try {
if (
connection === undefined &&
child.exitCode === null &&
child.signalCode === null
) {
try {
const [port, browserPath] = (
await readFile(join(profile, "DevToolsActivePort"), "utf8")
)
.trim()
.split("\n");
assert.match(port, /^\d+$/);
assert.match(browserPath, /^\/devtools\/browser\/[^\s]+$/);
const signal = AbortSignal.timeout(2_000);
await CdpConnection.connect(
`ws://127.0.0.1:${port}${browserPath}`,
"inspect_web_page",
signal,
undefined,
(opened) => {
connection = opened;
},
);
await connection.send("Browser.close", {}, undefined, signal);
} catch {
// CDP may be unavailable or already closing. The supervisor still owns
// verified process-tree termination, including descendants and retries.
}
}
} finally {
await withVerifierCleanup(async () => {
const stopped = await supervisor.stop();
assert.notEqual(stopped.status, "incomplete", JSON.stringify(stopped));
}, [
async () => {
await connection?.close();
connection = undefined;
},
]);
}
if (owned.cleanup !== undefined) {
const result = await owned.cleanup();
assert.equal(result.cleaned, true, JSON.stringify(result));
}
await Promise.race([
exited,
new Promise((_, reject) => {
timer = setTimeout(
() =>
reject(
new Error("Owned fixture browser did not exit after shutdown"),
),
5_000,
);
}),
]);
} finally {
clearTimeout(escalation);
clearTimeout(timer);
await connection?.close();
}
if (process.platform === "win32") return;
const host = createSystemProcessOwnershipHost();
const deadline = Date.now() + 5_000;
while (true) {
const members = liveProcesses(await host.listProcesses()).filter(
(member) => member.processGroupId === owned.ownership.processGroupId,
);
if (members.length === 0) return;
if (Date.now() >= deadline)
throw new Error(
`Owned fixture group remains: ${JSON.stringify(members.map(({ pid }) => pid))}`,
);
await new Promise((resolve) => setTimeout(resolve, 25));
}
};
}
+8 -77
View File
@@ -1,15 +1,12 @@
#!/usr/bin/env node
import { spawn } from "node:child_process";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { access, mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { access } from "node:fs/promises";
import { fileURLToPath } from "node:url";
import { CdpBrowserProvider } from "../dist/browser/CdpBrowserProvider.js";
import { importVerifierSource } from "./lib/verifier-source-loader.mjs";
import { startRuntimeFixtureBrowser } from "./lib/browser-runtime-fixture-lifecycle.mjs";
import { createBrowserScenarioProvider } from "../dist/composition/browserScenario.js";
import {
@@ -55,12 +52,6 @@ import { verifyScenarioStorageFingerprints } from "./lib/browser-scenario-storag
import { verifyScenarioServiceWorkers } from "./lib/browser-scenario-service-workers-e2e.mjs";
import { verifyBrowserUrlUserInfo } from "./lib/browser-url-userinfo-e2e.mjs";
const { BROWSER_DEVTOOLS_STARTUP_TIMEOUT_MS, waitForBrowserDevtoolsPort } =
await importVerifierSource(
new URL("./lib/browser-process-startup.mts", import.meta.url).href,
import.meta.url,
);
const SCENARIO_SECRET_VALUE = "rea-browser-verifier-secret";
const SCENARIO_URL_SECRET_VALUE = "rea-browser-url-verifier-secret";
process.env.REA_BROWSER_VERIFIER_SECRET = SCENARIO_SECRET_VALUE;
@@ -69,48 +60,17 @@ const verifierRun = createVerifierRun();
await requireOrdinaryWindowsVerifierCaller();
const executable = await browserExecutable();
let profile;
let site;
let browser;
let pageProxy;
let report;
await withVerifierCleanup(async () => {
profile = await mkdtemp(join(tmpdir(), "rea-real-browser-"));
site = await startBrowserVerifierSite();
browser = spawn(
browser = await startRuntimeFixtureBrowser(
executable,
[
"--headless=new",
"--password-store=basic",
...(process.env.REA_BROWSER_NO_SANDBOX === "true"
? ["--no-sandbox"]
: []),
"--remote-debugging-address=127.0.0.1",
"--remote-debugging-port=0",
`--user-data-dir=${profile}`,
"--no-first-run",
"--no-default-browser-check",
"--disable-background-networking",
"--disable-component-update",
"--disable-dev-shm-usage",
"--disable-sync",
"--metrics-recording-only",
`${site.origin}/app?startup=browser-secret-value`,
],
{ stdio: ["ignore", "ignore", "pipe"] },
`${site.origin}/app?startup=browser-secret-value`,
);
let stderr = "";
browser.stderr.on("data", (chunk) => {
if (stderr.length < 64 * 1_024) stderr += chunk.toString("utf8");
});
const port = await waitForBrowserDevtoolsPort({
child: browser,
executable,
activePortPath: join(profile, "DevToolsActivePort"),
stderr: () => stderr,
timeoutMs: BROWSER_DEVTOOLS_STARTUP_TIMEOUT_MS,
});
const endpoint = `http://127.0.0.1:${String(port)}`;
const endpoint = `http://127.0.0.1:${String(browser.port)}`;
const provider = new CdpBrowserProvider();
const target = await pageTarget(provider, endpoint);
const observed = await provider.inspectPage(
@@ -262,7 +222,7 @@ await withVerifierCleanup(async () => {
await verifyPageScopedTransport(provider, pageProxy, site.origin);
const browserSocketTransport = await verifyBrowserSocketTransport({
provider,
profile,
profile: browser.profile,
endpoint,
target,
origin: site.origin,
@@ -308,7 +268,7 @@ await withVerifierCleanup(async () => {
"Scenario CLI did not report external disconnect ownership",
);
const versionResponse = await fetch(`${endpoint}/json/version`);
if (!versionResponse.ok || browser.exitCode !== null)
if (!versionResponse.ok || browser.process.exitCode !== null)
throw new Error("Scenario attachment terminated its external browser");
const profilesBefore = await scenarioProfiles();
@@ -457,7 +417,7 @@ await withVerifierCleanup(async () => {
async () => {
if (browser !== undefined) {
process.stderr.write("Browser verifier cleanup: stopping owned Chrome\n");
await stopProcess(browser);
await browser.close();
}
},
async () => {
@@ -466,15 +426,6 @@ await withVerifierCleanup(async () => {
await site.close();
}
},
async () => {
if (profile !== undefined)
await rm(profile, {
recursive: true,
force: true,
maxRetries: 10,
retryDelay: 100,
});
},
]);
process.stdout.write(
`${JSON.stringify({
@@ -629,23 +580,3 @@ async function pageTarget(provider, endpoint) {
function delay(milliseconds) {
return new Promise((resolve) => setTimeout(resolve, milliseconds));
}
async function stopProcess(child) {
if (child.exitCode !== null || child.signalCode !== null) return;
const gracefulExit = waitForExit(child);
child.kill("SIGTERM");
const exited = await Promise.race([
gracefulExit.then(() => true),
delay(2_000).then(() => false),
]);
if (!exited) {
if (child.exitCode !== null || child.signalCode !== null) return;
const forcedExit = waitForExit(child);
child.kill("SIGKILL");
await forcedExit;
}
}
function waitForExit(child) {
return new Promise((resolve) => child.once("exit", resolve));
}
@@ -150,16 +150,16 @@ describe("browser process startup", () => {
it("keeps one owner for the DevToolsActivePort deadline", async () => {
expect(BROWSER_DEVTOOLS_STARTUP_TIMEOUT_MS).toBe(60_000);
const callers = [
"../../../scripts/verify-real-browser.mjs",
"../../../scripts/lib/browser-runtime-fixture-lifecycle.mjs",
];
for (const caller of callers) {
const source = await readFile(new URL(caller, import.meta.url), "utf8");
expect(devtoolsPortDeadlines(source)).toEqual([
["BROWSER_DEVTOOLS_STARTUP_TIMEOUT_MS"],
]);
}
const source = await readFile(
new URL(
"../../../scripts/lib/browser-runtime-fixture-lifecycle.mjs",
import.meta.url,
),
"utf8",
);
expect(devtoolsPortDeadlines(source)).toEqual([
["BROWSER_DEVTOOLS_STARTUP_TIMEOUT_MS"],
]);
});
});
@@ -1,5 +1,6 @@
import { execFile } from "node:child_process";
import { readdir } from "node:fs/promises";
import { readdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { promisify } from "node:util";
import { expect, it } from "vitest";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
@@ -44,3 +45,169 @@ it.skipIf(process.platform === "win32")(
expect(await readdir(root)).toEqual([]);
},
);
it.skipIf(process.platform === "win32")(
"retains browser ownership after denied cleanup and retries the whole process group",
async () => {
const root = await createTestTempDirectory("rea-runtime-owned-browser-");
const executable = await writeFixtureBrowser(root);
const helper = new URL(
"../../../scripts/lib/browser-runtime-fixture-lifecycle.mjs",
import.meta.url,
);
const script = `
import assert from 'node:assert/strict';
import { existsSync, readFileSync } from 'node:fs';
import { join } from 'node:path';
const { startRuntimeFixtureBrowser } = await import(${JSON.stringify(helper.href)});
const fixture = await startRuntimeFixtureBrowser(${JSON.stringify(executable)}, 'http://127.0.0.1');
const descendant = Number(readFileSync(join(fixture.profile, 'descendant'), 'utf8'));
fixture.process.emit('error', Object.assign(new Error('late EPERM fixture'), { code: 'EPERM' }));
const kill = process.kill;
let denied = false;
process.kill = function(pid, signal) {
if (pid < 0 && signal !== 0) {
denied = true;
throw Object.assign(new Error('denied fixture group cleanup'), { code: 'EPERM' });
}
return kill(pid, signal);
};
try {
await assert.rejects(fixture.close(), /incomplete/);
assert.equal(denied, true);
assert.equal(existsSync(fixture.profile), true);
assert.ok(fixture.process.listenerCount('error') > 0);
} finally {
process.kill = kill;
await fixture.close();
}
assert.equal(existsSync(fixture.profile), false);
assert.equal(fixture.process.listenerCount('error'), 0);
const { createSystemProcessOwnershipHost } = await import(${JSON.stringify(new URL("../../../dist/process/ProcessOwnershipObservation.js", import.meta.url).href)});
const { liveProcesses } = await import(${JSON.stringify(new URL("../../../dist/process/ProcessOwnershipProcessTree.js", import.meta.url).href)});
const live = liveProcesses(await createSystemProcessOwnershipHost().listProcesses());
assert.equal(live.some(item => item.pid === fixture.process.pid || item.pid === descendant), false);
// The process may finish while the separately owned socket still needs retry.
const { CdpConnection } = await import(${JSON.stringify(new URL("../../../dist/browser/CdpConnection.js", import.meta.url).href)});
const { WebSocketServer } = await import('ws');
const server = new WebSocketServer({ host: '127.0.0.1', port: 0 });
await new Promise(resolve => server.once('listening', resolve));
server.on('connection', socket => socket.on('message', data => {
const { id } = JSON.parse(data.toString());
socket.send(JSON.stringify({ id, result: {} }));
}));
const connect = CdpConnection.connect;
let connections = 0;
let closeAttempts = 0;
let socketOwner;
CdpConnection.connect = async function(...args) {
connections++;
args[0] = 'ws://127.0.0.1:' + server.address().port + '/devtools/browser/fixture';
const onConnected = args[4];
args[4] = (connection) => {
socketOwner = connection;
const close = connection.close.bind(connection);
connection.close = async () => {
closeAttempts++;
if (closeAttempts === 1) throw new Error('denied socket close');
await close();
};
onConnected?.(connection);
};
return connect.apply(this, args);
};
const socketFixture = await startRuntimeFixtureBrowser(${JSON.stringify(executable)}, 'http://127.0.0.1');
try {
await assert.rejects(socketFixture.close(), cause => hasFailure(cause, 'denied socket close'));
assert.ok(socketFixture.process.exitCode !== null || socketFixture.process.signalCode !== null);
assert.equal(existsSync(socketFixture.profile), true);
assert.ok(socketOwner, 'Acquire ownership before the WebSocket handshake');
assert.equal(server.clients.size, 1, 'Failed socket close must keep its actual open transport');
await socketFixture.close();
assert.equal(connections, 1, 'Retry must reuse the exact retained socket');
assert.equal(closeAttempts, 2);
assert.equal(existsSync(socketFixture.profile), false);
} finally {
CdpConnection.connect = connect;
await socketFixture.close();
await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve()));
}
// A late launcher exit must preserve the sibling/descendant cleanup lease
// even though startup never returned a browser handle.
process.kill = function(pid, signal) {
if (pid < 0 && signal !== 0)
throw Object.assign(new Error('denied startup cleanup'), { code: 'EPERM' });
return kill(pid, signal);
};
let startupFailure;
try {
await startRuntimeFixtureBrowser(${JSON.stringify(executable)}, 'http://127.0.0.1/startup-fail');
throw new Error('Expected startup failure');
} catch (cause) {
startupFailure = cause;
} finally {
process.kill = kill;
}
function hasFailure(cause, message) {
if (!cause) return false;
return cause?.message === message || hasFailure(cause?.cause, message) ||
(cause?.errors ?? []).some(error => hasFailure(error, message));
}
function findCleanupFailure(cause) {
if (cause.cleanupLeases?.length) return cause;
for (const error of cause.errors ?? []) {
const found = findCleanupFailure(error);
if (found) return found;
}
}
const retained = findCleanupFailure(startupFailure);
assert.ok(retained, 'Failed startup must retain exact cleanup authority');
const lease = retained.cleanupLeases[0];
const [failedProfile] = lease.cleanupResources;
const lateDescendant = Number(readFileSync(join(failedProfile, 'descendant'), 'utf8'));
assert.equal(existsSync(failedProfile), true);
await lease.release();
assert.equal(existsSync(failedProfile), false);
const afterRetry = liveProcesses(await createSystemProcessOwnershipHost().listProcesses());
assert.equal(afterRetry.some(item => item.pid === lateDescendant), false);
console.log(JSON.stringify({ retried: true, descendant_removed: true, startup_retry: true, socket_retry: true }));
`;
const { stdout, stderr } = await promisify(execFile)(
process.execPath,
["--input-type=module", "-e", script],
{ timeout: 15_000 },
);
expect(JSON.parse(stdout)).toEqual({
retried: true,
descendant_removed: true,
startup_retry: true,
socket_retry: true,
});
expect(stderr).toContain(
"Browser verifier process error: late EPERM fixture",
);
},
);
async function writeFixtureBrowser(root: string): Promise<string> {
const executable = join(root, "fixture-browser.cjs");
await writeFile(
executable,
`#!${process.execPath}
const { spawn } = require('node:child_process');
const { writeFileSync } = require('node:fs');
const { join } = require('node:path');
const profile = process.argv.find(value => value.startsWith('--user-data-dir=')).split('=').slice(1).join('=');
const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)']);
writeFileSync(join(profile, 'descendant'), String(child.pid));
if (!process.argv.at(-1).endsWith('/startup-fail'))
writeFileSync(join(profile, 'DevToolsActivePort'), '1\\n/devtools/browser/fixture');
else setTimeout(() => process.exit(3), 30);
console.error('owned fixture diagnostics');
setInterval(() => {}, 1000);
`,
{ mode: 0o755 },
);
return executable;
}