From 9a68ce57a0891551c3494bcdb945f81a987aadd6 Mon Sep 17 00:00:00 2001 From: ShuaixinHuang <1836656514@qq.com> Date: Tue, 3 Nov 2026 23:25:44 +0800 Subject: [PATCH 01/94] test(elf): cover big-endian ABI notes through CLI and MCP (#8803) --- tests/acceptance/elfNotesAbiEndian.test.ts | 113 +++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 tests/acceptance/elfNotesAbiEndian.test.ts diff --git a/tests/acceptance/elfNotesAbiEndian.test.ts b/tests/acceptance/elfNotesAbiEndian.test.ts new file mode 100644 index 0000000000..819b11051b --- /dev/null +++ b/tests/acceptance/elfNotesAbiEndian.test.ts @@ -0,0 +1,113 @@ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; + +import { toolContract } from "../../src/contracts/toolContracts.js"; +import { + elfNote, + elfNoteFixture, +} from "../../src/native/elf/ElfNotes.fixture.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +// Literal big-endian descriptors keep the expected values independent of the +// fixture's integer writers and the parser's byte-order branch. +const knownAbi = Buffer.from("00000000010203041020304080010203", "hex"); +const unknownAbi = Buffer.from("01000000010203041020304080010203", "hex"); +const extendedAbi = Buffer.concat([knownAbi, Buffer.from("01020304", "hex")]); +const shortAbi = knownAbi.subarray(0, 12); +const descriptorHash = (bytes: Buffer): string => + createHash("sha256").update(bytes).digest("hex"); + +for (const { bits, ownership, machine } of [ + { bits: 32, ownership: "program", machine: 20 }, + { bits: 64, ownership: "section", machine: 21 }, +] as const) + cliTest( + `preserves big-endian ELF${String(bits)} ABI facts through CLI and stdio MCP`, + async ({ cli }) => { + const root = await createTestTempDirectory("rea-elf-abi-endian-"); + const path = join(root, "selected.elf"); + const descriptors = [ + knownAbi, + unknownAbi, + shortAbi, + extendedAbi, + knownAbi, + ]; + const notes = Buffer.concat([ + ...descriptors + .slice(0, 4) + .map((bytes) => elfNote("GNU", 1, bytes, false)), + elfNote("OTHER", 1, knownAbi, false), + ]); + const bytes = elfNoteFixture({ + bits, + ownership, + machine, + little: false, + notes, + }); + await writeFile(path, bytes); + + const output = await cli.sharedCliRun!({ + arguments: ["inspect-elf-notes", path, "--json"], + }); + expect(output.exitCode, output.stderr).toBe(0); + const call = await cli.sharedMcpCall!(); + const response = await call("inspect_elf_notes", { path }); + expect(response.isError, JSON.stringify(response.content)).not.toBe(true); + + const schema = toolContract("inspect_elf_notes").outputSchema; + const reports = [output.json, response.structuredContent].map((value) => { + const evidence = schema.parse(value); + expect(evidence.subject).toMatchObject({ + local_path: path, + digest: { sha256: descriptorHash(bytes) }, + }); + const report = evidence.normalized_result; + expect(report).toMatchObject({ + format: `elf${String(bits)}`, + byte_order: "big", + machine, + coverage: { + status: "complete", + examined_tables: 1, + examined_entries: 5, + }, + }); + expect(report.entries).toHaveLength(5); + expect(report.entries[0]?.gnu?.abi_tag).toEqual({ + os: 0, + os_name: "linux", + major: 0x01020304, + minor: 0x10203040, + subminor: 0x80010203, + }); + expect(report.entries[1]?.gnu?.abi_tag).toEqual({ + os: 0x01000000, + os_name: null, + major: 0x01020304, + minor: 0x10203040, + subminor: 0x80010203, + }); + // Unexpected lengths stay raw; no ABI values are invented or truncated. + for (const index of [2, 3]) + expect(report.entries[index]?.gnu?.abi_tag).toBeNull(); + expect(report.entries[4]).toMatchObject({ + owner: "OTHER", + type_name: "unknown", + gnu: null, + }); + for (const [index, descriptor] of descriptors.entries()) + expect(report.entries[index]?.descriptor).toMatchObject({ + bytes: descriptor.length, + sha256: descriptorHash(descriptor), + }); + return report; + }); + expect(reports[0]).toEqual(reports[1]); + expect((await readFile(path)).equals(bytes)).toBe(true); + }, + ); From 26b033d51fd627af0cfdf47efcbdb346484e1c85 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 3 Nov 2026 23:48:22 +0800 Subject: [PATCH 02/94] fix(process): classify withheld Darwin environments before parser failures --- bridge/process/ProcessRunTokenReader.swift | 45 ++++++++++--------- .../darwinProcessRunTokenReader.test.ts | 23 ++++++++++ .../fixtures/processRunTokenParserProbe.swift | 37 +++++++++++++++ 3 files changed, 83 insertions(+), 22 deletions(-) diff --git a/bridge/process/ProcessRunTokenReader.swift b/bridge/process/ProcessRunTokenReader.swift index 599fbdb1b4..931eb48750 100644 --- a/bridge/process/ProcessRunTokenReader.swift +++ b/bridge/process/ProcessRunTokenReader.swift @@ -228,6 +228,22 @@ func processRunTokenObservation( parse: Result, codeSigningFlags: () -> UInt32? ) -> RunTokenObservation { + // Native inspection restrictions apply even when argv is incomplete or + // overwritten, before parsing reaches the environment. Keep this fact at + // the native boundary; cleanup still decides whether the process is related. + func withheldEnvironment() -> RunTokenObservation? { + guard let flags = codeSigningFlags() else { return nil } + if (flags & csPlatformBinary) != 0 { + return unavailable(pid: pid, reason: "platform_binary_environment_withheld") + } + if (flags & csRestrict) != 0 { + return unavailable(pid: pid, reason: "restricted_process_environment_withheld") + } + return nil + } + func unavailableEnvironment(_ reason: String) -> RunTokenObservation { + withheldEnvironment() ?? unavailable(pid: pid, reason: reason) + } switch parse { case .success(let parsed): if let runId = parsed.runId { @@ -238,40 +254,25 @@ func processRunTokenObservation( if parsed.environmentEntryCount > 0 { return RunTokenObservation(pid: pid, state: "readable", run_id: nil, reason: nil) } - let flags = codeSigningFlags() - if let flags, (flags & csPlatformBinary) != 0 { - return unavailable(pid: pid, reason: "platform_binary_environment_withheld") - } - if let flags, (flags & csRestrict) != 0 { - return unavailable(pid: pid, reason: "restricted_process_environment_withheld") - } + if let withheld = withheldEnvironment() { return withheld } return RunTokenObservation(pid: pid, state: "readable", run_id: nil, reason: nil) case .failure(let failure): switch failure { case .environmentUnavailable: - let flags = codeSigningFlags() - if let flags, (flags & csPlatformBinary) != 0 { - return unavailable(pid: pid, reason: "platform_binary_environment_withheld") - } - if let flags, (flags & csRestrict) != 0 { - return unavailable(pid: pid, reason: "restricted_process_environment_withheld") - } - return unavailable(pid: pid, reason: "environment_unavailable") + return unavailableEnvironment("environment_unavailable") case .ambiguousEnvironmentBoundary: - if let flags = codeSigningFlags(), (flags & csRestrict) != 0 { - return unavailable(pid: pid, reason: "restricted_process_environment_withheld") - } - return unavailable(pid: pid, reason: "ambiguous_environment_boundary") + return unavailableEnvironment("ambiguous_environment_boundary") + // These failures already establish that token bytes were available. case .duplicateToken: return unavailable(pid: pid, reason: "duplicate_run_token") case .invalidTokenEncoding: return unavailable(pid: pid, reason: "invalid_run_token_encoding") case .appleVectorUnavailable: - return unavailable(pid: pid, reason: "apple_vector_unavailable") + return unavailableEnvironment("apple_vector_unavailable") case .invalidBuffer: - return unavailable(pid: pid, reason: "malformed_procargs") + return unavailableEnvironment("malformed_procargs") case .invalidBufferAt(let stage): - return unavailable(pid: pid, reason: "malformed_procargs_\(stage.rawValue)") + return unavailableEnvironment("malformed_procargs_\(stage.rawValue)") } } } diff --git a/tests/boundary/process/darwinProcessRunTokenReader.test.ts b/tests/boundary/process/darwinProcessRunTokenReader.test.ts index 247d3e3639..63f77789c4 100644 --- a/tests/boundary/process/darwinProcessRunTokenReader.test.ts +++ b/tests/boundary/process/darwinProcessRunTokenReader.test.ts @@ -1317,6 +1317,29 @@ it.skipIf(!onDarwin)( missingAppleBoundaryFailsClosed: true, malformedBuffersRejected: true, invalidPointerSizeFailsClosed: true, + incompleteRestrictedArguments: [ + "malformed_procargs_path_padding", + "malformed_procargs_arguments", + "apple_vector_unavailable", + ].map((reason) => [ + { + state: "unavailable", + reason: "platform_binary_environment_withheld", + }, + { + state: "unavailable", + reason: "restricted_process_environment_withheld", + }, + { state: "unavailable", reason }, + ]), + signedReadableTokens: [true, true], + signedInvalidTokens: [ + "duplicate_run_token", + "invalid_run_token_encoding", + ].map((reason) => [ + { state: "unavailable", reason }, + { state: "unavailable", reason }, + ]), malformedStages: [ { state: "unavailable", reason: "malformed_procargs_header" }, { state: "unavailable", reason: "malformed_procargs_argument_count" }, diff --git a/tests/fixtures/processRunTokenParserProbe.swift b/tests/fixtures/processRunTokenParserProbe.swift index ed359869d8..335ce435b3 100644 --- a/tests/fixtures/processRunTokenParserProbe.swift +++ b/tests/fixtures/processRunTokenParserProbe.swift @@ -193,6 +193,40 @@ struct ProcessRunTokenParserProbe { parse: .failure(.invalidBuffer), codeSigningFlags: { 0 } ) + // A restricted native process can expose an incomplete argv layout before + // the parser reaches the withheld environment (CI observed mdworker here). + let incompleteRestrictedArguments = [ + RunTokenReadError.invalidBufferAt(.pathPadding), + .invalidBufferAt(.arguments), + .appleVectorUnavailable, + ].map { failure in + [UInt32(0x0400_0000), UInt32(0x0000_0800), nil].map { flags in + let observation = processRunTokenObservation( + pid: 1, + parse: .failure(failure), + codeSigningFlags: { flags } + ) + return ["state": observation.state, "reason": observation.reason ?? ""] + } + } + let signedReadableTokens = [UInt32(0x0400_0000), UInt32(0x0000_0800)].map { flags in + let observation = processRunTokenObservation( + pid: 1, + parse: .success(realToken), + codeSigningFlags: { flags } + ) + return observation.state == "readable" && observation.run_id == "owned-token" + } + let signedInvalidTokens = [RunTokenReadError.duplicateToken, .invalidTokenEncoding].map { failure in + [UInt32(0x0400_0000), UInt32(0x0000_0800)].map { flags in + let observation = processRunTokenObservation( + pid: 1, + parse: .failure(failure), + codeSigningFlags: { flags } + ) + return ["state": observation.state, "reason": observation.reason ?? ""] + } + } let duplicateTokenFailsClosed: Bool do { _ = try parseProcessEnvironment(in: fixture( @@ -311,6 +345,9 @@ struct ProcessRunTokenParserProbe { "malformedBuffersRejected": malformedBuffersRejected, "invalidPointerSizeFailsClosed": invalidPointerSizeFailsClosed, "malformedStages": malformedStages, + "incompleteRestrictedArguments": incompleteRestrictedArguments, + "signedReadableTokens": signedReadableTokens, + "signedInvalidTokens": signedInvalidTokens, "malformedObservation": [ "state": malformedObservation.state, "reason": malformedObservation.reason as Any? ?? NSNull(), From f3e5b33538e210dee41c98d72036e7849b51d6c3 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:14:46 +0800 Subject: [PATCH 03/94] fix(artifacts): settle transient DMG verification within its deadline --- src/artifacts/NativeDmgArtifactReader.ts | 11 ++++- .../native/nativeDmgArtifactReader.test.ts | 45 ++++++++++++------- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/artifacts/NativeDmgArtifactReader.ts b/src/artifacts/NativeDmgArtifactReader.ts index a904b49e0f..b5b5f931ef 100644 --- a/src/artifacts/NativeDmgArtifactReader.ts +++ b/src/artifacts/NativeDmgArtifactReader.ts @@ -36,7 +36,8 @@ export const DMG_EXISTING_ATTACHMENT_RECOVERY = "hdiutil refused a private read-only DMG attachment after listing an existing attachment of this image path. If it is still attached, ask its owner to release it when safe, then retry. REA does not borrow or detach existing attachments."; const DMG_CHECKSUM_ABSENT_LIMITATION = "hdiutil verify found no stored checksum in this disk image, so its integrity was not verified; the root SHA-256 identifies the inventoried bytes."; -const VERIFY_SETTLE_DELAYS_MS = [250, 500] as const; +const VERIFY_TRANSIENT_DELAY_MS = 250; +const VERIFY_UNKNOWN_SETTLE_DELAYS_MS = [250, 500] as const; // APFS ejects and busy reporting can lag a detach attempt. const DETACH_SETTLE_DELAYS_MS = [250, 500, 1000] as const; const delay = (milliseconds: number): Promise => @@ -545,7 +546,13 @@ export class NativeDmgArtifactReader implements ArtifactReader { }, ); } - const delayMs = VERIFY_SETTLE_DELAYS_MS[attempt]; + // An explicit resource refusal can outlast the initial settle window + // after image creation. Its retries share the existing command deadline; + // an ambiguous localized refusal keeps only the short diagnostic probe. + const delayMs = + cause.verifyDiagnostic === "Resource temporarily unavailable" + ? VERIFY_TRANSIENT_DELAY_MS + : VERIFY_UNKNOWN_SETTLE_DELAYS_MS[attempt]; // Retry only the producer's explicit transient diagnostic, or a localized // refusal of a readable target that may be the same, before any mount. if ( diff --git a/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts b/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts index 2af75bb630..76fda7c2b4 100644 --- a/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts +++ b/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts @@ -1246,11 +1246,16 @@ describe("native DMG verification resource availability", () => { exitCode: 1, }; - it("retries the explicit transient refusal before attachment and retains both attempts", async () => { + it("settles an explicit transient refusal within the verification deadline before attaching", async () => { const { host, state } = createOmittedInventoryHost(); state.inventoryMode = "listed"; state.detachAttempts = 4; - const delays = vi.fn(async () => {}); + let now = 0; + const clock = vi.spyOn(performance, "now").mockImplementation(() => now); + onTestFinished(() => clock.mockRestore()); + const delays = vi.fn(async (milliseconds: number) => { + now += milliseconds; + }); const timeouts: number[] = []; let attempts = 0; const reader = await createAttachedReader( @@ -1266,36 +1271,45 @@ describe("native DMG verification resource availability", () => { if (options === undefined) throw new Error("missing verify deadline"); timeouts.push(options.timeoutMs); - if (attempts === 1) return transientFailure; + if (attempts < 5) return transientFailure; } return host.run(arguments_, signal, options); }, }, ); onTestFinished(() => reader.close()); - expect(attempts).toBe(2); - expect(delays.mock.calls).toEqual([[250]]); + expect(attempts).toBe(5); + expect(now).toBeGreaterThan(750); expect(timeouts[0]).toBeLessThanOrEqual(120_000); - expect(timeouts[1]).toBeLessThanOrEqual(timeouts[0]!); - expect(reader.provenance().slice(0, 2)).toMatchObject([ + expect(timeouts.at(-1)).toBeLessThan(timeouts[0]!); + expect( + reader + .provenance() + .slice(0, 5) + .map(({ exit_code }) => exit_code), + ).toEqual([1, 1, 1, 1, 0]); + expect(reader.provenance().slice(0, 1)).toMatchObject([ { arguments: ["verify", "/tmp/image.dmg"], exit_code: 1, effects: ["read"], }, - { - arguments: ["verify", "/tmp/image.dmg"], - exit_code: 0, - effects: ["read"], - }, ]); await reader.close(); expect(state.attached).toBe(false); }); it("bounds repeated transient refusals and never attaches an unverified image", async () => { - const run = vi.fn(async () => transientFailure); - const delay = vi.fn(async () => {}); + let now = 0; + const clock = vi.spyOn(performance, "now").mockImplementation(() => now); + onTestFinished(() => clock.mockRestore()); + const run = vi.fn(async () => { + now += 40_000; + return transientFailure; + }); + const delay = vi.fn(async (milliseconds: number) => { + now += milliseconds; + }); const reader = new NativeDmgArtifactReader("/tmp/image.dmg", process.env, { run, delay, @@ -1306,7 +1320,8 @@ describe("native DMG verification resource availability", () => { }); await reader.close(); expect(run.mock.calls).toHaveLength(3); - expect(delay.mock.calls).toEqual([[250], [500]]); + expect(now).toBeGreaterThanOrEqual(120_000); + expect(delay.mock.calls).toHaveLength(2); expect(reader.provenance().map(({ exit_code }) => exit_code)).toEqual([ 1, 1, 1, ]); From 6be6596aaca5a9f584fd726b9ca7733d5b54f55a Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:14:47 +0800 Subject: [PATCH 04/94] test(mcp): bound schema compilation for native and web workflows --- .../boundary/mcp/mcpSchemaValidation.test.ts | 17 ++++++---- tests/boundary/mcp/webRuntimeMcp.test.ts | 34 +++++++++++++------ tests/fixtures/mcpSchemaValidation.ts | 29 ++++++++++++++-- 3 files changed, 60 insertions(+), 20 deletions(-) diff --git a/tests/boundary/mcp/mcpSchemaValidation.test.ts b/tests/boundary/mcp/mcpSchemaValidation.test.ts index 2ce015a4f9..e82a633d77 100644 --- a/tests/boundary/mcp/mcpSchemaValidation.test.ts +++ b/tests/boundary/mcp/mcpSchemaValidation.test.ts @@ -7,7 +7,8 @@ it("reuses equal schemas and recompiles changed schemas sharing an id", () => { const schema = { $schema: "https://json-schema.org/draft/2020-12/schema", $id: "https://fixture.test/result", - type: "string" as "string" | "integer", + $defs: { value: { type: "string" as "string" | "integer" } }, + $ref: "#/$defs/value", }; const first = validation.getValidator(schema); expect(validation.getValidator(structuredClone(schema))).toBe(first); @@ -16,7 +17,7 @@ it("reuses equal schemas and recompiles changed schemas sharing an id", () => { expect(validation.assertions.validate(schema, "recorded")).toBe(true); expect(validation.assertions.validate(schema, 42)).toBe(false); - schema.type = "integer"; + schema.$defs.value.type = "integer"; const changed = validation.getValidator(schema); expect(changed).not.toBe(first); expect(changed(42).valid).toBe(true); @@ -31,13 +32,17 @@ it("reuses equal schemas and recompiles changed schemas sharing an id", () => { } }); -it("preserves SDK format validation and declared dialect dispatch", () => { +it.each([ + "https://json-schema.org/draft/2020-12/schema", + "https://json-schema.org/draft/2019-09/schema", + "http://json-schema.org/draft-07/schema#", +])("preserves SDK format validation and references for %s", (dialect) => { const validation = createMcpSchemaValidation(); try { const schema = { - $schema: "http://json-schema.org/draft-07/schema#", - type: "string", - format: "email", + $schema: dialect, + $defs: { address: { type: "string", format: "email" } }, + $ref: "#/$defs/address", } as const; const validator = validation.getValidator(schema); expect(validator("analyst@example.test").valid).toBe(true); diff --git a/tests/boundary/mcp/webRuntimeMcp.test.ts b/tests/boundary/mcp/webRuntimeMcp.test.ts index 68ea80bdd2..7f66b2eba9 100644 --- a/tests/boundary/mcp/webRuntimeMcp.test.ts +++ b/tests/boundary/mcp/webRuntimeMcp.test.ts @@ -1,6 +1,6 @@ import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { Ajv2020 } from "ajv/dist/2020.js"; -import { expect, it, onTestFinished } from "vitest"; +import { afterAll, expect, it, onTestFinished } from "vitest"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { WebRuntimeService } from "../../../src/application/WebRuntimeService.js"; import { recordingWebRuntimePort } from "../../../src/application/WebRuntimeService.fixture.js"; import { CdpWebRuntimeProvider } from "../../../src/browser/execution/CdpWebRuntimeProvider.js"; @@ -18,6 +18,9 @@ import { startRuntimeBrowser, } from "../../fixtures/webRuntime.js"; +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); + it("advertises actual SDK schemas, instrumentation effects and session-owned inline runtime evidence", async () => { const session = createTestBinarySession(() => { throw new Error("Binary provider must not start"); @@ -28,7 +31,10 @@ it("advertises actual SDK schemas, instrumentation effects and session-owned inl webRuntime: new WebRuntimeService(recordingWebRuntimePort()), }, ); - const client = new Client({ name: "web-runtime-contract", version: "1" }); + const client = new Client( + { name: "web-runtime-contract", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); onTestFinished(async () => { await client.close(); await server.close(); @@ -39,7 +45,7 @@ it("advertises actual SDK schemas, instrumentation effects and session-owned inl await server.connect(serverTransport); await client.connect(clientTransport); const tools = (await client.listTools()).tools; - const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const ajv = schemaValidation.assertions; const input = { cdp_endpoint: "http://127.0.0.1:9222", target_id: "allowed-page", @@ -118,7 +124,10 @@ cliTest( { kind: "session", session }, { webRuntime: new WebRuntimeService(provider) }, ); - const client = new Client({ name: "runtime-frame-identity", version: "1" }); + const client = new Client( + { name: "runtime-frame-identity", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); onTestFinished(async () => { await client.close(); await server.close(); @@ -193,10 +202,10 @@ cliTest( { kind: "session", session }, { webRuntime: new WebRuntimeService(provider) }, ); - const client = new Client({ - name: "runtime-negative-columns", - version: "1", - }); + const client = new Client( + { name: "runtime-negative-columns", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); onTestFinished(async () => { await client.close(); await server.close(); @@ -224,7 +233,7 @@ cliTest( }, }); expect(response.isError).not.toBe(true); - const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const ajv = schemaValidation.assertions; expect( ajv.validate(outputSchema, response.structuredContent), JSON.stringify(ajv.errors), @@ -293,7 +302,10 @@ it.each([ { kind: "session", session }, { webRuntime: new WebRuntimeService(new CdpWebRuntimeProvider()) }, ); - const client = new Client({ name: "web-runtime-partial", version: "1" }); + const client = new Client( + { name: "web-runtime-partial", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); onTestFinished(async () => { await client.close(); await server.close(); diff --git a/tests/fixtures/mcpSchemaValidation.ts b/tests/fixtures/mcpSchemaValidation.ts index 3e81a0a8d6..c082006275 100644 --- a/tests/fixtures/mcpSchemaValidation.ts +++ b/tests/fixtures/mcpSchemaValidation.ts @@ -2,7 +2,10 @@ import type { JsonSchemaType, JsonSchemaValidator, } from "@modelcontextprotocol/client"; -import { AjvJsonSchemaValidator } from "@modelcontextprotocol/client/validators/ajv"; +import { + addFormats, + AjvJsonSchemaValidator, +} from "@modelcontextprotocol/client/validators/ajv"; import { Ajv2020 } from "ajv/dist/2020.js"; import type { ValidateFunction } from "ajv"; @@ -19,9 +22,27 @@ export const createMcpSchemaValidation = () => { const key = JSON.stringify(schema); let validator = sdkValidators.get(key); if (validator === undefined) { - // Each full schema owns its default provider, including dialect dispatch. + // Keep shared definitions as compiled functions instead of expanding + // them at every reference. Other dialects retain SDK dispatch. + const provider = + schema.$schema === undefined || + schema.$schema === "https://json-schema.org/draft/2020-12/schema" + ? new AjvJsonSchemaValidator( + addFormats( + new Ajv2020({ + strict: false, + validateFormats: true, + validateSchema: false, + allErrors: true, + inlineRefs: false, + code: { optimize: false }, + }), + ), + ) + : new AjvJsonSchemaValidator(); + // Each full schema owns its provider, including dialect dispatch. // Distinct schemas may legally reuse the same $id in separate servers. - validator = new AjvJsonSchemaValidator().getValidator(JSON.parse(key)); + validator = provider.getValidator(JSON.parse(key)); sdkValidators.set(key, validator); } return validator as JsonSchemaValidator; @@ -44,6 +65,8 @@ export const createMcpSchemaValidation = () => { validator = new Ajv2020({ strict: false, validateFormats: false, + inlineRefs: false, + code: { optimize: false }, }).compile(JSON.parse(key)); assertionValidators.set(key, validator); } From c3647ade1e557c4179a8b825a37e0c2193cd2526 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 4 Nov 2026 02:01:44 +0800 Subject: [PATCH 05/94] fix(objdump)!: preserve unknown executable identity (#8824) * fix(objdump)!: preserve unknown executable identity * fix(objdump): avoid claiming rejected architecture output * fix(objdump): align launch errors with command identity * test(objdump): avoid coupling identity checks to wording * test(objdump): align launch failure expectations --- docs/reverse-engineering-tools.md | 10 ++- .../reverse/ReverseEngineeringService.ts | 10 ++- src/cli/reverseEngineeringCommands.ts | 13 ++-- .../reverseEngineeringToolContracts.ts | 2 +- src/domain/reverseEngineering.ts | 2 +- src/objdump/ObjdumpProviderIdentity.ts | 4 +- tests/boundary/cli/objdumpIdentity.test.ts | 71 +++++++++++++++++++ .../cli/reverseEngineeringCommands.test.ts | 2 +- tests/composition/reverseEngineering.test.ts | 2 +- .../reverseEngineeringAdmission.test.ts | 2 +- .../reverseProviderLaunchErrors.test.ts | 2 +- .../reverseEngineeringCompositionEdgeCases.ts | 8 +-- 12 files changed, 102 insertions(+), 26 deletions(-) create mode 100644 tests/boundary/cli/objdumpIdentity.test.ts diff --git a/docs/reverse-engineering-tools.md b/docs/reverse-engineering-tools.md index 645fef254f..10e7ab43cc 100644 --- a/docs/reverse-engineering-tools.md +++ b/docs/reverse-engineering-tools.md @@ -14,7 +14,13 @@ can run. ## GNU objdump -`inspect_with_objdump` and `rea inspect-with-objdump` capture stdout and stderr as inline Evidence. Artifact inspection Evidence records SHA-256 digests before and after the provider call and whether those observed digests differ; this cannot prove the exact bytes consumed if the artifact changes during execution. Choose one artifact operation: `file_headers`, `section_headers`, `symbols`, `relocations`, `dwarf`, `disassemble`, or `disassemble_all`. `disassemble` maps to `-d`; `disassemble_all` maps to `-D` and has different symbol-boundary semantics. The pathless `inspect_with_objdump {"operation":"architectures"}` MCP call or `rea list-objdump-architectures` CLI command maps to `-i` and produces subjectless provider-capability Evidence describing BFD targets compiled into the installed binary, not a guarantee that a target is supported. +REA sends GNU-style arguments to the selected command but does not verify its +implementation or version. Evidence identifies the command profile as +`objdump.command` and retains the selected executable; it does not claim that +the executable is GNU objdump. Other implementations may accept some commands +while differing in supported options and output. + +`inspect_with_objdump` and `rea inspect-with-objdump` capture stdout and stderr as inline Evidence. Artifact inspection Evidence records SHA-256 digests before and after the provider call and whether those observed digests differ; this cannot prove the exact bytes consumed if the artifact changes during execution. Choose one artifact operation: `file_headers`, `section_headers`, `symbols`, `relocations`, `dwarf`, `disassemble`, or `disassemble_all`. `disassemble` maps to `-d`; `disassemble_all` maps to `-D` and has different symbol-boundary semantics. The pathless `inspect_with_objdump {"operation":"architectures"}` MCP call or `rea list-objdump-architectures` CLI command maps to `-i` and produces subjectless Evidence containing the selected command’s output, not a guarantee that any target is supported. GNU objdump uses this option to list its compiled-in BFD targets. Failure to launch the selected executable reports provider unavailability, the configured command and the host error under `boundary: "provider-launch"`. @@ -25,7 +31,7 @@ their rejected fields in input issues before artifact reads or provider launch. For DWARF, REA does not follow local debug links by default. Set `follow_debug_links` to `true` to pass `-WK`; external debug-file identities are not bound to Evidence when following links is enabled. The CLI flag is `--follow-debug-links`; `--no-follow-debug-links` or `--follow-debug-links=false` keeps the default. A separate `true` or `false` word after the flag is not its value. REA clears `DEBUGINFOD_URLS` in the objdump child environment to disable debuginfod network access on GNU binutils versions that do not support `-WE`. The complete raw output is authoritative; REA does not normalize it into a version-independent disassembly schema. -For a known native symbol, use `operation: "disassemble", symbol: "my_function"` in MCP or `--symbol my_function` in CLI. REA passes `--disassemble=my_function` to objdump before collecting output, rather than producing the whole disassembly and filtering it afterward. Omit `symbol` to keep full code-section disassembly. Other operations reject this selector. Use a nonempty, NUL-free name with well-formed Unicode; unsupported values are rejected before reading the artifact or starting objdump. Symbol names are passed unchanged; matching, versioned symbols, aliases and the end of the disassembly follow the installed GNU objdump's semantics. Evidence records the selector and actual arguments. An empty match does not establish artifact-wide absence, and the selected observation does not contain the omitted disassembly; request the complete operation when needed. +For a known native symbol, use `operation: "disassemble", symbol: "my_function"` in MCP or `--symbol my_function` in CLI. REA passes `--disassemble=my_function` to objdump before collecting output, rather than producing the whole disassembly and filtering it afterward. Omit `symbol` to keep full code-section disassembly. Other operations reject this selector. Use a nonempty, NUL-free name with well-formed Unicode; unsupported values are rejected before reading the artifact or starting objdump. Symbol names are passed unchanged; matching, versioned symbols, aliases and the end of the disassembly follow the selected executable's semantics. Evidence records the selector and actual arguments. An empty match does not establish artifact-wide absence, and the selected observation does not contain the omitted disassembly; request the complete operation when needed. ```sh rea inspect-with-objdump ./program disassemble diff --git a/src/application/reverse/ReverseEngineeringService.ts b/src/application/reverse/ReverseEngineeringService.ts index e4117b31a8..02fe3b53c3 100644 --- a/src/application/reverse/ReverseEngineeringService.ts +++ b/src/application/reverse/ReverseEngineeringService.ts @@ -549,7 +549,7 @@ export class ReverseEngineeringService { ? "REA_OBJDUMP_COMMAND" : "REA_RIZIN_COMMAND"; earlyFailure = new AnalysisCapabilityUnavailableError( - provider, + profile.identity.id, operation, `Unable to run configured command (${String(code ?? "unknown error")})`, { @@ -708,10 +708,10 @@ export class ReverseEngineeringService { const limitations = [ ...(absolutePath === undefined ? [ - "The objdump version is not reported; the output lists BFD targets advertised by the installed tool and does not establish support for the selected artifact.", + "The objdump implementation and version are unknown. The captured output is the selected command's response and does not establish support for an artifact.", ] : [ - "The executable version and supported formats are reported as unknown because no reliable version probe was performed.", + "The executable implementation, version and supported formats are unknown because no reliable identity probe was performed.", ]), "A non-zero process exit is preserved as an observed command result.", ...(operation === "inspect_with_objdump" && @@ -759,9 +759,7 @@ export class ReverseEngineeringService { `Command completion is unknown because the operation failed with ${failure._tag}: ${failure.message}`, ]), ...(absolutePath === undefined - ? [ - "This output lists BFD targets advertised by the installed objdump and is independent of the selected artifact.", - ] + ? [] : [ "When both are available, pre-run and post-run digests detect artifact changes across the provider call but cannot prove the exact bytes consumed during the call.", ]), diff --git a/src/cli/reverseEngineeringCommands.ts b/src/cli/reverseEngineeringCommands.ts index 9998c2217b..59e715fb6d 100644 --- a/src/cli/reverseEngineeringCommands.ts +++ b/src/cli/reverseEngineeringCommands.ts @@ -1,5 +1,6 @@ import { z } from "incur"; import type { Logger } from "pino"; +import { OBJDUMP_PROVIDER_IDENTITY } from "../objdump/ObjdumpProviderIdentity.js"; import { createReverseEngineeringService } from "../composition/reverseEngineering.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; @@ -192,7 +193,7 @@ export const registerReverseEngineeringCommands = ( const service = createService(environment); cli.command(CLI_COMMANDS.inspectWithObjdump, { description: - "Inspect a local artifact with GNU objdump and retain its raw output as Evidence", + "Inspect a local artifact using GNU-style objdump arguments and retain the selected command output as Evidence", args: z.object({ path: z .string() @@ -200,7 +201,7 @@ export const registerReverseEngineeringCommands = ( .describe("Path to the local artifact to inspect."), operation: z .enum(OBJDUMP_ARTIFACT_OPERATIONS) - .describe("GNU objdump inspection operation to run."), + .describe("GNU-style objdump operation to run."), }), options: z.object({ ...analysisEvidenceDeliveryOptions, @@ -208,7 +209,7 @@ export const registerReverseEngineeringCommands = ( .string() .optional() .describe( - "Native symbol to select with the disassemble operation; GNU objdump controls matching and boundaries.", + "Native symbol to select with the disassemble operation; the selected executable controls matching and boundaries.", ), followDebugLinks: z .boolean() @@ -222,7 +223,7 @@ export const registerReverseEngineeringCommands = ( withCommandCancellation((signal) => logCliCommand(logger, CLI_COMMANDS.inspectWithObjdump, async () => { const delivery = { - providerId: "gnu.objdump", + providerId: OBJDUMP_PROVIDER_IDENTITY.id, operation: "inspect_with_objdump", signal, detail: options.detail, @@ -255,7 +256,7 @@ export const registerReverseEngineeringCommands = ( ), }); cli.command(CLI_COMMANDS.listObjdumpArchitectures, { - description: "List BFD targets advertised by the installed GNU objdump", + description: "Capture the selected objdump command response to -i", args: z.object({}), options: z.object({ ...analysisEvidenceDeliveryOptions, @@ -268,7 +269,7 @@ export const registerReverseEngineeringCommands = ( CLI_COMMANDS.listObjdumpArchitectures, async () => { const delivery = { - providerId: "gnu.objdump", + providerId: OBJDUMP_PROVIDER_IDENTITY.id, operation: "inspect_with_objdump", signal, detail: options.detail, diff --git a/src/contracts/reverseEngineeringToolContracts.ts b/src/contracts/reverseEngineeringToolContracts.ts index a0aa37f50e..d2aacf238b 100644 --- a/src/contracts/reverseEngineeringToolContracts.ts +++ b/src/contracts/reverseEngineeringToolContracts.ts @@ -17,7 +17,7 @@ export const REVERSE_ENGINEERING_TOOL_CONTRACTS = [ ...toolContractMetadata("inspect_with_objdump"), kind: "native-provider", description: - "Run caller-selected GNU objdump inspection against a local artifact, or request operation=architectures without a path to list BFD targets advertised by the installed binary. For operation=disassemble, optional symbol selects native symbol-scoped disassembly before output collection; omission preserves all code sections. Matching and symbol boundaries follow GNU objdump. Preserves decoded stdout and stderr inline with SHA-256 Evidence for artifact inspections, up to a shared 16 MiB combined limit; invalid UTF-8 bytes are represented by replacement characters and flagged in Evidence limitations. output_truncated reports output beyond that limit and completion_status is unknown if buffer enforcement terminated the process. The raw output is authoritative and REA does not claim a stable normalized format. DWARF does not follow local debug links by default. Set follow_debug_links to true to follow them; identities of external debug files are not bound to Evidence. REA clears DEBUGINFOD_URLS for objdump so DWARF processing does not make debuginfod network requests. Default detail complete returns original Evidence. Explicit detail summary retains the complete parent and returns its command observation once, without the raw mirror; parent_evidence_id supports later inspect_analysis_view summary queries and complete Evidence export. Architecture output is subjectless and does not establish support for any artifact. Optional command_timeout_ms is the per-phase wall clock in milliseconds (default 120000, range 1..2147483647); omitted means 120000 and the value applies on the next call. It does not raise the 16 MiB output cap.", + "Run a caller-selected objdump command using GNU-style arguments against a local artifact, or request operation=architectures without a path to capture its response to -i. The executable implementation and version are not verified; other implementations may differ in options and output. For operation=disassemble, optional symbol selects native symbol-scoped disassembly before output collection; omission preserves all code sections. Matching and symbol boundaries follow the selected executable. Preserves decoded stdout and stderr inline with SHA-256 Evidence for artifact inspections, up to a shared 16 MiB combined limit; invalid UTF-8 bytes are represented by replacement characters and flagged in Evidence limitations. output_truncated reports output beyond that limit and completion_status is unknown if buffer enforcement terminated the process. The raw output is authoritative and REA does not claim a stable normalized format. DWARF does not follow local debug links by default. Set follow_debug_links to true to follow them; identities of external debug files are not bound to Evidence. REA clears DEBUGINFOD_URLS for objdump so DWARF processing does not make debuginfod network requests. Default detail complete returns original Evidence. Explicit detail summary retains the complete parent and returns its command observation once, without the raw mirror; parent_evidence_id supports later inspect_analysis_view summary queries and complete Evidence export. Architecture output is subjectless and does not establish support for any artifact. Optional command_timeout_ms is the per-phase wall clock in milliseconds (default 120000, range 1..2147483647); omitted means 120000 and the value applies on the next call. It does not raise the 16 MiB output cap.", inputSchema: objdumpInputSchema, outputSchema: analysisDetailOutputSchemaOf( commandObservationSchema, diff --git a/src/domain/reverseEngineering.ts b/src/domain/reverseEngineering.ts index f79e0ccafd..03601ae2c9 100644 --- a/src/domain/reverseEngineering.ts +++ b/src/domain/reverseEngineering.ts @@ -53,7 +53,7 @@ const objdumpDisassemblyInputSchema = objdumpArtifactInputSchema.extend({ "Symbol must be well-formed Unicode.", ) .describe( - "Native symbol to disassemble, expressed as well-formed Unicode. Matching and symbol boundaries follow GNU objdump; omitted means all code sections.", + "Native symbol to disassemble, expressed as well-formed Unicode. Matching and symbol boundaries follow the selected executable; omitted means all code sections.", ) .optional(), }); diff --git a/src/objdump/ObjdumpProviderIdentity.ts b/src/objdump/ObjdumpProviderIdentity.ts index aa4d074b42..a039305b08 100644 --- a/src/objdump/ObjdumpProviderIdentity.ts +++ b/src/objdump/ObjdumpProviderIdentity.ts @@ -1,6 +1,6 @@ /** Source-owned static provider identity and public operation names. */ export const OBJDUMP_PROVIDER_IDENTITY = { - id: "gnu.objdump", - name: "GNU objdump", + id: "objdump.command", + name: "objdump command", version: null, } as const; diff --git a/tests/boundary/cli/objdumpIdentity.test.ts b/tests/boundary/cli/objdumpIdentity.test.ts new file mode 100644 index 0000000000..82a342c3de --- /dev/null +++ b/tests/boundary/cli/objdumpIdentity.test.ts @@ -0,0 +1,71 @@ +import { expect } from "vitest"; +import { randomUUID } from "node:crypto"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "does not attribute a caller-selected command to GNU objdump", + async ({ cli }) => { + const result = await cli.run({ + arguments: [ + "inspect-with-objdump", + process.execPath, + "file_headers", + "--format", + "json", + ], + environment: { REA_OBJDUMP_COMMAND: process.execPath }, + }); + + expect(result.exitCode).toBe(0); + expect(result.json).toMatchObject({ + provider: { + id: "objdump.command", + name: "objdump command", + version: null, + }, + parameters: { provider_executable: process.execPath }, + normalized_result: { + exit_code: expect.any(Number), + completion_status: "complete", + }, + }); + + const architectures = await cli.run({ + arguments: ["list-objdump-architectures", "--format", "json"], + environment: { REA_OBJDUMP_COMMAND: process.execPath }, + input: "", + }); + expect(architectures.exitCode).toBe(0); + expect(architectures.json).toMatchObject({ + provider: { + id: "objdump.command", + name: "objdump command", + version: null, + }, + subject: null, + }); + + const missing = join(tmpdir(), `missing-objdump-${randomUUID()}`); + const failed = await cli.run({ + arguments: [ + "inspect-with-objdump", + process.execPath, + "file_headers", + "--format", + "json", + ], + environment: { REA_OBJDUMP_COMMAND: missing }, + }); + expect(failed.exitCode).toBe(1); + expect(failed.json).toMatchObject({ + code: "capability_unavailable", + details: { + provider_id: "objdump.command", + diagnostics: { command: missing }, + }, + }); + }, +); diff --git a/tests/boundary/cli/reverseEngineeringCommands.test.ts b/tests/boundary/cli/reverseEngineeringCommands.test.ts index 854daed96f..a3482c8814 100644 --- a/tests/boundary/cli/reverseEngineeringCommands.test.ts +++ b/tests/boundary/cli/reverseEngineeringCommands.test.ts @@ -250,7 +250,7 @@ it("promotes only complete production cleanup-only Evidence after CLI cleanup re const result = await runObjdump(service, path); expect(result).toMatchObject({ - provider: { id: "gnu.objdump" }, + provider: { id: "objdump.command" }, normalized_result: { completion_status: "complete" }, }); expect(producedCleanupFailure?.cleanupOnly).toBe(true); diff --git a/tests/composition/reverseEngineering.test.ts b/tests/composition/reverseEngineering.test.ts index cc5987bcb6..4c353e2818 100644 --- a/tests/composition/reverseEngineering.test.ts +++ b/tests/composition/reverseEngineering.test.ts @@ -73,7 +73,7 @@ describe("reverse engineering command services", () => { exit_code: 0, output_truncated: false, }); - expect(result.value.provider.id).toBe("gnu.objdump"); + expect(result.value.provider.id).toBe("objdump.command"); expect(result.value.subject?.local_path).toBe(path); } finally { await rm(directory, { recursive: true, force: true }); diff --git a/tests/composition/reverseEngineeringAdmission.test.ts b/tests/composition/reverseEngineeringAdmission.test.ts index 6c7511f7c0..3ca64cf576 100644 --- a/tests/composition/reverseEngineeringAdmission.test.ts +++ b/tests/composition/reverseEngineeringAdmission.test.ts @@ -292,7 +292,7 @@ it.each([ expect(result.error.cause).toBe(code === "ABORT_ERR" ? reason : failure); if (code === "EACCES") { expect(result.error).toMatchObject({ - providerId: "objdump", + providerId: "objdump.command", diagnostics: { boundary: "provider-launch", command: "objdump", diff --git a/tests/composition/reverseProviderLaunchErrors.test.ts b/tests/composition/reverseProviderLaunchErrors.test.ts index 5cd951a8ba..8513f9b08e 100644 --- a/tests/composition/reverseProviderLaunchErrors.test.ts +++ b/tests/composition/reverseProviderLaunchErrors.test.ts @@ -39,7 +39,7 @@ it.each(["ENOENT", "EACCES", "EPERM", "ENOTDIR"] as const)( code: "capability_unavailable", retryable: false, details: { - provider_id: "objdump", + provider_id: "objdump.command", diagnostics: { boundary: "provider-launch", command, diff --git a/tests/support/reverseEngineeringCompositionEdgeCases.ts b/tests/support/reverseEngineeringCompositionEdgeCases.ts index 6886226a70..8863974da8 100644 --- a/tests/support/reverseEngineeringCompositionEdgeCases.ts +++ b/tests/support/reverseEngineeringCompositionEdgeCases.ts @@ -327,7 +327,7 @@ const registerSharedReverseEngineeringEdgeCasesPart3 = (): void => { ...(code === "ETIMEDOUT" ? { timeoutMs: 120_000 } : {}), ...(code === "EACCES" || code === "EPERM" ? { - providerId: "objdump", + providerId: "objdump.command", diagnostics: { boundary: "provider-launch", command: "objdump", @@ -337,7 +337,7 @@ const registerSharedReverseEngineeringEdgeCasesPart3 = (): void => { : {}), ...(code === "unknown" ? { - providerId: "objdump", + providerId: "objdump.command", reason: "Unable to run configured command (unknown error)", } : {}), @@ -425,7 +425,7 @@ const registerSharedReverseEngineeringEdgeCasesPart4 = (): void => { ok: false, error: { _tag: "AnalysisCapabilityUnavailableError", - providerId: "objdump", + providerId: "objdump.command", diagnostics: { boundary: "provider-launch", command: "objdump", @@ -535,7 +535,7 @@ const registerSharedReverseEngineeringEdgeCasesPart5 = (): void => { ok: false, error: { _tag: "AnalysisCapabilityUnavailableError", - providerId: "objdump", + providerId: "objdump.command", reason: "Unable to run configured command (ENOENT)", cause: { message: "launch failed" }, }, From e75d1c741ccf3e3ce6ff1e4533b60febde6543ec Mon Sep 17 00:00:00 2001 From: Rishikesh Gowda K K Date: Wed, 4 Nov 2026 09:16:11 +0530 Subject: [PATCH 06/94] feat(elf): decode Linux core dump process and memory mapping notes (#8830) * feat(elf): decode Linux core dump process and memory mapping notes Inspect Linux core dump note tables and decode structured metadata without requiring external tools or debuggers: - NT_PRPSINFO (type 3): decode process ID, parent ID, process group, session, UID, GID, execution state, filename, and command line arguments bounded by the text budget. - NT_FILE (type 0x46494c45): decode mapped memory ranges (start_address, end_address, file_offset_pages) and mapped file paths with page size. - NT_SIGINFO (type 0x53494749): decode faulting signal number, errno, and code. - NT_AUXV (type 6): decode auxiliary vector pairs with known AT_* tag names. - Support both 32-bit and 64-bit, little-endian and big-endian targets. - Retain unknown core note types gracefully without partial success errors. - Add comprehensive unit tests, fixtures, acceptance tests, and documentation. * fix(elf): preserve raw path bytes and enforce auxv termination * fix(elf): account for path hex text budget and enforce auxv trailing padding * fix(elf): account for row and scan limits, ABI signal order, and lossless source IDs * fix(elf): cache auxv padding scans, pre-charge entries, and isolate page-size test --- docs/elf-notes.md | 8 +- .../native/binaryDiagnosticsToolContracts.ts | 2 +- src/domain/native/elfNotes.ts | 66 +++ src/native/elf/ElfNoteParser.test.ts | 378 ++++++++++++++++++ src/native/elf/ElfNoteParser.ts | 359 ++++++++++++++++- src/native/elf/ElfNotes.fixture.ts | 161 +++++++- tests/acceptance/elfNotesCore.test.ts | 185 +++++++++ 7 files changed, 1149 insertions(+), 10 deletions(-) create mode 100644 tests/acceptance/elfNotesCore.test.ts diff --git a/docs/elf-notes.md b/docs/elf-notes.md index dff138c6c9..38d27c3787 100644 --- a/docs/elf-notes.md +++ b/docs/elf-notes.md @@ -38,6 +38,12 @@ were examined; an unavailable section name does not erase those note facts. For GNU notes, `gnu.build_id_hex`, `gnu.abi_tag`, or `gnu.gold_version` are filled when the descriptor matches that type. Other owners keep `gnu` null. +For Linux `CORE` notes, `core.prpsinfo`, `core.file`, `core.siginfo`, or +`core.auxv` are decoded for process arguments and status (`NT_PRPSINFO`), mapped +file memory regions and paths (`NT_FILE`), fault signal and code (`NT_SIGINFO`), and +auxiliary vectors (`NT_AUXV`) across 32-bit and 64-bit images. Other owners or unparsed +notes keep `core` null. + `PT_NOTE` and `SHT_NOTE` are examined independently. The same file range may appear twice when both tables declare it. @@ -48,7 +54,7 @@ the image never carried notes in another container or overlay. The selected file is limited to 64 MiB by default (512 MiB maximum). At most 32 note entries are examined by default across every table. Retained note -names, section names, and GNU descriptor text share a scan budget based on +names, section names, GNU descriptor text, and core dump string representations share a scan budget based on the source bytes plus any decoded section-name table, capped at `max_file_bytes`. The raw source and decoded table must also fit together within that ceiling. If admission fails, the error reports the required diff --git a/src/contracts/native/binaryDiagnosticsToolContracts.ts b/src/contracts/native/binaryDiagnosticsToolContracts.ts index 7bc97ecbc4..0099a0c9f9 100644 --- a/src/contracts/native/binaryDiagnosticsToolContracts.ts +++ b/src/contracts/native/binaryDiagnosticsToolContracts.ts @@ -111,7 +111,7 @@ export const BINARY_DIAGNOSTICS_TOOL_CONTRACTS = [ ...toolContractMetadata("inspect_elf_notes"), kind: "native-provider", description: - "Inspect one explicit local ELF32/ELF64 image's PT_NOTE program headers and SHT_NOTE sections without executing it or opening a disassembler. Returns note-table file ranges, owners, types, descriptor SHA-256 and hex previews, plus GNU build-id, ABI-tag and gold-version fields when those descriptors match. A missing table is observed absence of those declarations, not proof the image never carried notes in another container. Reads a stable regular file; malformed ELF headers, truncated notes and exhausted budgets return no partial success. Portable on supported Node hosts; no debug package or catalog is required.", + "Inspect one explicit local ELF32/ELF64 image's PT_NOTE program headers and SHT_NOTE sections without executing it or opening a disassembler. Returns note-table file ranges, owners, types, descriptor SHA-256 and hex previews, plus GNU build-id, ABI-tag and gold-version fields and Linux CORE process, mapping, signal, and auxv notes when those descriptors match. A missing table is observed absence of those declarations, not proof the image never carried notes in another container. Reads a stable regular file; malformed ELF headers, truncated notes and exhausted budgets return no partial success. Portable on supported Node hosts; no debug package or catalog is required.", inputSchema: inspectElfNotesInputSchema, outputSchema: evidenceResultOf(elfNotesSchema), examples: [ diff --git a/src/domain/native/elfNotes.ts b/src/domain/native/elfNotes.ts index b6d60974de..915b009332 100644 --- a/src/domain/native/elfNotes.ts +++ b/src/domain/native/elfNotes.ts @@ -17,6 +17,10 @@ export const ELF_NOTE_TYPE_NAMES = [ "gnu_build_id", "gnu_gold_version", "gnu_property", + "core_prpsinfo", + "core_file", + "core_siginfo", + "core_auxv", "unknown", ] as const; export type ElfNoteTypeName = (typeof ELF_NOTE_TYPE_NAMES)[number]; @@ -64,6 +68,22 @@ export const gnuNoteTypeName = (type: number): ElfNoteTypeName => { } }; +/** Map CORE n_type when the owner is CORE; other types stay unknown. */ +export const coreNoteTypeName = (type: number): ElfNoteTypeName => { + switch (type) { + case 3: + return "core_prpsinfo"; + case 6: + return "core_auxv"; + case 0x46494c45: + return "core_file"; + case 0x53494749: + return "core_siginfo"; + default: + return "unknown"; + } +}; + /** Complete bounded inspection of ELF PT_NOTE and SHT_NOTE tables. */ export const inspectElfNotesInputSchema = z.strictObject({ path: localPathStringSchema.describe( @@ -95,6 +115,51 @@ const gnuAbiTagSchema = z.strictObject({ subminor: uint, }); +const coreFileMappingSchema = z.strictObject({ + start_address: z.string().regex(/^0x[0-9a-f]+$/), + end_address: z.string().regex(/^0x[0-9a-f]+$/), + file_offset_pages: z.string().regex(/^0x[0-9a-f]+$/), + path: z.string(), + path_hex: z.string().regex(/^[0-9a-f]*$/), +}); + +const coreFileSchema = z.strictObject({ + count: uint, + page_size: uint, + mappings: z.array(coreFileMappingSchema), +}); + +const corePrpsinfoSchema = z.strictObject({ + state: uint, + pid: z.number().int(), + ppid: z.number().int(), + pgrp: z.number().int(), + sid: z.number().int(), + uid: uint, + gid: uint, + filename: z.string(), + psargs: z.string(), +}); + +const coreSiginfoSchema = z.strictObject({ + signo: z.number().int(), + code: z.number().int(), + errno: z.number().int(), +}); + +const coreAuxvEntrySchema = z.strictObject({ + type: uint, + type_name: z.string().nullable(), + value: z.string().regex(/^0x[0-9a-f]+$/), +}); + +export const coreNoteSchema = z.strictObject({ + prpsinfo: corePrpsinfoSchema.nullable(), + file: coreFileSchema.nullable(), + siginfo: coreSiginfoSchema.nullable(), + auxv: z.array(coreAuxvEntrySchema).nullable(), +}); + const noteEntrySchema = z.strictObject({ index: uint, table_kind: z.enum(["program", "section"]), @@ -125,6 +190,7 @@ const noteEntrySchema = z.strictObject({ gold_version: z.string().nullable(), }) .nullable(), + core: coreNoteSchema.nullable(), }); /** Observed ELF note tables; absence is an empty list, not proof notes never existed. */ diff --git a/src/native/elf/ElfNoteParser.test.ts b/src/native/elf/ElfNoteParser.test.ts index 8a82254c3e..957465f099 100644 --- a/src/native/elf/ElfNoteParser.test.ts +++ b/src/native/elf/ElfNoteParser.test.ts @@ -8,6 +8,10 @@ import { readElfDirectoryFromBuffer } from "../metadata/NativeSectionDirectory.j import { compressedElfNoteFixture, compressedElfNoteSectionFixture, + coreAuxvNote, + coreFileNote, + corePrpsinfoNote, + coreSiginfoNote, elfNote, elfNoteFixture, elfNoteTablePastEnd, @@ -702,3 +706,377 @@ it.each([ } }, ); + +it("decodes Linux core dump notes (NT_PRPSINFO, NT_FILE, NT_SIGINFO, NT_AUXV) for 64-bit little-endian", () => { + const notes = Buffer.concat([ + corePrpsinfoNote({ + bits: 64, + little: true, + state: 0, + sname: "R", + pid: 4242, + ppid: 1, + pgrp: 4242, + sid: 4242, + uid: 1001, + gid: 1002, + fname: "test-proc", + psargs: "/usr/bin/test-proc --daemon -v", + }), + coreFileNote( + [ + { + start: 0x400000n, + end: 0x405000n, + file_ofs: 0n, + path: "/usr/bin/test-proc", + }, + { + start: 0x7fff0000n, + end: 0x7fff2000n, + file_ofs: 10n, + path: "/lib/x86_64-linux-gnu/libc.so.6", + }, + ], + 4096, + 64, + true, + ), + coreSiginfoNote(11, 1, 0, true), // SIGSEGV, SEGV_MAPERR + coreAuxvNote( + [ + [6, 4096n], // AT_PAGESZ + [3, 0x400040n], // AT_PHDR + [9, 0x401000n], // AT_ENTRY + [999, 0x1234n], // Unknown auxv tag + ], + 64, + true, + ), + ]); + + const bytes = elfNoteFixture({ + bits: 64, + little: true, + type: 4, // ET_CORE + ownership: "program", + notes, + }); + + const report = parse(bytes); + expect(report.entries).toHaveLength(4); + + // 1. NT_PRPSINFO + expect(report.entries[0]).toMatchObject({ + owner: "CORE", + type: 3, + type_name: "core_prpsinfo", + gnu: null, + core: { + prpsinfo: { + state: 0, + pid: 4242, + ppid: 1, + pgrp: 4242, + sid: 4242, + uid: 1001, + gid: 1002, + filename: "test-proc", + psargs: "/usr/bin/test-proc --daemon -v", + }, + file: null, + siginfo: null, + auxv: null, + }, + }); + + // 2. NT_FILE + expect(report.entries[1]).toMatchObject({ + owner: "CORE", + type: 0x46494c45, + type_name: "core_file", + gnu: null, + core: { + prpsinfo: null, + file: { + count: 2, + page_size: 4096, + mappings: [ + { + start_address: "0x400000", + end_address: "0x405000", + file_offset_pages: "0x0", + path: "/usr/bin/test-proc", + path_hex: Buffer.from("/usr/bin/test-proc").toString("hex"), + }, + { + start_address: "0x7fff0000", + end_address: "0x7fff2000", + file_offset_pages: "0xa", + path: "/lib/x86_64-linux-gnu/libc.so.6", + path_hex: Buffer.from("/lib/x86_64-linux-gnu/libc.so.6").toString( + "hex", + ), + }, + ], + }, + siginfo: null, + auxv: null, + }, + }); + + // 3. NT_SIGINFO + expect(report.entries[2]).toMatchObject({ + owner: "CORE", + type: 0x53494749, + type_name: "core_siginfo", + gnu: null, + core: { + prpsinfo: null, + file: null, + siginfo: { + signo: 11, + errno: 0, + code: 1, + }, + auxv: null, + }, + }); + + // 4. NT_AUXV + expect(report.entries[3]).toMatchObject({ + owner: "CORE", + type: 6, + type_name: "core_auxv", + gnu: null, + core: { + prpsinfo: null, + file: null, + siginfo: null, + auxv: [ + { type: 6, type_name: "AT_PAGESZ", value: "0x1000" }, + { type: 3, type_name: "AT_PHDR", value: "0x400040" }, + { type: 9, type_name: "AT_ENTRY", value: "0x401000" }, + { type: 999, type_name: null, value: "0x1234" }, + ], + }, + }); +}); + +it("decodes Linux core dump notes for 32-bit big-endian", () => { + const notes = Buffer.concat([ + corePrpsinfoNote({ + bits: 32, + little: false, + state: 1, + pid: 123, + ppid: 1, + pgrp: 123, + sid: 123, + uid: 500, + gid: 500, + fname: "proc32", + psargs: "proc32 arg", + }), + coreFileNote( + [ + { + start: 0x10000, + end: 0x12000, + file_ofs: 0, + path: "/bin/proc32", + }, + ], + 4096, + 32, + false, + ), + coreAuxvNote( + [ + [6, 4096], + [11, 500], + ], + 32, + false, + ), + ]); + + const bytes = elfNoteFixture({ + bits: 32, + little: false, + type: 4, + ownership: "program", + notes, + }); + + const report = parse(bytes); + expect(report.entries).toHaveLength(3); + expect(report.entries[0]?.core?.prpsinfo).toMatchObject({ + state: 1, + pid: 123, + uid: 500, + gid: 500, + filename: "proc32", + psargs: "proc32 arg", + }); + expect(report.entries[1]?.core?.file).toMatchObject({ + count: 1, + page_size: 4096, + mappings: [ + { + start_address: "0x10000", + end_address: "0x12000", + file_offset_pages: "0x0", + path: "/bin/proc32", + path_hex: Buffer.from("/bin/proc32").toString("hex"), + }, + ], + }); + expect(report.entries[2]?.core?.auxv).toEqual([ + { type: 6, type_name: "AT_PAGESZ", value: "0x1000" }, + { type: 11, type_name: "AT_UID", value: "0x1f4" }, + ]); +}); + +it("tolerates malformed core notes by returning null without throwing", () => { + const truncatedPrpsinfo = elfNote("CORE", 3, Buffer.alloc(20)); // requires 136 for 64-bit + const truncatedFile = elfNote( + "CORE", + 0x46494c45, + Buffer.from("00000000000000010000000000001000", "hex"), // count 1, page_size 4096, but no map entries or filenames + ); + const truncatedSiginfo = elfNote("CORE", 0x53494749, Buffer.alloc(4)); // requires 12 bytes + const unterminatedAuxv = elfNote( + "CORE", + 6, + Buffer.from("00000000000000060000000000001000", "hex"), // AT_PAGESZ=4096 but lacks AT_NULL terminator + ); + const truncatedAuxv = elfNote("CORE", 6, Buffer.alloc(7)); // not multiple of entry size + const notes = Buffer.concat([ + truncatedPrpsinfo, + truncatedFile, + truncatedSiginfo, + unterminatedAuxv, + truncatedAuxv, + ]); + + const bytes = elfNoteFixture({ + bits: 64, + ownership: "program", + notes, + }); + + const report = parse(bytes); + expect(report.entries).toHaveLength(5); + expect(report.entries[0]?.core).toBeNull(); + expect(report.entries[0]?.type_name).toBe("core_prpsinfo"); + expect(report.entries[1]?.core).toBeNull(); + expect(report.entries[1]?.type_name).toBe("core_file"); + expect(report.entries[2]?.core).toBeNull(); + expect(report.entries[2]?.type_name).toBe("core_siginfo"); + expect(report.entries[3]?.core).toBeNull(); + expect(report.entries[3]?.type_name).toBe("core_auxv"); + expect(report.entries[4]?.core).toBeNull(); + expect(report.entries[4]?.type_name).toBe("core_auxv"); +}); + +it("decodes siginfo using target ABI for MIPS machines", () => { + const desc = Buffer.alloc(128); + desc.writeInt32LE(11, 0); // signo = SIGSEGV + desc.writeInt32LE(1, 4); // code = SEGV_MAPERR (offset 4 on MIPS) + desc.writeInt32LE(0, 8); // errno = 0 (offset 8 on MIPS) + const mipsSiginfo = elfNote("CORE", 0x53494749, desc, true); + const bytes = elfNoteFixture({ + bits: 32, + machine: 8, // EM_MIPS + ownership: "program", + notes: mipsSiginfo, + }); + const report = parse(bytes); + expect(report.entries[0]?.core?.siginfo).toEqual({ + signo: 11, + code: 1, + errno: 0, + }); +}); + +it("preserves negative process IDs and signal numbers without rewriting to zero", () => { + const prps = corePrpsinfoNote({ + pid: -1, + ppid: -1, + pgrp: -1, + sid: -1, + }); + const sig = coreSiginfoNote(-1, -2, -3); + const bytes = elfNoteFixture({ + bits: 64, + ownership: "program", + notes: Buffer.concat([prps, sig]), + }); + const report = parse(bytes); + expect(report.entries[0]?.core?.prpsinfo).toMatchObject({ + pid: -1, + ppid: -1, + pgrp: -1, + sid: -1, + }); + expect(report.entries[1]?.core?.siginfo).toMatchObject({ + signo: -1, + code: -2, + errno: -3, + }); +}); + +it("rejects 64-bit auxv tags and out-of-range page sizes with core: null without schema error", () => { + const validAuxv = coreAuxvNote([[6, 4096n]], 64, true); + const wideTagAuxv = Buffer.from(validAuxv); + wideTagAuxv.writeBigUInt64LE(0x100000000n, 20); + + const validFile = coreFileNote( + [ + { + start: 0x400000n, + end: 0x401000n, + file_ofs: 0n, + path: "/bin/app", + }, + ], + 4096, + 64, + true, + ); + const widePageSizeFile = Buffer.from(validFile); + widePageSizeFile.writeBigUInt64LE(0x100000000n, 28); + + const bytes = elfNoteFixture({ + bits: 64, + ownership: "program", + notes: Buffer.concat([wideTagAuxv, widePageSizeFile]), + }); + const report = parse(bytes); + expect(report.entries[0]?.core).toBeNull(); + expect(report.entries[1]?.core).toBeNull(); +}); + +it("does not spend text budget on malformed or discarded NT_FILE paths", () => { + const header = Buffer.alloc(16); + header.writeBigUInt64LE(2n, 0); // count = 2 + header.writeBigUInt64LE(4096n, 8); // page_size = 4096 + const mappings = Buffer.alloc(48); // 2 mappings * 24 bytes + const paths = Buffer.from( + "/bin/app\0unterminated_path_without_null", + "latin1", + ); + const badFileNote = elfNote( + "CORE", + 0x46494c45, + Buffer.concat([header, mappings, paths]), + ); + const bytes = elfNoteFixture({ + bits: 64, + ownership: "program", + notes: badFileNote, + }); + const report = parse(bytes); + expect(report.entries[0]?.core).toBeNull(); +}); diff --git a/src/native/elf/ElfNoteParser.ts b/src/native/elf/ElfNoteParser.ts index 4112f9726d..1d60e43add 100644 --- a/src/native/elf/ElfNoteParser.ts +++ b/src/native/elf/ElfNoteParser.ts @@ -19,6 +19,7 @@ import type { InspectElfNotesInput, } from "../../domain/native/elfNotes.js"; import { + coreNoteTypeName, elfObjectTypeName, gnuNoteTypeName, } from "../../domain/native/elfNotes.js"; @@ -289,6 +290,322 @@ const decodeGnu = ( }; }; +const AUXV_TYPE_NAMES: Readonly> = { + 0: "AT_NULL", + 1: "AT_IGNORE", + 2: "AT_EXECFD", + 3: "AT_PHDR", + 4: "AT_PHENT", + 5: "AT_PHNUM", + 6: "AT_PAGESZ", + 7: "AT_BASE", + 8: "AT_FLAGS", + 9: "AT_ENTRY", + 11: "AT_UID", + 12: "AT_EUID", + 13: "AT_GID", + 14: "AT_EGID", + 15: "AT_PLATFORM", + 16: "AT_HWCAP", + 17: "AT_CLKTCK", + 23: "AT_SECURE", + 25: "AT_RANDOM", + 26: "AT_HWCAP2", + 31: "AT_EXECFN", + 33: "AT_SYSINFO_EHDR", +}; + +const hexWord = (val: bigint | number): string => `0x${val.toString(16)}`; + +const decodeCorePrpsinfo = ( + descriptor: Buffer, + little: boolean, + wide: boolean, + text: TextBudget, +): ElfNotes["entries"][number]["core"] => { + const minLen = wide ? 136 : 124; + if (descriptor.length < minLen) return null; + const u8 = (off: number) => descriptor.readUInt8(off); + const u16 = (off: number) => + little ? descriptor.readUInt16LE(off) : descriptor.readUInt16BE(off); + const u32 = (off: number) => + little ? descriptor.readUInt32LE(off) : descriptor.readUInt32BE(off); + const i32 = (off: number) => + little ? descriptor.readInt32LE(off) : descriptor.readInt32BE(off); + + const state = u8(0); + const uid = wide ? u32(16) : u16(8); + const gid = wide ? u32(20) : u16(10); + const pid = wide ? i32(24) : i32(12); + const ppid = wide ? i32(28) : i32(16); + const pgrp = wide ? i32(32) : i32(20); + const sid = wide ? i32(36) : i32(24); + + const fnameOff = wide ? 40 : 28; + const fnameBytes = descriptor.subarray(fnameOff, fnameOff + 16); + const fnameEnd = fnameBytes.indexOf(0); + const fname = ( + fnameEnd < 0 ? fnameBytes : fnameBytes.subarray(0, fnameEnd) + ).toString("latin1"); + chargeText(text, fname.length); + + const psargsOff = wide ? 56 : 44; + const psargsBytes = descriptor.subarray(psargsOff, psargsOff + 80); + const psargsEnd = psargsBytes.indexOf(0); + const psargs = ( + psargsEnd < 0 ? psargsBytes : psargsBytes.subarray(0, psargsEnd) + ).toString("latin1"); + chargeText(text, psargs.length); + + return { + prpsinfo: { + state, + pid, + ppid, + pgrp, + sid, + uid, + gid, + filename: fname, + psargs, + }, + file: null, + siginfo: null, + auxv: null, + }; +}; + +const decodeCoreFile = ( + descriptor: Buffer, + little: boolean, + wide: boolean, + text: TextBudget, +): ElfNotes["entries"][number]["core"] => { + const wordSize = wide ? 8 : 4; + const headerSize = wordSize * 2; + if (descriptor.length < headerSize) return null; + + const readWord = (off: number): bigint => { + if (wide) { + return little + ? descriptor.readBigUInt64LE(off) + : descriptor.readBigUInt64BE(off); + } + const val = little + ? descriptor.readUInt32LE(off) + : descriptor.readUInt32BE(off); + return BigInt(val); + }; + + const countBig = readWord(0); + const pageSizeBig = readWord(wordSize); + if (countBig > 0xffffffffn || pageSizeBig > 0xffffffffn || pageSizeBig === 0n) + return null; + const count = Number(countBig); + const pageSize = Number(pageSizeBig); + + const entrySize = wordSize * 3; + const neededBytes = BigInt(headerSize) + countBig * BigInt(entrySize); + if (BigInt(descriptor.length) < neededBytes) return null; + const tableBytes = Number(neededBytes); + + const rowBytes = count * (wide ? 54 : 30); + if (rowBytes > text.limit - text.used) { + throw new ArtifactReaderFailure( + "limit", + `ELF note text exceeds the ${String(text.limit)}-byte budget.`, + { boundary: "memory" }, + ); + } + + const scanKey = `file:${String(descriptor.byteOffset)}:${String(descriptor.length)}`; + const cachedScan = text.scans.get(scanKey); + if (cachedScan === -1) return null; + if (cachedScan === undefined) { + chargeText(text, descriptor.length - tableBytes, "scanned"); + } + + let strPos = tableBytes; + let totalPathBytes = 0; + for (let i = 0; i < count; i++) { + if (strPos >= descriptor.length) { + text.scans.set(scanKey, -1); + return null; + } + const nullIdx = descriptor.indexOf(0, strPos); + if (nullIdx < 0) { + text.scans.set(scanKey, -1); + return null; + } + totalPathBytes += nullIdx - strPos; + strPos = nullIdx + 1; + } + text.scans.set(scanKey, count); + + const pathBytes = totalPathBytes * 3; + chargeText(text, rowBytes + pathBytes); + + const mappings: { + start_address: string; + end_address: string; + file_offset_pages: string; + path: string; + path_hex: string; + }[] = new Array(count); + + strPos = tableBytes; + for (let i = 0; i < count; i++) { + const entryOff = headerSize + i * entrySize; + const start = readWord(entryOff); + const end = readWord(entryOff + wordSize); + const offsetPages = readWord(entryOff + wordSize * 2); + const nullIdx = descriptor.indexOf(0, strPos); + const pathBuf = descriptor.subarray(strPos, nullIdx); + strPos = nullIdx + 1; + mappings[i] = { + start_address: hexWord(start), + end_address: hexWord(end), + file_offset_pages: hexWord(offsetPages), + path: pathBuf.toString("latin1"), + path_hex: pathBuf.toString("hex"), + }; + } + + return { + prpsinfo: null, + file: { + count, + page_size: pageSize, + mappings, + }, + siginfo: null, + auxv: null, + }; +}; + +const decodeCoreSiginfo = ( + descriptor: Buffer, + little: boolean, + machine: number, +): ElfNotes["entries"][number]["core"] => { + if (descriptor.length < 12) return null; + const i32 = (off: number) => + little ? descriptor.readInt32LE(off) : descriptor.readInt32BE(off); + const isMips = machine === 8 || machine === 10; + const signo = i32(0); + const errno = isMips ? i32(8) : i32(4); + const code = isMips ? i32(4) : i32(8); + return { + prpsinfo: null, + file: null, + siginfo: { + signo, + errno, + code, + }, + auxv: null, + }; +}; + +const decodeCoreAuxv = ( + descriptor: Buffer, + little: boolean, + wide: boolean, + text: TextBudget, +): ElfNotes["entries"][number]["core"] => { + const wordSize = wide ? 8 : 4; + const entrySize = wordSize * 2; + if (descriptor.length < entrySize || descriptor.length % entrySize !== 0) + return null; + const totalPairs = descriptor.length / entrySize; + + const scanKey = `auxv:${String(descriptor.byteOffset)}:${String(descriptor.length)}`; + let nullIndex = text.scans.get(scanKey); + if (nullIndex === -1) return null; + + const readWord = (off: number): bigint => { + if (wide) { + return little + ? descriptor.readBigUInt64LE(off) + : descriptor.readBigUInt64BE(off); + } + return BigInt( + little ? descriptor.readUInt32LE(off) : descriptor.readUInt32BE(off), + ); + }; + + if (nullIndex === undefined) { + chargeText(text, descriptor.length, "scanned"); + let found = false; + for (let i = 0; i < totalPairs; i++) { + const off = i * entrySize; + const aTypeBig = readWord(off); + if (aTypeBig > 0xffffffffn) { + text.scans.set(scanKey, -1); + return null; + } + if (aTypeBig === 0n) { + found = true; + nullIndex = i; + for (let j = off + entrySize; j < descriptor.length; j++) { + if (descriptor[j] !== 0) { + text.scans.set(scanKey, -1); + return null; + } + } + break; + } + } + if (!found || nullIndex === undefined) { + text.scans.set(scanKey, -1); + return null; + } + text.scans.set(scanKey, nullIndex); + } + + const rowBytes = nullIndex * (wide ? 38 : 30); + chargeText(text, rowBytes); + + const auxv: { type: number; type_name: string | null; value: string }[] = + new Array(nullIndex); + for (let i = 0; i < nullIndex; i++) { + const off = i * entrySize; + const aType = Number(readWord(off)); + const aVal = readWord(off + wordSize); + auxv[i] = { + type: aType, + type_name: AUXV_TYPE_NAMES[aType] ?? null, + value: hexWord(aVal), + }; + } + + return { + prpsinfo: null, + file: null, + siginfo: null, + auxv, + }; +}; + +const decodeCore = ( + owner: string, + type: number, + descriptor: Buffer, + little: boolean, + wide: boolean, + machine: number, + text: TextBudget, +): ElfNotes["entries"][number]["core"] => { + if (owner !== "CORE") return null; + if (type === 3) return decodeCorePrpsinfo(descriptor, little, wide, text); + if (type === 0x46494c45) + return decodeCoreFile(descriptor, little, wide, text); + if (type === 0x53494749) + return decodeCoreSiginfo(descriptor, little, machine); + if (type === 6) return decodeCoreAuxv(descriptor, little, wide, text); + return null; +}; + /** Decode PT_NOTE and SHT_NOTE tables from one in-memory ELF image. */ export const parseElfNotes = ( bytes: Buffer, @@ -439,7 +756,17 @@ export const parseElfNotes = ( examined, text, )) - parsed.push(toEntry(note, "program", index, reader.little, text)); + parsed.push( + toEntry( + note, + "program", + index, + reader.little, + wide, + header.machine, + text, + ), + ); } } if (shnum !== 0) { @@ -494,7 +821,17 @@ export const parseElfNotes = ( examined, text, )) - parsed.push(toEntry(note, "section", index, reader.little, text)); + parsed.push( + toEntry( + note, + "section", + index, + reader.little, + wide, + header.machine, + text, + ), + ); } } const absent = tables.length === 0; @@ -524,9 +861,12 @@ const toEntry = ( tableKind: "program" | "section", tableIndex: number, little: boolean, + wide: boolean, + machine: number, text: TextBudget, ): Omit => { const gnuOwner = note.owner === "GNU"; + const coreOwner = note.owner === "CORE"; const range = `${String(note.descriptor_offset)}:${String(note.descsz)}`; let digest = text.hashes.get(range); if (digest === undefined) { @@ -541,7 +881,11 @@ const toEntry = ( namesz: note.namesz, descsz: note.descsz, type: note.type, - type_name: gnuOwner ? gnuNoteTypeName(note.type) : "unknown", + type_name: gnuOwner + ? gnuNoteTypeName(note.type) + : coreOwner + ? coreNoteTypeName(note.type) + : "unknown", owner: note.owner, owner_bytes_hex: note.owner_bytes_hex, descriptor: { @@ -554,5 +898,14 @@ const toEntry = ( : null, }, gnu: decodeGnu(note.owner, note.type, note.descriptor, little, text), + core: decodeCore( + note.owner, + note.type, + note.descriptor, + little, + wide, + machine, + text, + ), }; }; diff --git a/src/native/elf/ElfNotes.fixture.ts b/src/native/elf/ElfNotes.fixture.ts index 6d31995d9b..63b2662b42 100644 --- a/src/native/elf/ElfNotes.fixture.ts +++ b/src/native/elf/ElfNotes.fixture.ts @@ -11,7 +11,7 @@ const notePadding = (declared: number): number => (declared === 8 ? 8 : 4); const writeWord = ( bytes: Buffer, offset: number, - value: number, + value: bigint | number, bits: 32 | 64, little: boolean, ): void => { @@ -20,8 +20,8 @@ const writeWord = ( else bytes.writeBigUInt64BE(BigInt(value), offset); return; } - if (little) bytes.writeUInt32LE(value, offset); - else bytes.writeUInt32BE(value, offset); + if (little) bytes.writeUInt32LE(Number(value), offset); + else bytes.writeUInt32BE(Number(value), offset); }; const write16 = ( @@ -40,8 +40,8 @@ const write32 = ( value: number, little: boolean, ): void => { - if (little) bytes.writeUInt32LE(value, offset); - else bytes.writeUInt32BE(value, offset); + if (little) bytes.writeUInt32LE(Number(value), offset); + else bytes.writeUInt32BE(Number(value), offset); }; /** Encode one ELF note, padding name/descriptor to the table alignment. */ @@ -402,3 +402,154 @@ export const elfNoteTablePastEnd = (): Buffer => { writeWord(program, 48, 4, 64, true); return concat(header, program, note); }; + +export interface CorePrpsinfoOptions { + readonly bits?: 32 | 64; + readonly little?: boolean; + readonly state?: number; + readonly sname?: string; + readonly zomb?: number; + readonly nice?: number; + readonly flag?: number; + readonly uid?: number; + readonly gid?: number; + readonly pid?: number; + readonly ppid?: number; + readonly pgrp?: number; + readonly sid?: number; + readonly fname?: string; + readonly psargs?: string; +} + +/** Linux CORE NT_PRPSINFO note (type 3). */ +export const corePrpsinfoNote = (options: CorePrpsinfoOptions = {}): Buffer => { + const bits = options.bits ?? 64; + const little = options.little ?? true; + const desc = Buffer.alloc(bits === 64 ? 136 : 124); + desc[0] = options.state ?? 0; + desc[1] = (options.sname ?? "R").charCodeAt(0); + desc[2] = options.zomb ?? 0; + desc[3] = options.nice ?? 0; + if (bits === 64) { + if (little) desc.writeBigUInt64LE(BigInt(options.flag ?? 0), 8); + else desc.writeBigUInt64BE(BigInt(options.flag ?? 0), 8); + write32(desc, 16, options.uid ?? 1000, little); + write32(desc, 20, options.gid ?? 1000, little); + if (little) { + desc.writeInt32LE(options.pid ?? 1234, 24); + desc.writeInt32LE(options.ppid ?? 1, 28); + desc.writeInt32LE(options.pgrp ?? 1234, 32); + desc.writeInt32LE(options.sid ?? 1234, 36); + } else { + desc.writeInt32BE(options.pid ?? 1234, 24); + desc.writeInt32BE(options.ppid ?? 1, 28); + desc.writeInt32BE(options.pgrp ?? 1234, 32); + desc.writeInt32BE(options.sid ?? 1234, 36); + } + const fname = Buffer.from(`${options.fname ?? "app"}\0`, "latin1"); + fname.copy(desc, 40, 0, Math.min(fname.length, 16)); + const psargs = Buffer.from( + `${options.psargs ?? "./app --flag"}\0`, + "latin1", + ); + psargs.copy(desc, 56, 0, Math.min(psargs.length, 80)); + } else { + write32(desc, 4, options.flag ?? 0, little); + write16(desc, 8, options.uid ?? 1000, little); + write16(desc, 10, options.gid ?? 1000, little); + if (little) { + desc.writeInt32LE(options.pid ?? 1234, 12); + desc.writeInt32LE(options.ppid ?? 1, 16); + desc.writeInt32LE(options.pgrp ?? 1234, 20); + desc.writeInt32LE(options.sid ?? 1234, 24); + } else { + desc.writeInt32BE(options.pid ?? 1234, 12); + desc.writeInt32BE(options.ppid ?? 1, 16); + desc.writeInt32BE(options.pgrp ?? 1234, 20); + desc.writeInt32BE(options.sid ?? 1234, 24); + } + const fname = Buffer.from(`${options.fname ?? "app"}\0`, "latin1"); + fname.copy(desc, 28, 0, Math.min(fname.length, 16)); + const psargs = Buffer.from( + `${options.psargs ?? "./app --flag"}\0`, + "latin1", + ); + psargs.copy(desc, 44, 0, Math.min(psargs.length, 80)); + } + return elfNote("CORE", 3, desc, little); +}; + +export interface CoreFileMapping { + readonly start: bigint | number; + readonly end: bigint | number; + readonly file_ofs: bigint | number; + readonly path: string; +} + +/** Linux CORE NT_FILE note (type 0x46494c45). */ +export const coreFileNote = ( + mappings: readonly CoreFileMapping[], + pageSize = 4096, + bits: 32 | 64 = 64, + little = true, +): Buffer => { + const wordBytes = bits === 64 ? 8 : 4; + const headerBytes = 2 * wordBytes; + const mapEntryBytes = 3 * wordBytes; + const filenames = Buffer.concat( + mappings.map((m) => Buffer.from(`${m.path}\0`, "latin1")), + ); + const desc = Buffer.alloc( + headerBytes + mappings.length * mapEntryBytes + filenames.length, + ); + writeWord(desc, 0, mappings.length, bits, little); + writeWord(desc, wordBytes, pageSize, bits, little); + let offset = headerBytes; + for (const m of mappings) { + writeWord(desc, offset, m.start, bits, little); + writeWord(desc, offset + wordBytes, m.end, bits, little); + writeWord(desc, offset + 2 * wordBytes, m.file_ofs, bits, little); + offset += mapEntryBytes; + } + filenames.copy(desc, offset); + return elfNote("CORE", 0x46494c45, desc, little); +}; + +/** Linux CORE NT_SIGINFO note (type 0x53494749). */ +export const coreSiginfoNote = ( + signo = 11, + code = 1, + errno = 0, + little = true, +): Buffer => { + const desc = Buffer.alloc(128); + if (little) { + desc.writeInt32LE(signo, 0); + desc.writeInt32LE(errno, 4); + desc.writeInt32LE(code, 8); + } else { + desc.writeInt32BE(signo, 0); + desc.writeInt32BE(errno, 4); + desc.writeInt32BE(code, 8); + } + return elfNote("CORE", 0x53494749, desc, little); +}; + +/** Linux CORE NT_AUXV note (type 6). */ +export const coreAuxvNote = ( + entries: readonly [key: number, value: bigint | number][], + bits: 32 | 64 = 64, + little = true, +): Buffer => { + const wordBytes = bits === 64 ? 8 : 4; + const desc = Buffer.alloc((entries.length + 1) * 2 * wordBytes); + let offset = 0; + for (const [key, value] of entries) { + writeWord(desc, offset, key, bits, little); + writeWord(desc, offset + wordBytes, value, bits, little); + offset += 2 * wordBytes; + } + writeWord(desc, offset, 0, bits, little); + writeWord(desc, offset + wordBytes, 0, bits, little); + return elfNote("CORE", 6, desc, little); +}; diff --git a/tests/acceptance/elfNotesCore.test.ts b/tests/acceptance/elfNotesCore.test.ts new file mode 100644 index 0000000000..7f9076cd25 --- /dev/null +++ b/tests/acceptance/elfNotesCore.test.ts @@ -0,0 +1,185 @@ +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; + +import { toolContract } from "../../src/contracts/toolContracts.js"; +import { + coreAuxvNote, + coreFileNote, + corePrpsinfoNote, + coreSiginfoNote, + elfNoteFixture, +} from "../../src/native/elf/ElfNotes.fixture.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +const descriptorHash = (bytes: Buffer): string => + createHash("sha256").update(bytes).digest("hex"); + +const buildCoreNotes = (): Buffer => + Buffer.concat([ + corePrpsinfoNote({ + bits: 64, + little: true, + state: 0, + sname: "R", + pid: 31337, + ppid: 1, + pgrp: 31337, + sid: 31337, + uid: 1000, + gid: 1000, + fname: "target-service", + psargs: "/opt/target-service --worker 1", + }), + coreFileNote( + [ + { + start: 0x555555554000n, + end: 0x555555558000n, + file_ofs: 0n, + path: "/opt/target-service", + }, + { + start: 0x7ffff7dc0000n, + end: 0x7ffff7de0000n, + file_ofs: 0n, + path: "/lib/x86_64-linux-gnu/libc.so.6", + }, + ], + 4096, + 64, + true, + ), + coreSiginfoNote(11, 1, 0, true), + coreAuxvNote( + [ + [6, 4096n], + [3, 0x555555554040n], + ], + 64, + true, + ), + ]); + +const assertCoreEntries = (entries: unknown[]): void => { + expect(entries).toHaveLength(4); + expect(entries[0]).toMatchObject({ + owner: "CORE", + type: 3, + type_name: "core_prpsinfo", + gnu: null, + core: { + prpsinfo: { + state: 0, + pid: 31337, + ppid: 1, + pgrp: 31337, + sid: 31337, + uid: 1000, + gid: 1000, + filename: "target-service", + psargs: "/opt/target-service --worker 1", + }, + }, + }); + expect(entries[1]).toMatchObject({ + owner: "CORE", + type: 0x46494c45, + type_name: "core_file", + gnu: null, + core: { + file: { + count: 2, + page_size: 4096, + mappings: [ + { + start_address: "0x555555554000", + end_address: "0x555555558000", + file_offset_pages: "0x0", + path: "/opt/target-service", + path_hex: Buffer.from("/opt/target-service").toString("hex"), + }, + { + start_address: "0x7ffff7dc0000", + end_address: "0x7ffff7de0000", + file_offset_pages: "0x0", + path: "/lib/x86_64-linux-gnu/libc.so.6", + path_hex: Buffer.from("/lib/x86_64-linux-gnu/libc.so.6").toString( + "hex", + ), + }, + ], + }, + }, + }); + expect(entries[2]).toMatchObject({ + owner: "CORE", + type: 0x53494749, + type_name: "core_siginfo", + gnu: null, + core: { + siginfo: { signo: 11, errno: 0, code: 1 }, + }, + }); + expect(entries[3]).toMatchObject({ + owner: "CORE", + type: 6, + type_name: "core_auxv", + gnu: null, + core: { + auxv: [ + { type: 6, type_name: "AT_PAGESZ", value: "0x1000" }, + { type: 3, type_name: "AT_PHDR", value: "0x555555554040" }, + ], + }, + }); +}; + +cliTest( + "decodes Linux core dump notes through CLI and stdio MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-elf-core-notes-"); + const path = join(root, "core.elf"); + const notes = buildCoreNotes(); + const bytes = elfNoteFixture({ + bits: 64, + little: true, + type: 4, + ownership: "program", + notes, + }); + await writeFile(path, bytes); + + const output = await cli.sharedCliRun!({ + arguments: ["inspect-elf-notes", path, "--json"], + }); + expect(output.exitCode, output.stderr).toBe(0); + + const call = await cli.sharedMcpCall!(); + const response = await call("inspect_elf_notes", { path }); + expect(response.isError, JSON.stringify(response.content)).not.toBe(true); + + const schema = toolContract("inspect_elf_notes").outputSchema; + for (const raw of [output.json, response.structuredContent]) { + const evidence = schema.parse(raw); + expect(evidence.subject).toMatchObject({ + local_path: path, + digest: { sha256: descriptorHash(bytes) }, + }); + const report = evidence.normalized_result; + expect(report).toMatchObject({ + format: "elf64", + byte_order: "little", + machine: 62, + coverage: { + status: "complete", + examined_tables: 1, + examined_entries: 4, + }, + }); + assertCoreEntries(report.entries); + } + }, +); From f8f9095d2d280c6d5ed8f62f28bd073b9f050f20 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:49:36 +0800 Subject: [PATCH 07/94] feat(kuna): verify release 1.990 (#8813) --- THIRD_PARTY_NOTICES.md | 2 +- docs/kuna-provider.md | 2 +- docs/testing-lanes.md | 5 +++++ src/kuna/KunaRelease.ts | 2 +- .../providers/kuna/kunaBatchDecompileMcp.test.ts | 2 +- tests/boundary/providers/kuna/kunaDoctor.test.ts | 10 +++++----- tests/boundary/providers/kuna/kunaInstallation.test.ts | 8 ++++---- tests/boundary/providers/kuna/kunaProvider.test.ts | 8 ++++---- 8 files changed, 22 insertions(+), 17 deletions(-) diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 61426ff708..98d4af57fe 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -60,7 +60,7 @@ pin inspection sources; they are not included in the npm package. | ghidra-nativeaot | [MIT](https://github.com/Washi1337/ghidra-nativeaot/blob/effeb734fc570c32650f88b159608979dc7b423e/LICENSE.md) | Pinned inspection source; derived parser attribution above. | | JADX | [Apache-2.0](https://github.com/skylot/jadx/blob/v1.5.6/LICENSE) | The audited jadx-headless-mcp source uses JADX 1.5.6 for Android decompilation; the engine JAR is separately supplied. | | jadx-headless-mcp | [Apache-2.0](https://github.com/1013503897/jadx-headless-mcp/blob/5844800a486d2248fa949b74c9896285f5b54de2/LICENSE) | v0.7.1; pinned submodule and user-supplied JAR. | -| Kuna | [Apache-2.0](https://github.com/Noelo-Lab/kuna/blob/v1.989/LICENSE), with its [NOTICE](https://github.com/Noelo-Lab/kuna/blob/v1.989/NOTICE) | Bring your own `kuna`; REA verified 1.989 and never downloads, installs or bundles it. REA invokes it as a separate process. | +| Kuna | [Apache-2.0](https://github.com/Noelo-Lab/kuna/blob/v1.990/LICENSE), with its [NOTICE](https://github.com/Noelo-Lab/kuna/blob/v1.990/NOTICE) | Bring your own `kuna`; REA verified 1.990 and never downloads, installs or bundles it. REA invokes it as a separate process. | | Apktool | [Apache-2.0](https://github.com/iBotPeaches/Apktool/blob/master/LICENSE.md) | User-supplied Android resource decoder. | | Binwalk | [MIT](https://github.com/ReFirmLabs/binwalk/blob/4fdab3d464d97b68e0af9088df3f9e2e1545b21c/LICENSE) | v3.1.0; pinned submodule and user-supplied firmware engine. | | Unblob | [MIT](https://github.com/onekey-sec/unblob/blob/1fcc7a0a584a70a96c31f5a276c20944d199a089/LICENSE) | 26.6.4; pinned submodule and user-supplied firmware engine. | diff --git a/docs/kuna-provider.md b/docs/kuna-provider.md index cf5199206c..81c27b2e6c 100644 --- a/docs/kuna-provider.md +++ b/docs/kuna-provider.md @@ -1,6 +1,6 @@ # Kuna provider -[Kuna](https://github.com/Noelo-Lab/kuna) is an Apache-2.0 decompiler distributed as one `kuna` executable. For a new installation, use Kuna 1.989, REA's recommended verified release. REA uses a `kuna` you supply; it does not download, install, update or bundle Kuna. REA accepts other releases; `rea doctor` and analysis Evidence report them as unverified. Verification compares the reported version; it does not authenticate the executable. Analysis profiles retain the observed version and executable digest, independently of REA's current verified-release policy. The 1.989 real-provider checks ran on macOS arm64; Linux and other hosts have not been verified with that release. +[Kuna](https://github.com/Noelo-Lab/kuna) is an Apache-2.0 decompiler distributed as one `kuna` executable. For a new installation, use Kuna 1.990, REA's recommended verified release. REA uses a `kuna` you supply; it does not download, install, update or bundle Kuna. REA accepts other releases; `rea doctor` and analysis Evidence report them as unverified. Verification compares the reported version; it does not authenticate the executable. Analysis profiles retain the observed version and executable digest, independently of REA's current verified-release policy. The 1.990 real-provider checks ran on macOS arm64; Linux and other hosts have not been verified with that release. Kuna is static: it reads the selected image and never executes it. diff --git a/docs/testing-lanes.md b/docs/testing-lanes.md index 223ed210d1..5663dca76b 100644 --- a/docs/testing-lanes.md +++ b/docs/testing-lanes.md @@ -459,6 +459,11 @@ symbol-bearing and stripped Mach-O fixtures, including cold and warm reads, references, and cleanup. Linux, other hosts, and the larger 1.833 target matrix were not verified with 1.989. +Kuna 1.990 passed the real-provider lane on macOS arm64 with the generated +symbol-bearing and stripped Mach-O fixtures, including cold and warm reads, +references, timeout recovery, and cleanup. Linux, other hosts, and the larger +1.833 target matrix were not verified with 1.990. + ## DOS Ghidra analysis `npm run verify:ghidra:dos` requires the supported Ghidra and JDK installation diff --git a/src/kuna/KunaRelease.ts b/src/kuna/KunaRelease.ts index 1d46a2aa30..11123df10a 100644 --- a/src/kuna/KunaRelease.ts +++ b/src/kuna/KunaRelease.ts @@ -1,5 +1,5 @@ /** Kuna release whose CLI and project layout REA verified; others report as unverified. */ -export const KUNA_VERIFIED_VERSION = "1.989"; +export const KUNA_VERIFIED_VERSION = "1.990"; /** Stable provider ID shared by readiness, doctor and Evidence. */ export const KUNA_PROVIDER_ID = "kuna"; diff --git a/tests/boundary/providers/kuna/kunaBatchDecompileMcp.test.ts b/tests/boundary/providers/kuna/kunaBatchDecompileMcp.test.ts index 59365f847e..9b752aa986 100644 --- a/tests/boundary/providers/kuna/kunaBatchDecompileMcp.test.ts +++ b/tests/boundary/providers/kuna/kunaBatchDecompileMcp.test.ts @@ -93,7 +93,7 @@ test("keeps each Kuna no-pseudocode observation's limitations on its batch item" // profile commits only analysis inputs. Composed and retained views must // preserve the same warning for agents reading those results. const verificationWarning = - "Kuna 1.833 is not REA's verified 1.989 release"; + "Kuna 1.833 is not REA's verified 1.990 release"; expect(batch.limitations).toContainEqual( expect.stringContaining(verificationWarning), ); diff --git a/tests/boundary/providers/kuna/kunaDoctor.test.ts b/tests/boundary/providers/kuna/kunaDoctor.test.ts index 6de81dfeba..014365a61f 100644 --- a/tests/boundary/providers/kuna/kunaDoctor.test.ts +++ b/tests/boundary/providers/kuna/kunaDoctor.test.ts @@ -34,7 +34,7 @@ it("reports an unconfigured Kuna as a missing optional engine", async () => { it("publishes the caller's explicit settings once the installation validates", async () => { const root = await createTestTempDirectory("rea-kuna-doctor-"); const kuna = await writeFakeKuna(root, { - version: { stdout: "kuna 1.989\n" }, + version: { stdout: "kuna 1.990\n" }, }); const cache = join(root, "cache"); @@ -48,7 +48,7 @@ it("publishes the caller's explicit settings once the installation validates", a expect(inspection).toMatchObject({ configured: true, available: true, - providerVersion: "1.989", + providerVersion: "1.990", registrationEnvironment: { REA_KUNA_COMMAND: kuna, REA_KUNA_CACHE_DIR: cache, @@ -59,10 +59,10 @@ it("publishes the caller's explicit settings once the installation validates", a name: "installation", ok: true, detail: expect.stringMatching( - /reports Kuna 1\.989 \(sha256 [a-f0-9]{64}\)\.$/u, + /reports Kuna 1\.990 \(sha256 [a-f0-9]{64}\)\.$/u, ), remediation: null, - details: { version: "1.989", verified_version: "1.989" }, + details: { version: "1.990", verified_version: "1.990" }, }, { name: "cache", @@ -88,7 +88,7 @@ it("flags an unverified release without failing the installation", async () => { "You may continue with the selected unverified release.", ), remediation: expect.stringContaining("setting REA_KUNA_COMMAND"), - details: { verified_version: "1.989" }, + details: { verified_version: "1.990" }, }); }); diff --git a/tests/boundary/providers/kuna/kunaInstallation.test.ts b/tests/boundary/providers/kuna/kunaInstallation.test.ts index 6082bf5063..801b758438 100644 --- a/tests/boundary/providers/kuna/kunaInstallation.test.ts +++ b/tests/boundary/providers/kuna/kunaInstallation.test.ts @@ -22,7 +22,7 @@ const it = test.skipIf(process.platform === "win32"); it("identifies a caller-selected kuna by version and digest", async () => { const root = await createTestTempDirectory("rea-kuna-install-"); const kuna = await writeFakeKuna(root, { - version: { stdout: "kuna 1.989\n" }, + version: { stdout: "kuna 1.990\n" }, }); const link = join(root, "current-kuna"); await symlink(kuna, link); @@ -37,7 +37,7 @@ it("identifies a caller-selected kuna by version and digest", async () => { value: { configuration: { command: link, cacheRoot: join(root, "cache") }, executable: await realpath(kuna), - version: "1.989", + version: "1.990", verified: true, // A `#!` launcher reports its interpreter; ownership checks must not expect the path. script: true, @@ -104,14 +104,14 @@ fs.renameSync(process.argv[1] + ".new", process.argv[1]); it("accepts another release but reports it as unverified", async () => { const root = await createTestTempDirectory("rea-kuna-unverified-"); const kuna = await writeFakeKuna(root, { - version: { stdout: "kuna 1.988\n" }, + version: { stdout: "kuna 1.989\n" }, }); const result = await inspectKunaInstallation({ REA_KUNA_COMMAND: kuna }); expect(result).toMatchObject({ ok: true, - value: { version: "1.988", verified: false }, + value: { version: "1.989", verified: false }, }); }); diff --git a/tests/boundary/providers/kuna/kunaProvider.test.ts b/tests/boundary/providers/kuna/kunaProvider.test.ts index 690b4fee9c..0d7fc6a0f2 100644 --- a/tests/boundary/providers/kuna/kunaProvider.test.ts +++ b/tests/boundary/providers/kuna/kunaProvider.test.ts @@ -182,13 +182,13 @@ it("waits for an aborted pre-session hash before retrying its late close owner", it("commits the observed Kuna release and digest to the analysis profile", async () => { const { profile, kuna } = await session("rea-kuna-profile-", { - version: { stdout: "kuna 1.989\n" }, + version: { stdout: "kuna 1.990\n" }, }); expect(profile).toMatchObject({ - provider: { id: "kuna", name: "Kuna", version: "1.989" }, + provider: { id: "kuna", name: "Kuna", version: "1.990" }, parameters: { - kuna_version: "1.989", + kuna_version: "1.990", kuna_sha256: expect.stringMatching(/^[0-9a-f]{64}$/u), kuna_command: kuna, jobs: 1, @@ -208,7 +208,7 @@ it("reports an unverified release on observations without binding policy into th }); const client = open(); const inventory = await run(client, "list_procedures"); - const warning = "Kuna 1.988 is not REA's verified 1.989 release"; + const warning = "Kuna 1.988 is not REA's verified 1.990 release"; expect(inventory.limitations.join("\n")).toContain(warning); const dossier = await run(client, "analyze_function", { procedure: "main" }); expect(dossier.limitations.join("\n")).toContain(warning); From 5a11bf09a5b8420584d097e0069194bb08c8a3d0 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:53:38 +0800 Subject: [PATCH 08/94] fix(mcp): surface byte search for encoded literals (#8817) --- src/contracts/officialToolContracts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/contracts/officialToolContracts.ts b/src/contracts/officialToolContracts.ts index 16ba5d0b66..9788b866a9 100644 --- a/src/contracts/officialToolContracts.ts +++ b/src/contracts/officialToolContracts.ts @@ -224,7 +224,7 @@ export const OFFICIAL_TOOL_CONTRACTS = [ ), official( "search_strings", - `Search every provider-reported string object or candidate using literal matching by default or regex when requested. Results are deterministic and complete over that provider inventory, not every text sequence in artifact bytes. Kuna scans mapped sequences of at least five characters, including unterminated fragments, without computing references. Ghidra may omit packed or undefined literals, including Rust strings; an empty result does not prove text is absent. If an address is known, xrefs can look for references; read_bytes can inspect bytes where supported (not available with Kuna). Hopper retains typed byte decoding, native display, and explicit decoding unknowns. Adjacent long-literal fragments match independently. ${hopperRegexDeadline} Select detail summary for counts and retained native-inventory pages; complete remains the default.`, + `Search every provider-reported string object or candidate using literal matching by default or regex when requested. Results are deterministic and complete over that provider inventory, not every text sequence in artifact bytes. Kuna scans mapped sequences of at least five characters, including unterminated fragments, without computing references. Ghidra may omit packed or undefined literals, including Rust strings; an empty result does not prove text is absent. For known encoded bytes, Ghidra search_bytes can find byte matches; a hit does not establish a string or use. If an address is known, xrefs can look for references; read_bytes can inspect bytes where supported (not available with Kuna). Hopper retains typed byte decoding, native display, and explicit decoding unknowns. Adjacent long-literal fragments match independently. ${hopperRegexDeadline} Select detail summary for counts and retained native-inventory pages; complete remains the default.`, z.object({ ...nativeSearchInput, detail: analysisDetailSchema }), { evidenceParameters: "omit-detail" }, ), From d7df9df3393f77c78a76a06144ff4f515d5db928 Mon Sep 17 00:00:00 2001 From: Pierre Date: Wed, 4 Nov 2026 04:58:44 +0100 Subject: [PATCH 09/94] fix(ghidra): mark empty xrefs results as unresolved, not unreferenced (#8831) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ghidra): mark empty xrefs results as unresolved, not unreferenced Ghidra 12.2 resolves no references to data addresses on some Mach-O imports, and xrefs returned a bare empty list with nothing separating 'no references exist' from 'this provider resolved no references', so callers concluded addresses were unreferenced. projectExecution now attaches ghidraEmptyXrefsLimitation to an empty xrefs result: the list reflects what this analysis resolved, not proof the address is unreferenced. Non-empty results carry no caveat. Covered by behavior tests for both branches; validated against FM9-Edit-arm64 (expect: limitation) once the daemon session reopens. * fix(ghidra): drop circular trace_feature cross-check from empty xrefs caveat trace_feature calls xrefs internally, so recommending it as an independent cross-check repeats the same lookup the caveat qualifies. Keep disassembly as the single independent check. * fix(ghidra): report actionable-reference absence and invalidate stale snapshots Review follow-up on the empty xrefs caveat: - The bridge filters synthetic entry-point references before observation, so "resolved no references" could be false; the caveat now reports that no actionable references were returned and names the omission. - Drop the trace_feature cross-check: traceReferences observes xrefs per matched string, repeating the same lookup instead of independent evidence. - Bump instruction_references semantics to v3 so analysis snapshots captured before the caveat replay no bare empty results; the profile commitment in the snapshot binding no longer matches. * feat(ghidra): mark empty function-reference facets as unresolved too Extends the empty-xrefs caveat to the sibling ReferenceManager consumers review follow-up identified: procedure_references and analyze_function reference facets built on the same observation path could still present empty arrays as collected facts. - ghidraEmptyFunctionReferencesLimitation attaches when procedure_references returns no references for the requested direction and when a dossier observes no references in either direction; callers/callees (function-manager derived, observed working on the reported target) stay caveat-free. - One emptyReferenceCaveat owner replaces the inline xrefs branch. - function_references semantics bump to v3 so snapshots captured before the caveat do not replay bare empty facets; raw profile digests recomputed. * fix(ghidra): cover annotation and instruction reference facets in the caveat Review follow-up: annotate_native_function returns a dossier built by the same analyzeFunction collection and inspect_native_instruction builds incoming_references with visitObservedReferencesTo; both could present empty arrays as collected facts under the reported ReferenceManager failure. - annotate_native_function attaches the caveat when its dossier observes no references in either direction. - inspect_native_instruction attaches it when incoming_references is an explicit empty array (null or omission stays unavailable, matching the field contract). - Split the diagnostics describe to keep each block under the lint bound. - Behavior tests cover attach and non-attach for both operations. * fix(ghidra): name the honest cross-check limits in the reference caveats Review follow-up: the motivating failure was a string address, so 'cross-check the disassembly at this address' pointed at nothing — data and string addresses have no instruction to disassemble — and inspect_native_instruction's incoming_references is built through the same visitObservedReferencesTo path, so it was not independent either. - xrefs caveat now states that no independent in-product cross-check exists for the empty result, that surrounding disassembly only helps at a code address, and that data or string addresses need an external disassembly scan (otool -tV adrp/add pairing). - Function-facet caveat separates outgoing references (own disassembly is independent) from incoming references (no in-product check; scan other code externally). * fix(ghidra): use the address-scoped caveat for inspect_native_instruction Greptile review follow-up: inspect_native_instruction checks incoming references to one exact address, so the function-scoped caveat misdescribed the scope; an instruction can be unreferenced while its function has references elsewhere, and the address may have no containing function. Attach the address-scoped xrefs caveat instead. * fix(ghidra): qualify each empty reference direction independently Analyze-function and annotate-native-function results attached the unresolved-references caveat only when BOTH reference directions were empty, so a dossier with one populated direction presented its empty direction as a collected fact. Split the combined function-references caveat into direction-scoped limitations and attach one per empty direction (analyze_function, annotate_native_function dossier readback, procedure_references by its requested direction). * fix(ghidra): qualify empty outgoing instruction references inspect_native_instruction results carry two independent ReferenceManager facets: the incoming list and the outgoing references built from instruction.getReferencesFrom(). Only the incoming list was checked, so an instruction with a populated incoming list and no outgoing references presented the empty outgoing facet as a collected fact, and any populated incoming list suppressed the caveat entirely. Each facet is now qualified independently: the empty incoming list keeps the address-scoped caveat; the empty outgoing list attaches the new instruction-scoped outgoing limitation (this stream is not entry-point filtered, so it claims no entry-point omission). Truth table covered in describe("Ghidra empty instruction reference facets"). * fix(ghidra): gate the instruction outgoing caveat on decoded status Non-decoded inspect_native_instruction results (data, not-in-listing, not-instruction-boundary, outside-memory) return before the bridge ever queries instruction.getReferencesFrom(), so their empty references list is an unobserved facet, not an empty ReferenceManager observation. The unconditional outgoing caveat claimed the ReferenceManager returned no references from an instruction that does not exist at that address. The outgoing limitation now attaches only when status is decoded; the address-scoped incoming caveat still applies per the empty incoming list, matching the original motivating data-address case. * fix(ghidra): dossier incoming omission and external outgoing recovery Two unqualified cases remained in the function-reference caveats: - The analyze_function/annotation dossier excludes incoming references whose source is inside the function's own body (a self-recursive call is enough), so the projected empty list can mean 'no outside source observed' or 'only in-body sources observed'. The dossier now uses a dedicated incoming limitation naming the exclusion, pointing at the outgoing edges/disassembly where in-body edges remain visible, and at procedure_references(incoming) - the same ReferenceManager stream without the in-body exclusion - as the in-product discriminator. procedure_references keeps the generic incoming caveat: its incoming stream is not in-body filtered. - An external function has no local body, so the outgoing caveat's 'function's own disassembly' recommendation was impossible. Records classified external now get an external-specific outgoing limitation naming the defining artifact and this program's call sites instead (procedure_references outgoing, analyze_function, annotation readback). * fix(ghidra): qualify dossier incoming caveat observation The dossier-specific incoming caveat claimed the ReferenceManager observed no incoming references from outside the body, but visitObservedReference (ReaGhidraBridge.java:3973-3974) drops entry-point references before the body filter (1619-1621), so the manager may have returned references the dossier never reports. Reframe as 'returned no actionable' and name both omissions; correct the recovery claim: procedure_references(incoming) lacks the in-body filter but omits synthetic entry-point references as well. --- src/ghidra/GhidraAnalysisProfile.ts | 4 +- src/ghidra/GhidraProvider.behavior.test.ts | 449 +++++++++++++++++++++ src/ghidra/GhidraProviderCapabilities.ts | 28 ++ src/ghidra/GhidraProviderClient.ts | 136 ++++++- src/ghidra/GhidraRawProfileDigest.test.ts | 4 +- src/ghidra/GhidraSnapshotCache.test.ts | 14 +- 6 files changed, 626 insertions(+), 9 deletions(-) diff --git a/src/ghidra/GhidraAnalysisProfile.ts b/src/ghidra/GhidraAnalysisProfile.ts index 3e3802f846..117df3ce52 100644 --- a/src/ghidra/GhidraAnalysisProfile.ts +++ b/src/ghidra/GhidraAnalysisProfile.ts @@ -57,7 +57,7 @@ const sharedAnalysisSemantics = { annotation_policy: "atomic-function-entry-metadata-v1", load_image_observations: "source-mappings-entry-context-relocation-symbol-v3", function_body_evidence: "complete-inclusive-ranges-v1", - function_references: "complete-body-and-entry-reference-manager-v2", + function_references: "complete-body-and-entry-reference-manager-v3", call_relationship_coverage: "function-body-call-coverage-v5", function_dossier_coverage: "observed-facets-and-call-relationships-v1", string_inventory_evidence: "defined-data-coverage-v1", @@ -67,7 +67,7 @@ const sharedAnalysisSemantics = { "address-first-unique-target-source-mapping-v2", instruction_flow_evidence: "decoded-return-pcode-v1", instruction_status: "unlisted-address-without-decode-v1", - instruction_references: "exact-address-incoming-without-synthetic-entry-v2", + instruction_references: "exact-address-incoming-without-synthetic-entry-v3", function_boundary_observations: "ghidra-function-boundary-limitations-v4", } as const; diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index e5728fe7f9..369d74d014 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -3,6 +3,12 @@ import { EXISTING_PROJECT_WINDOWS_REMEDIATION, EXISTING_PROJECT_WINDOWS_UNAVAILABLE, existingProjectCapabilities, + ghidraEmptyIncomingReferencesLimitation, + ghidraEmptyDossierIncomingLimitation, + ghidraEmptyExternalOutgoingLimitation, + ghidraEmptyInstructionOutgoingLimitation, + ghidraEmptyOutgoingReferencesLimitation, + ghidraEmptyXrefsLimitation, } from "./GhidraProviderCapabilities.js"; import { AnalysisProviderRegistry } from "../application/binary/AnalysisProviderRegistry.js"; import { composeBinarySession } from "../application/binary/BinarySessionComposition.js"; @@ -44,6 +50,16 @@ import { nativeAotPeVirtualSectionFixture, } from "../../tests/fixtures/nativeaotPe.js"; import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; +import { functionDossierSchema } from "../domain/native/functionDossier.js"; +import { + compactInstruction, + compactReference, +} from "../domain/native/nativeInstruction.fixture.js"; +import { + ghidraFunctionDossier, + ghidraFunctionIdentity, + ghidraReferenceEdge, +} from "../domain/ghidraValues.fixture.js"; import { parseConfig } from "../config/parseConfig.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; @@ -3122,6 +3138,439 @@ const sessionInfo = () => ({ }, }); +describe("Ghidra empty reference diagnostics", () => { + const emptyReferenceProvider = (result: JsonValue) => + provider(installationHost(), () => ({ + start: () => Promise.resolve(ok(sessionInfo())), + callTool: () => Promise.resolve(ok(result)), + close: () => Promise.resolve(ok(null)), + })); + + const observation = (value: JsonValue) => ({ value, limitations: [] }); + + it("names unresolved references in the limitations of an empty xrefs result", async () => { + const ghidra = emptyReferenceProvider([]); + const client = await createElfClient(ghidra); + try { + const execution = await client.execute("xrefs", { + address: "0x1005409a2", + }); + if (!execution.ok) throw execution.error; + expect(execution.value.result).toEqual([]); + expect(execution.value.limitations).toContain(ghidraEmptyXrefsLimitation); + } finally { + await client.close(); + } + }); + + it("does not add the unresolved-references caveat to a non-empty xrefs result", async () => { + const ghidra = emptyReferenceProvider(["0x100003670"]); + const client = await createElfClient(ghidra); + try { + const execution = await client.execute("xrefs", { + address: "0x1005409a2", + }); + if (!execution.ok) throw execution.error; + expect(execution.value.result).toEqual(["0x100003670"]); + expect(execution.value.limitations).not.toContain( + ghidraEmptyXrefsLimitation, + ); + } finally { + await client.close(); + } + }); + + it.each(["incoming", "outgoing"] as const)( + "qualifies the empty %s direction in procedure_references limitations", + async (direction) => { + const ghidra = emptyReferenceProvider( + observation({ + procedure: ghidraFunctionIdentity(), + direction, + references: [], + reference_kinds_available: true, + unresolved_calls: [], + }), + ); + const client = await createElfClient(ghidra); + try { + const execution = await client.execute("procedure_references", { + procedure: "0x401000", + direction, + }); + if (!execution.ok) throw execution.error; + expect(execution.value.result).toMatchObject({ references: [] }); + const expected = + direction === "outgoing" + ? ghidraEmptyOutgoingReferencesLimitation + : ghidraEmptyIncomingReferencesLimitation; + const otherDirection = + direction === "outgoing" + ? ghidraEmptyIncomingReferencesLimitation + : ghidraEmptyOutgoingReferencesLimitation; + expect(execution.value.limitations).toContain(expected); + expect(execution.value.limitations).not.toContain(otherDirection); + } finally { + await client.close(); + } + }, + ); + + it("does not add the caveat to procedure_references that returned references", async () => { + const ghidra = emptyReferenceProvider( + observation({ + procedure: ghidraFunctionIdentity(), + direction: "outgoing", + references: [ghidraReferenceEdge()], + reference_kinds_available: true, + unresolved_calls: [], + }), + ); + const client = await createElfClient(ghidra); + try { + const execution = await client.execute("procedure_references", { + procedure: "0x401000", + direction: "outgoing", + }); + if (!execution.ok) throw execution.error; + expect(execution.value.limitations).not.toContain( + ghidraEmptyIncomingReferencesLimitation, + ); + expect(execution.value.limitations).not.toContain( + ghidraEmptyOutgoingReferencesLimitation, + ); + } finally { + await client.close(); + } + }); + + it("names the external-specific outgoing caveat for an external procedure_references result", async () => { + const identity = ghidraFunctionIdentity(); + const ghidra = emptyReferenceProvider( + observation({ + procedure: { + ...identity, + classification: { ...identity.classification, external: true }, + }, + direction: "outgoing", + references: [], + reference_kinds_available: true, + unresolved_calls: [], + }), + ); + const client = await createElfClient(ghidra); + try { + const execution = await client.execute("procedure_references", { + procedure: "0x401000", + direction: "outgoing", + }); + if (!execution.ok) throw execution.error; + expect(execution.value.limitations).toContain( + ghidraEmptyExternalOutgoingLimitation, + ); + expect(execution.value.limitations).not.toContain( + ghidraEmptyOutgoingReferencesLimitation, + ); + } finally { + await client.close(); + } + }); +}); + +describe("Ghidra empty function-reference facets", () => { + const emptyReferenceProvider = (result: JsonValue) => + provider(installationHost(), () => ({ + start: () => Promise.resolve(ok(sessionInfo())), + callTool: () => Promise.resolve(ok(result)), + close: () => Promise.resolve(ok(null)), + })); + + const observation = (value: JsonValue) => ({ value, limitations: [] }); + + it("names each unresolved direction in the limitations of a dossier without any references", async () => { + const dossier = functionDossierSchema.parse(ghidraFunctionDossier()); + const ghidra = emptyReferenceProvider( + observation({ + ...dossier, + incoming_references: [], + outgoing_references: [], + }), + ); + const client = await createElfClient(ghidra); + try { + const execution = await client.execute("analyze_function", { + procedure: "0x401000", + }); + if (!execution.ok) throw execution.error; + expect(execution.value.limitations).toContain( + ghidraEmptyDossierIncomingLimitation, + ); + expect(execution.value.limitations).not.toContain( + ghidraEmptyIncomingReferencesLimitation, + ); + expect(execution.value.limitations).toContain( + ghidraEmptyOutgoingReferencesLimitation, + ); + } finally { + await client.close(); + } + }); + + it("qualifies the empty incoming direction when a dossier observed only outgoing references", async () => { + const dossier = functionDossierSchema.parse(ghidraFunctionDossier()); + const ghidra = emptyReferenceProvider( + observation({ ...dossier, incoming_references: [] }), + ); + const client = await createElfClient(ghidra); + try { + const execution = await client.execute("analyze_function", { + procedure: "0x401000", + }); + if (!execution.ok) throw execution.error; + expect(execution.value.limitations).toContain( + ghidraEmptyDossierIncomingLimitation, + ); + expect(execution.value.limitations).not.toContain( + ghidraEmptyIncomingReferencesLimitation, + ); + expect(execution.value.limitations).not.toContain( + ghidraEmptyOutgoingReferencesLimitation, + ); + } finally { + await client.close(); + } + }); + + it("qualifies the empty outgoing direction when a dossier observed only incoming references", async () => { + const dossier = functionDossierSchema.parse(ghidraFunctionDossier()); + const ghidra = emptyReferenceProvider( + observation({ + ...dossier, + incoming_references: [ghidraReferenceEdge()], + outgoing_references: [], + }), + ); + const client = await createElfClient(ghidra); + try { + const execution = await client.execute("analyze_function", { + procedure: "0x401000", + }); + if (!execution.ok) throw execution.error; + expect(execution.value.limitations).toContain( + ghidraEmptyOutgoingReferencesLimitation, + ); + expect(execution.value.limitations).not.toContain( + ghidraEmptyDossierIncomingLimitation, + ); + } finally { + await client.close(); + } + }); + + it("qualifies an external function's empty facets without local-disassembly guidance", async () => { + const dossier = functionDossierSchema.parse(ghidraFunctionDossier()); + const ghidra = emptyReferenceProvider( + observation({ + ...dossier, + procedure: { + ...dossier.procedure, + classification: { + ...dossier.procedure.classification, + external: true, + }, + }, + incoming_references: [], + outgoing_references: [], + }), + ); + const client = await createElfClient(ghidra); + try { + const execution = await client.execute("analyze_function", { + procedure: "0x401000", + }); + if (!execution.ok) throw execution.error; + expect(execution.value.limitations).toContain( + ghidraEmptyIncomingReferencesLimitation, + ); + expect(execution.value.limitations).toContain( + ghidraEmptyExternalOutgoingLimitation, + ); + expect(execution.value.limitations).not.toContain( + ghidraEmptyDossierIncomingLimitation, + ); + expect(execution.value.limitations).not.toContain( + ghidraEmptyOutgoingReferencesLimitation, + ); + } finally { + await client.close(); + } + }); +}); + +describe("Ghidra empty annotation reference facets", () => { + const emptyReferenceProvider = (result: JsonValue) => + provider(installationHost(), () => ({ + start: () => Promise.resolve(ok(sessionInfo())), + callTool: () => Promise.resolve(ok(result)), + close: () => Promise.resolve(ok(null)), + })); + + it("names each unresolved direction when annotation readback observes no references", async () => { + const dossier = functionDossierSchema.parse(ghidraFunctionDossier()); + const ghidra = emptyReferenceProvider({ + annotations: { + address: "0x401000", + name: "fixture_main", + comment: null, + inline_comment: null, + }, + dossier: { ...dossier, incoming_references: [], outgoing_references: [] }, + effects: { + scope: "session-analysis-database", + source_bytes_modified: false, + persists_after_close: false, + }, + }); + const client = await createElfClient(ghidra); + try { + const execution = await client.execute("annotate_native_function", { + procedure: "0x401000", + name: "fixture_main", + }); + if (!execution.ok) throw execution.error; + expect(execution.value.limitations).toContain( + ghidraEmptyDossierIncomingLimitation, + ); + expect(execution.value.limitations).toContain( + ghidraEmptyOutgoingReferencesLimitation, + ); + } finally { + await client.close(); + } + }); + + it("qualifies the empty direction when annotation readback observed one direction only", async () => { + const dossier = functionDossierSchema.parse(ghidraFunctionDossier()); + const ghidra = emptyReferenceProvider({ + annotations: { + address: "0x401000", + name: "fixture_main", + comment: null, + inline_comment: null, + }, + dossier: { + ...dossier, + incoming_references: [ghidraReferenceEdge()], + outgoing_references: [], + }, + effects: { + scope: "session-analysis-database", + source_bytes_modified: false, + persists_after_close: false, + }, + }); + const client = await createElfClient(ghidra); + try { + const execution = await client.execute("annotate_native_function", { + procedure: "0x401000", + name: "fixture_main", + }); + if (!execution.ok) throw execution.error; + expect(execution.value.limitations).toContain( + ghidraEmptyOutgoingReferencesLimitation, + ); + expect(execution.value.limitations).not.toContain( + ghidraEmptyIncomingReferencesLimitation, + ); + } finally { + await client.close(); + } + }); +}); + +describe("Ghidra empty instruction reference facets", () => { + const emptyReferenceProvider = (result: JsonValue) => + provider(installationHost(), () => ({ + start: () => Promise.resolve(ok(sessionInfo())), + callTool: () => Promise.resolve(ok(result)), + close: () => Promise.resolve(ok(null)), + })); + + const instruction = (incoming: JsonValue[], outgoing: JsonValue[]) => ({ + ...compactInstruction(outgoing), + incoming_references: incoming, + }); + const nonDecoded = (status: JsonValue, incoming: JsonValue[]) => ({ + ...compactInstruction(), + status, + bytes: null, + length: null, + mnemonic: null, + raw_disassembly: null, + incoming_references: incoming, + }); + const executeInspect = async ( + result: JsonValue, + ): Promise => { + const ghidra = emptyReferenceProvider(result); + const client = await createElfClient(ghidra); + try { + const execution = await client.execute("inspect_native_instruction", { + document: null, + address: "0x401000", + }); + if (!execution.ok) throw execution.error; + return execution.value.limitations; + } finally { + await client.close(); + } + }; + + it("names each unresolved facet when neither incoming nor outgoing references were observed", async () => { + const limitations = await executeInspect(instruction([], [])); + expect(limitations).toContain(ghidraEmptyXrefsLimitation); + expect(limitations).toContain(ghidraEmptyInstructionOutgoingLimitation); + }); + + it("qualifies the empty incoming list when only outgoing references were observed", async () => { + const limitations = await executeInspect( + instruction([], [compactReference()]), + ); + expect(limitations).toContain(ghidraEmptyXrefsLimitation); + expect(limitations).not.toContain(ghidraEmptyInstructionOutgoingLimitation); + }); + + it("qualifies the empty outgoing references when only an incoming list was observed", async () => { + const limitations = await executeInspect( + instruction([compactReference()], []), + ); + expect(limitations).toContain(ghidraEmptyInstructionOutgoingLimitation); + expect(limitations).not.toContain(ghidraEmptyXrefsLimitation); + }); + + it("adds no caveat when both instruction reference facets observed references", async () => { + const limitations = await executeInspect( + instruction([compactReference()], [compactReference()]), + ); + expect(limitations).not.toContain(ghidraEmptyXrefsLimitation); + expect(limitations).not.toContain(ghidraEmptyInstructionOutgoingLimitation); + }); + + it("keeps only the address caveat when a data address resolves no incoming references", async () => { + const limitations = await executeInspect(nonDecoded("data", [])); + expect(limitations).toContain(ghidraEmptyXrefsLimitation); + expect(limitations).not.toContain(ghidraEmptyInstructionOutgoingLimitation); + }); + + it("adds no caveat for a data address whose incoming references were observed", async () => { + const limitations = await executeInspect( + nonDecoded("data", [compactReference()]), + ); + expect(limitations).not.toContain(ghidraEmptyXrefsLimitation); + expect(limitations).not.toContain(ghidraEmptyInstructionOutgoingLimitation); + }); +}); + describe("Ghidra PDB coverage", () => { it.each([ { name: "missing bridge report", pdb: undefined, warns: true }, diff --git a/src/ghidra/GhidraProviderCapabilities.ts b/src/ghidra/GhidraProviderCapabilities.ts index 999abae9d6..cccfefe7bb 100644 --- a/src/ghidra/GhidraProviderCapabilities.ts +++ b/src/ghidra/GhidraProviderCapabilities.ts @@ -51,6 +51,34 @@ const stringInventoryLimitation = const searchLimitation = "Regex mode uses Java Pattern semantics. Matcher or compiler stack exhaustion returns a resource constraint without closing the session; literal mode avoids regex recursion and returns complete matching values."; +/** Attached to an empty xrefs result: absence of resolved references is not proof. */ +export const ghidraEmptyXrefsLimitation = + "Ghidra's ReferenceManager returned no actionable references to this address in this session. An empty list reflects what this analysis resolved, not proof that the address is unreferenced; synthetic entry-point references are omitted. No independent in-product cross-check exists for this empty result: surrounding disassembly can confirm direct references at a code address, and data or string addresses need an external disassembly scan (for example otool -tV, pairing adrp with add or ldr) before concluding there are no xrefs."; + +/** Attached to an empty incoming-reference facet of a function result. */ +export const ghidraEmptyIncomingReferencesLimitation = + "Ghidra's ReferenceManager returned no actionable incoming references for this function in this session. An empty incoming list reflects what this analysis resolved, not proof that nothing references this function; synthetic entry-point references are omitted. Incoming references have no independent in-product check and require scanning other code (for example an external otool -tV adrp/add pairing) before concluding there are none."; + +/** + * Attached to an empty incoming-reference facet of an analyzed-function + * dossier, which omits synthetic entry-point references and excludes sources + * inside the function's own body. + */ +export const ghidraEmptyDossierIncomingLimitation = + "Ghidra's ReferenceManager returned no actionable incoming references from outside this function's body in this session. An empty incoming list reflects what this analysis resolved, not proof that nothing references this function: synthetic entry-point references are omitted before the dossier's body filter, and sources inside the body are excluded (a self-recursive call is one), so the list can be empty even when such edges exist. In-body edges remain visible in the outgoing references and disassembly, and procedure_references(incoming) reads the same ReferenceManager stream without the in-body exclusion (it omits synthetic entry-point references as well) to separate the cases. Sources outside the program's observed references still require an external scan (for example otool -tV, pairing adrp with add or ldr) before concluding there are none."; + +/** Attached to an empty outgoing-reference facet of an external function. */ +export const ghidraEmptyExternalOutgoingLimitation = + "Ghidra's ReferenceManager returned no actionable outgoing references for this external function in this session. An external function's defining body is not part of this program database, so its outgoing list cannot describe what the function references: this program's call sites (its incoming references and callers) and the defining artifact are the checks before concluding anything about its behavior."; + +/** Attached to an empty outgoing-reference facet of a function result. */ +export const ghidraEmptyOutgoingReferencesLimitation = + "Ghidra's ReferenceManager returned no actionable outgoing references for this function in this session. An empty outgoing list reflects what this analysis resolved, not proof that the function references nothing; synthetic entry-point references are omitted. The function's own disassembly is an independent check for direct outgoing references before concluding there are none."; + +/** Attached to an empty outgoing-reference list of one instruction result. */ +export const ghidraEmptyInstructionOutgoingLimitation = + "Ghidra's ReferenceManager returned no actionable outgoing references from this instruction in this session. An empty list reflects what this analysis resolved, not proof that the instruction references nothing. The instruction's own disassembly is an independent check for direct outgoing references (a direct operand naming a target with no recorded reference contradicts the empty result) before concluding there are none."; + /** Limitation text for one admitted operation, including the common base. */ export const limitationsFor = ( operation: string, diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index 3040df85fc..77bb0f1adf 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -74,6 +74,7 @@ import { isGhidraFunctionOperation, parseGhidraFunctionInput, parseGhidraFunctionResult, + type GhidraFunctionOperation, } from "./GhidraFunctionValues.js"; import { isGhidraInventoryOperation, @@ -103,6 +104,12 @@ import type { NativeAotPeResult } from "../domain/native/nativeAotPe.js"; import { GhidraNativeAotSnapshotOwner } from "./GhidraNativeAotSnapshotOwner.js"; import { ghidraHealthLimitations, + ghidraEmptyXrefsLimitation, + ghidraEmptyIncomingReferencesLimitation, + ghidraEmptyDossierIncomingLimitation, + ghidraEmptyExternalOutgoingLimitation, + ghidraEmptyInstructionOutgoingLimitation, + ghidraEmptyOutgoingReferencesLimitation, windowsP0Limitations, limitationsFor, } from "./GhidraProviderCapabilities.js"; @@ -155,6 +162,111 @@ interface GhidraExecutionMetadata { readonly relationshipCoverage?: RelationshipCoverage; } +/** The ReferenceManager observation record of one function-reference result. */ +const referenceObservationRecord = ( + normalized: JsonValue, +): Readonly> | undefined => + typeof normalized === "object" && + normalized !== null && + !Array.isArray(normalized) + ? (normalized as Readonly>) + : undefined; + +const isEmptyArray = (value: JsonValue | undefined | null): boolean => + Array.isArray(value) && value.length === 0; + +/** + * Caveats for the empty reference directions observed in one record: each + * empty facet is qualified independently, because the provider's empty answer + * is unresolved, not proof of absence for that direction. External functions + * (no local body) and the in-body-filtered dossier incoming list get wording + * that matches what was actually observed. + */ +const isExternalProcedureRecord = ( + record: Readonly>, +): boolean => { + const procedure = referenceObservationRecord(record.procedure ?? null); + const classification = + procedure === undefined + ? undefined + : referenceObservationRecord(procedure.classification ?? null); + return classification?.external === true; +}; + +const incomingCaveat = (external: boolean): string => + external + ? ghidraEmptyIncomingReferencesLimitation + : ghidraEmptyDossierIncomingLimitation; + +const outgoingCaveat = (external: boolean): string => + external + ? ghidraEmptyExternalOutgoingLimitation + : ghidraEmptyOutgoingReferencesLimitation; + +const emptyReferenceDirectionCaveats = ( + record: Readonly>, +): readonly string[] => { + const external = isExternalProcedureRecord(record); + return [ + ...(isEmptyArray(record.incoming_references) + ? [incomingCaveat(external)] + : []), + ...(isEmptyArray(record.outgoing_references) + ? [outgoingCaveat(external)] + : []), + ]; +}; + +/** Caveat for a procedure_references result that resolved no references. */ +const emptyProcedureReferencesCaveat = ( + record: Readonly>, +): string | undefined => { + if (!isEmptyArray(record.references)) return undefined; + return record.direction === "outgoing" + ? outgoingCaveat(isExternalProcedureRecord(record)) + : ghidraEmptyIncomingReferencesLimitation; +}; + +/** + * Caveats for an empty ReferenceManager observation, in limitation order: + * the provider's empty answer is unresolved, not proof of absence. xrefs + * reports per address; the function-reference facets per empty direction. + */ +const emptyReferenceCaveat = ( + operation: GhidraFunctionOperation, + normalized: JsonValue, +): readonly string[] => { + if (operation === "xrefs") + return isEmptyArray(normalized) ? [ghidraEmptyXrefsLimitation] : []; + const record = referenceObservationRecord(normalized); + if (record === undefined) return []; + if (operation === "procedure_references") { + const caveat = emptyProcedureReferencesCaveat(record); + return caveat === undefined ? [] : [caveat]; + } + if (operation === "analyze_function") + return emptyReferenceDirectionCaveats(record); + if (operation === "annotate_native_function") { + const dossier = referenceObservationRecord(record.dossier ?? null); + return dossier === undefined ? [] : emptyReferenceDirectionCaveats(dossier); + } + if (operation === "inspect_native_instruction") { + // The incoming list and the outgoing references are independent + // ReferenceManager facets; each empty facet is qualified on its own. + // The outgoing list is only collected for a decoded instruction — other + // statuses return before the ReferenceManager is queried at all. + return [ + ...(isEmptyArray(record.incoming_references) + ? [ghidraEmptyXrefsLimitation] + : []), + ...(record.status === "decoded" && isEmptyArray(record.references) + ? [ghidraEmptyInstructionOutgoingLimitation] + : []), + ]; + } + return []; +}; + /** Build one AnalysisClient for an admitted Ghidra target and profile. */ export const createGhidraProviderClient = (input: { readonly config: AppConfig; @@ -846,6 +958,21 @@ export const createGhidraProviderClient = (input: { : parseGhidraInventoryResult(operation, called.value); if (!result.ok) return result; let { value: normalized, ...observation } = result.value; + let executionMetadata: GhidraExecutionMetadata = observation; + const unresolvedReferences = isGhidraFunctionOperation(operation) + ? emptyReferenceCaveat(operation, normalized) + : []; + if (unresolvedReferences.length > 0) { + // An empty list is the provider's observation, not proof: distinguish + // "no references were resolved" from "there are no references". + executionMetadata = { + ...executionMetadata, + limitations: [ + ...(executionMetadata.limitations ?? []), + ...unresolvedReferences, + ], + }; + } if ( (operation === "search_procedures" || operation === "search_strings") && input.value.mode === "literal" && @@ -856,7 +983,7 @@ export const createGhidraProviderClient = (input: { normalized, input.value.pattern, ); - observation = { ...observation, rawResult: called.value }; + executionMetadata = { ...executionMetadata, rawResult: called.value }; } if ( operation === "inspect_native_data_type" && @@ -881,7 +1008,10 @@ export const createGhidraProviderClient = (input: { const metadata = nativeAotOwner?.readTypeDetail(address); if (metadata !== undefined) { // Preserve the database reply independently of the derived overlay. - observation = { ...observation, rawResult: called.value }; + executionMetadata = { + ...executionMetadata, + rawResult: called.value, + }; normalized = jsonValueSchema.parse({ ...normalized, metadata_recovery: metadata, @@ -889,7 +1019,7 @@ export const createGhidraProviderClient = (input: { } } } - return ok(projectExecution(operation, normalized, observation)); + return ok(projectExecution(operation, normalized, executionMetadata)); }, executeIfReady: async (operation, parameters, options) => { if (operation !== "list_documents" && operation !== "list_procedures") diff --git a/src/ghidra/GhidraRawProfileDigest.test.ts b/src/ghidra/GhidraRawProfileDigest.test.ts index bf5aa9fd1d..1be64da13f 100644 --- a/src/ghidra/GhidraRawProfileDigest.test.ts +++ b/src/ghidra/GhidraRawProfileDigest.test.ts @@ -10,9 +10,9 @@ import { GhidraProvider } from "./GhidraProvider.js"; // status, load-image relocation-symbol and procedure search revisions that // prevent reuse of snapshots with earlier results. const RAW_DIGESTS = { - "6502": "7850ec8d885a9b5e1e6791637b57dfcea2d4359d55754ceeaf858fc43b22293d", + "6502": "4cdde67a88973466321e24d144620b9363bc6f3eed2526ccfdd52e81d204b251", "x86-disabled-analyzer": - "55fb2b19e10d32bea95a88a097029ab2ad705b4b81dc74844662191799dbb161", + "08bcaf0939ecefeba82c4b0b9cda6e7bdb975d3b8ad9b9e4aa9fdbd052958fd5", } as const; const configFor = (environment: NodeJS.ProcessEnv) => { diff --git a/src/ghidra/GhidraSnapshotCache.test.ts b/src/ghidra/GhidraSnapshotCache.test.ts index 2c3e207c9c..f439964ae2 100644 --- a/src/ghidra/GhidraSnapshotCache.test.ts +++ b/src/ghidra/GhidraSnapshotCache.test.ts @@ -319,14 +319,24 @@ describe("Ghidra snapshot profile admission", () => { }, { parameter: "instruction_references", - current: "exact-address-incoming-without-synthetic-entry-v2", + current: "exact-address-incoming-without-synthetic-entry-v3", + legacy: "exact-address-incoming-without-synthetic-entry-v2", + }, + { + parameter: "instruction_references", + current: "exact-address-incoming-without-synthetic-entry-v3", legacy: "exact-address-incoming-v1", }, { parameter: "instruction_references", - current: "exact-address-incoming-without-synthetic-entry-v2", + current: "exact-address-incoming-without-synthetic-entry-v3", legacy: undefined, }, + { + parameter: "function_references", + current: "complete-body-and-entry-reference-manager-v3", + legacy: "complete-body-and-entry-reference-manager-v2", + }, { parameter: "call_relationship_coverage", current: "function-body-call-coverage-v5", From 650a4593461ca90e71731425148d9325f3a27b19 Mon Sep 17 00:00:00 2001 From: ShuaixinHuang <1836656514@qq.com> Date: Wed, 4 Nov 2026 16:10:30 +0800 Subject: [PATCH 10/94] fix(cli): share boolean timing argument handling (#8857) * fix(cli): share boolean timing argument handling * fix(cli): preserve executable launcher mode --- scripts/rea.mjs | 11 +-- src/cliBuiltinArguments.ts | 45 ++++++++++-- tests/acceptance/setup/updateAliasCli.test.ts | 15 ++++ tests/boundary/cli/cliTimingArguments.test.ts | 68 +++++++++++++++++++ tests/boundary/cli/mcpDoctorHelpCli.test.ts | 28 ++++++++ 5 files changed, 151 insertions(+), 16 deletions(-) create mode 100644 tests/boundary/cli/cliTimingArguments.test.ts diff --git a/scripts/rea.mjs b/scripts/rea.mjs index 57e2b02485..a3d24b59e1 100755 --- a/scripts/rea.mjs +++ b/scripts/rea.mjs @@ -70,15 +70,8 @@ if (!(await compiledRuntimeExists(runtimeFiles))) { guardClosedStdout(); // Doctor runs outside Incur, so handle its global timing flag here. Keep a // token consumed by --format as a value, even when it looks like a flag. - const outputValues = new Set( - builtinArguments - .splitIncurBuiltinArguments(doctorArguments) - .outputValues.map(({ index }) => index), - ); - const doctorCliArguments = doctorArguments.filter( - (argument, index) => argument !== "--timings" || outputValues.has(index), - ); - const timingRequested = doctorCliArguments.length !== doctorArguments.length; + const { arguments: doctorCliArguments, enabled: timingRequested } = + builtinArguments.splitCliTimingArguments(doctorArguments); const started = performance.now(); try { const { runProductionMcpDoctorCli } = diff --git a/src/cliBuiltinArguments.ts b/src/cliBuiltinArguments.ts index 2dafd941ae..088aa73241 100644 --- a/src/cliBuiltinArguments.ts +++ b/src/cliBuiltinArguments.ts @@ -31,6 +31,38 @@ export const INCUR_BUILTIN_FLAGS: ReadonlySet = new Set([ "--timings", ]); +/** Supported boolean spelling under the pinned Incur global parser. */ +const cliTimingFlagValue = (token: string): boolean | undefined => + token === "--timings" + ? true + : token === "--no-timings" + ? false + : token.startsWith("--timings=") + ? token.slice("--timings=".length) === "true" + : undefined; + +/** Remove only timing globals, preserving valued output data and the -- boundary. */ +export const splitCliTimingArguments = ( + arguments_: readonly string[], +): { arguments: string[]; enabled: boolean } => { + const values = new Set( + splitIncurBuiltinArguments(arguments_).outputValues.map( + ({ index }) => index, + ), + ); + let enabled = false; + let ended = false; + const remaining = arguments_.filter((token, index) => { + if (values.has(index)) return true; + if (token === "--") ended = true; + const value = ended ? undefined : cliTimingFlagValue(token); + if (value === undefined) return true; + enabled = value; + return false; + }); + return { arguments: remaining, enabled }; +}; + /** Command tokens and output-flag values as Incur's builtin pass assigns them. */ export interface IncurBuiltinArgumentSplit { /** Tokens left for command resolution and command option parsing. */ @@ -79,12 +111,7 @@ export const splitIncurBuiltinArguments = ( let globalsEnded = false; const remaining = command.filter((token) => { if (token === "--") globalsEnded = true; - if ( - !globalsEnded && - (token === "--timings" || - token === "--no-timings" || - token.startsWith("--timings=")) - ) { + if (!globalsEnded && cliTimingFlagValue(token) !== undefined) { flags.add(token); return false; } @@ -119,7 +146,11 @@ export const normalizeCliArguments = ( normalized.push(flag, argument.slice(separator + 1)); else { if (argument === "--update") updates.add(normalized.length); - else if (!OUTPUT_SWITCHES.has(argument)) rootUpdate = false; + else if ( + !OUTPUT_SWITCHES.has(argument) && + cliTimingFlagValue(argument) === undefined + ) + rootUpdate = false; normalized.push(argument); } } diff --git a/tests/acceptance/setup/updateAliasCli.test.ts b/tests/acceptance/setup/updateAliasCli.test.ts index 9fd0536d38..7587c4638b 100644 --- a/tests/acceptance/setup/updateAliasCli.test.ts +++ b/tests/acceptance/setup/updateAliasCli.test.ts @@ -77,6 +77,21 @@ cliTest( const alias = await run(["--json", "--update"]); expect(alias.exitCode, alias.stderr).toBe(0); expect(alias.json).toEqual(canonical.json); + for (const flag of ["--no-timings", "--timings=false", "--timings=true"]) { + const canonicalTimed = await run(["update", "--json", flag]); + const aliasTimed = await run([flag, "--update", "--json"]); + expect(canonicalTimed.exitCode, canonicalTimed.stdout).toBe(0); + expect(aliasTimed.exitCode, aliasTimed.stdout).toBe(0); + expect(aliasTimed.json).toEqual(canonicalTimed.json); + expect(aliasTimed.json).toEqual(canonical.json); + for (const result of [canonicalTimed, aliasTimed]) { + if (flag === "--timings=true") + expect(JSON.parse(result.stderr)).toMatchObject({ + rea_timings: { version: 1, command_ms: expect.any(Number) }, + }); + else expect(result.stderr).toBe(""); + } + } const filtered = await run([ "--update", "--format=json", diff --git a/tests/boundary/cli/cliTimingArguments.test.ts b/tests/boundary/cli/cliTimingArguments.test.ts new file mode 100644 index 0000000000..867e3b5167 --- /dev/null +++ b/tests/boundary/cli/cliTimingArguments.test.ts @@ -0,0 +1,68 @@ +import { it, expect } from "vitest"; +import { Parser, z } from "incur"; +import { + splitCliTimingArguments, + normalizeCliArguments, + splitIncurBuiltinArguments, +} from "../../../src/cliBuiltinArguments.js"; + +it.each([ + [], + ["--timings"], + ["--no-timings"], + ["--timings=false"], + ["--timings=true"], + ["--timings=other"], + ["--timings="], + ["--timings", "--no-timings"], + ["--no-timings", "--timings=true"], + ["--timings=false", "--timings"], + ["--", "--timings"], +])("matches pinned boolean global parsing for %j", (...input: string[]) => { + const actual = splitCliTimingArguments(input); + const parsed = Parser.parseGlobals( + input, + z.object({ timings: z.boolean().default(false) }), + ); + expect(actual).toEqual({ + arguments: parsed.rest, + enabled: parsed.parsed.timings, + }); +}); +it.each([ + "--timings", + "--no-timings", + "--timings=false", + "--timings=true", + "--timings=other", + "--timings=", +])("preserves output values and root alias with %s", (flag) => { + expect(splitCliTimingArguments(["--format", flag])).toEqual({ + arguments: ["--format", flag], + enabled: false, + }); + expect(normalizeCliArguments([flag, "--update", "--json"])).toEqual([ + "update", + flag, + "--json", + ]); + expect(normalizeCliArguments(["--update", flag, "--json"])).toEqual([ + "update", + flag, + "--json", + ]); + expect(splitIncurBuiltinArguments([flag, "mcp", "doctor"]).command).toEqual([ + "mcp", + "doctor", + ]); +}); +it("retains the separator and its following timing-shaped data", () => { + expect(normalizeCliArguments(["--", "--update", "--timings=false"])).toEqual([ + "--", + "--update", + "--timings=false", + ]); + expect(splitCliTimingArguments(["--format", "--", "--timings=true"])).toEqual( + { arguments: ["--format", "--"], enabled: true }, + ); +}); diff --git a/tests/boundary/cli/mcpDoctorHelpCli.test.ts b/tests/boundary/cli/mcpDoctorHelpCli.test.ts index 1d5f0cdeac..abf2e40c31 100644 --- a/tests/boundary/cli/mcpDoctorHelpCli.test.ts +++ b/tests/boundary/cli/mcpDoctorHelpCli.test.ts @@ -153,3 +153,31 @@ describe("MCP doctor help boundary", () => { CLI_INTEGRATION_TIMEOUT_MS, ); }); + +workspaceCliTest( + "accepts the same timing spellings as the ordinary CLI", + async ({ cli }) => { + for (const [flag, enabled] of [ + ["--no-timings", false], + ["--timings=false", false], + ["--timings=true", true], + ["--timings=other", false], + ["--timings=", false], + ] as const) { + const result = await cli.run({ + arguments: ["mcp", "doctor", "--json", flag], + }); + expect(result.exitCode, result.stdout).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ + healthy: true, + adapter: "production-stdio", + }); + if (enabled) + expect(JSON.parse(result.stderr)).toMatchObject({ + rea_timings: { version: 1, command_ms: expect.any(Number) }, + }); + else expect(result.stderr).toBe(""); + } + }, + CLI_INTEGRATION_TIMEOUT_MS, +); From 1998e65705afd04dee078724932fa98d446248b9 Mon Sep 17 00:00:00 2001 From: ShuaixinHuang <1836656514@qq.com> Date: Wed, 4 Nov 2026 16:10:46 +0800 Subject: [PATCH 11/94] fix(ghidra): discover installed versioned Homebrew JDKs (#8853) * fix(ghidra): discover installed versioned Homebrew JDKs * test(ghidra): protect Homebrew Java candidate precedence * fix(ghidra): continue automatic JDK discovery after a failed keg --- docs/installation.md | 6 +- src/ghidra/GhidraInstallation.test.ts | 207 ++++++++++++++++++++++++++ src/ghidra/GhidraInstallation.ts | 56 +++++-- 3 files changed, 256 insertions(+), 13 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index a9ff7d110a..c788ae6168 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1160,7 +1160,11 @@ selected formula's `libexec/openjdk.jdk/Contents/Home`, for example Java version bounds declared by your Ghidra installation; REA does not select another JDK when `JAVA_HOME` is explicit or the PATH JVM runs. On macOS, when the PATH launcher cannot start a JVM, it tries executable Homebrew -`opt/openjdk` candidates, preferring the host architecture's prefix. Failed +`opt/openjdk` and installed `opt/openjdk@` candidates, preferring the host +architecture's prefix and the unversioned formula within each prefix. Versioned +aliases are considered in ascending major order within Ghidra's declared Java +bounds; the selected executable still undergoes the normal Java probe and +admission checks. REA does not invoke Homebrew or install a JDK. Failed discovery probes remain available in installation diagnostics. Doctor distinguishes an absent Java command from a command that runs and fails, and retains bounded probe output and any parsed properties when the selected runtime cannot be admitted. diff --git a/src/ghidra/GhidraInstallation.test.ts b/src/ghidra/GhidraInstallation.test.ts index f830572f61..3e0daeffc0 100644 --- a/src/ghidra/GhidraInstallation.test.ts +++ b/src/ghidra/GhidraInstallation.test.ts @@ -27,6 +27,7 @@ import { ghidraJavaEnvironment, discoverGhidraInstallationOnPath, inspectGhidraInstallation, + systemGhidraInstallationHost, type GhidraInstallationHost, } from "./GhidraInstallation.js"; import { @@ -1732,6 +1733,212 @@ describe("Ghidra Java probe os.arch extraction", () => { }); }); +it("enumerates installed opt aliases without requiring a Homebrew executable", async () => { + const root = await createTestTempDirectory("rea-ghidra-opt-aliases-"); + try { + await mkdir(join(root, "openjdk@21")); + await mkdir(join(root, "unrelated")); + const systemHost = systemGhidraInstallationHost(); + expect(systemHost.directoryEntries?.(root).toSorted()).toEqual([ + "openjdk@21", + "unrelated", + ]); + expect(systemHost.directoryEntries?.(join(root, "absent"))).toEqual([]); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +it.each(["arm64", "x64"] as const)( + "discovers installed versioned Homebrew Java on macOS %s", + async (architecture) => { + const prefix = architecture === "arm64" ? "/opt/homebrew" : "/usr/local"; + const command = `${prefix}/opt/openjdk@21/libexec/openjdk.jdk/Contents/Home/bin/java`; + const commands: string[] = []; + const inspection = await inspectGhidraInstallation( + { + resourceOwner: inspectionResources.owner, + environment: { PATH: "/usr/bin" }, + installDir: INSTALL, + platform: "darwin", + architecture, + }, + host({ + directoryEntries: (directory) => + directory === `${prefix}/opt` + ? ["openjdk@21", "unrelated", "openjdk@21/../foreign"] + : [], + executable: (candidate) => + candidate === "/usr/bin/java" || + candidate === command || + candidate.endsWith("/mac_x86_64/decompile") || + host().executable(candidate), + probeJava: (candidate) => { + commands.push(candidate); + return candidate === command + ? javaProbeOk({ + ...JAVA, + home: command.slice(0, -9), + osName: "Mac OS X", + osArch: architecture === "arm64" ? "aarch64" : "x86_64", + }) + : missingJavaProbe; + }, + }), + ); + expect(commands).toEqual(["/usr/bin/java", command]); + expect(inspection.status).toBe("available"); + expect(inspection.javaCommand).toBe(command); + }, +); + +it.each([ + ["arm64", true, "/opt/homebrew", "/usr/local"], + ["x64", true, "/usr/local", "/opt/homebrew"], + ["arm64", false, "/opt/homebrew", "/usr/local"], + ["x64", false, "/usr/local", "/opt/homebrew"], +] as const)( + "orders installed Homebrew Java candidates on macOS %s with PATH Java %s", + async (architecture, pathJava, preferred, other) => { + const suffix = "/libexec/openjdk.jdk/Contents/Home/bin/java"; + const candidates = [ + `${preferred}/opt/openjdk${suffix}`, + `${preferred}/opt/openjdk@21${suffix}`, + `${preferred}/opt/openjdk@25${suffix}`, + `${other}/opt/openjdk${suffix}`, + `${other}/opt/openjdk@21${suffix}`, + `${other}/opt/openjdk@25${suffix}`, + ]; + for (const selected of [0, 1, 2, 3, undefined]) { + const commands: string[] = []; + const inspection = await inspectGhidraInstallation( + { + resourceOwner: inspectionResources.owner, + environment: { PATH: pathJava ? "/usr/bin" : "/empty-bin" }, + installDir: INSTALL, + platform: "darwin", + architecture, + }, + host({ + readText: () => + "application.version=12.1.4\napplication.java.min=21\napplication.java.max=25\n", + directoryEntries: () => ["openjdk@25", "openjdk@21"], + executable: (candidate) => + (pathJava && candidate === "/usr/bin/java") || + candidates.includes(candidate) || + candidate.endsWith("/mac_x86_64/decompile") || + host().executable(candidate), + probeJava: (command) => { + commands.push(command); + return selected !== undefined && command === candidates[selected] + ? javaProbeOk({ + ...JAVA, + home: command.slice(0, -9), + major: selected === 2 ? 25 : 21, + version: selected === 2 ? "25.0.1" : JAVA.version, + osName: "Mac OS X", + osArch: architecture === "arm64" ? "aarch64" : "x86_64", + }) + : missingJavaProbe; + }, + }), + ); + expect(commands).toEqual([ + ...(pathJava ? ["/usr/bin/java"] : []), + ...candidates.slice( + 0, + selected === undefined ? undefined : selected + 1, + ), + ]); + expect(inspection.status).toBe( + selected === undefined ? "unavailable" : "available", + ); + if (selected !== undefined) + expect(inspection.javaCommand).toBe(candidates[selected]); + } + }, +); + +it.each([ + "explicit", + "path", + "missing", + "wrong version", + "wrong architecture", + "cancelled", +] as const)( + "preserves Java selection and admission during versioned discovery: %s", + async (scenario) => { + const directoryEntries = vi.fn(() => [ + "openjdk@17", + "openjdk@21", + "openjdk@999", + "openjdk@21/../foreign", + ]); + const keg = + "/opt/homebrew/opt/openjdk@21/libexec/openjdk.jdk/Contents/Home/bin/java"; + const commands: string[] = []; + const controller = new AbortController(); + const result = inspectGhidraInstallation( + { + resourceOwner: inspectionResources.owner, + environment: { PATH: "/usr/bin" }, + installDir: INSTALL, + platform: "darwin", + architecture: "arm64", + signal: controller.signal, + ...(scenario === "explicit" ? { javaHome: "/selected-jdk" } : {}), + }, + host({ + readText: () => + "application.version=12.1.4\napplication.java.min=21\napplication.java.max=21\n", + directoryEntries, + executable: (candidate) => + candidate === "/usr/bin/java" || + candidate === keg || + host().executable(candidate), + probeJava: (command) => { + commands.push(command); + if (scenario === "cancelled" && command === keg) { + controller.abort("controlled cancellation"); + throw new AnalysisCancelledError("ghidra:installation"); + } + if ( + scenario === "explicit" || + scenario === "missing" || + (scenario !== "path" && command !== keg) + ) + return missingJavaProbe; + return javaProbeOk({ + ...JAVA, + home: "/selected-jdk", + major: scenario === "wrong version" ? 17 : 21, + version: scenario === "wrong version" ? "17.0.1" : "21.0.11", + osName: "Mac OS X", + osArch: scenario === "wrong architecture" ? "x86_64" : "aarch64", + }); + }, + }), + ); + if (scenario === "cancelled") { + await expect(result).rejects.toBeInstanceOf(AnalysisCancelledError); + } else { + const inspection = await result; + expect(inspection.status).toBe( + scenario === "path" ? "available" : "unavailable", + ); + if (scenario === "wrong version" || scenario === "wrong architecture") + expect(inspection.javaCommand).toBe(keg); + } + if (scenario === "explicit" || scenario === "path") { + expect(directoryEntries).not.toHaveBeenCalled(); + expect(commands).toEqual([ + scenario === "explicit" ? "/selected-jdk/bin/java" : "/usr/bin/java", + ]); + } else expect(commands).toEqual(["/usr/bin/java", keg]); + }, +); + it("explains the macOS keg-only JDK case without probing an unselected installation", async () => { const probed: string[] = []; const inspection = await inspectGhidraInstallation( diff --git a/src/ghidra/GhidraInstallation.ts b/src/ghidra/GhidraInstallation.ts index e3b49766b4..474e0a612d 100644 --- a/src/ghidra/GhidraInstallation.ts +++ b/src/ghidra/GhidraInstallation.ts @@ -19,7 +19,13 @@ import { readGhidraInstallationProperties, type GhidraInstallationFileOptions, } from "./GhidraInstallationProperties.js"; -import { constants, accessSync, realpathSync, statSync } from "node:fs"; +import { + constants, + accessSync, + readdirSync, + realpathSync, + statSync, +} from "node:fs"; import { delimiter as pathDelimiter, dirname, posix, win32 } from "node:path"; import type { ProviderRejectionCode } from "../domain/providerSelection.js"; @@ -274,6 +280,8 @@ export interface GhidraInstallationHost { options: GhidraInstallationFileOptions, ): string | undefined | Promise; executable(path: string): boolean; + /** Optional best-effort enumeration of installed Homebrew opt aliases. */ + directoryEntries?(path: string): readonly string[]; /** * Read a bounded header prefix of one executable so admission can reject a * tool built for another platform or architecture. An omitted seam, or an @@ -512,17 +520,35 @@ const installationCoordinates = async ( nativeDecompilerHeader.error.cleanupIncomplete) ) throw nativeDecompilerHeader.error; + const parsedProperties = + properties === undefined + ? undefined + : ghidraApplicationProperties(properties); + const javaRequirement = ghidraJavaRequirement(parsedProperties); const homebrewJavaCandidates = () => (architecture === "arm64" ? ["/opt/homebrew", "/usr/local"] : ["/usr/local", "/opt/homebrew"] ) - .map((prefix) => - path.join( - prefix, - "opt/openjdk/libexec/openjdk.jdk/Contents/Home/bin/java", - ), - ) + .flatMap((prefix) => { + const opt = path.join(prefix, "opt"); + const versioned = (host.directoryEntries?.(opt) ?? []) + .filter((name) => /^openjdk@[1-9][0-9]*$/u.test(name)) + .map((name) => ({ + name, + major: Number(name.slice("openjdk@".length)), + })) + .filter( + ({ major }) => + Number.isSafeInteger(major) && + ghidraJavaMajorAccepted(major, javaRequirement), + ) + .sort((left, right) => left.major - right.major) + .map(({ name }) => name); + return ["openjdk", ...versioned].map((formula) => + path.join(opt, formula, "libexec/openjdk.jdk/Contents/Home/bin/java"), + ); + }) .filter((candidate) => host.executable(candidate)); const selectedJava: { command: string; @@ -587,7 +613,7 @@ const installationCoordinates = async ( if ( platform === "darwin" && options.javaHome === undefined && - (selectedJava.source === "path" || selectedJava.source === "default") && + selectedJava.source !== "configured" && javaProbe !== undefined && !javaProbe.ok && !(javaProbe.error instanceof AnalysisError) && @@ -600,6 +626,7 @@ const installationCoordinates = async ( failure: javaProbe.error, }); for (const candidate of homebrewJavaCandidates()) { + if (candidate === selectedJava.command) continue; const fallback = await host.probeJava( candidate, ghidraJavaEnvironment(undefined, options.environment, platform), @@ -629,10 +656,7 @@ const installationCoordinates = async ( analyzeHeadlessPath, nativeDecompilerPath, nativeDecompilerHeader, - properties: - properties === undefined - ? undefined - : ghidraApplicationProperties(properties), + properties: parsedProperties, ...(javaDiscoveryFailures.length === 0 ? {} : { javaDiscoveryFailures }), javaCommand, java: javaProbe?.ok === true ? javaProbe.value : undefined, @@ -1074,6 +1098,14 @@ const javaRemediation = ( export const systemGhidraInstallationHost = (): GhidraInstallationHost => ({ readText: readGhidraInstallationProperties, + directoryEntries(path) { + try { + return readdirSync(path); + } catch { + // Optional discovery cannot replace explicit Java selection or admission. + return []; + } + }, executableHeader: readGhidraExecutableHeader, executable(path) { try { From 772c1bc40167ce387bc9d7dd43f63ca043407079 Mon Sep 17 00:00:00 2001 From: RRFZ <247457381+Plastic-time@users.noreply.github.com> Date: Wed, 4 Nov 2026 03:45:48 -0500 Subject: [PATCH 12/94] fix(pe): reject empty and non-minimal certificate serial INTEGERs (#8836) --- src/native/pe/PeCertificateAsn1.ts | 29 +++++++- src/native/pe/PeCertificateParser.test.ts | 86 +++++++++++++++++++++++ 2 files changed, 112 insertions(+), 3 deletions(-) diff --git a/src/native/pe/PeCertificateAsn1.ts b/src/native/pe/PeCertificateAsn1.ts index 1f8985b048..2f476d4f20 100644 --- a/src/native/pe/PeCertificateAsn1.ts +++ b/src/native/pe/PeCertificateAsn1.ts @@ -214,7 +214,22 @@ const decodeDirectoryString = ( } }; -const integerHex = (content: Buffer, budget: DecodedNameBudget): string => { +// X.690 8.3.1 and 8.3.2 hold for BER as well as DER: an INTEGER has at least +// one content octet, and its first nine bits are never all zeros or all ones. +// Padded encodings would otherwise collapse onto the minimal serial's hex. +const integerHex = ( + content: Buffer, + budget: DecodedNameBudget, + label: string, +): string => { + if (content.length === 0) return fail(`${label} INTEGER is empty.`); + const second = content[1]; + if ( + second !== undefined && + ((content[0] === 0x00 && second < 0x80) || + (content[0] === 0xff && second >= 0x80)) + ) + return fail(`${label} INTEGER is not minimally encoded.`); let start = 0; while (start < content.length - 1 && content[start] === 0) start += 1; chargeDecodedName(budget, (content.length - start) * 2); @@ -444,7 +459,11 @@ export const parseCertificate = ( return { start: node.start, end: node.end, - serial_hex: integerHex(serial.content, names), + serial_hex: integerHex( + serial.content, + names, + "TBSCertificate serialNumber", + ), issuer: decodeName(issuer, names), subject: decodeName(subject, names), not_before: validityTime(notBefore, names), @@ -484,7 +503,11 @@ const signerIdentifier = ( return fail("SignerInfo issuerAndSerialNumber is truncated."); return { issuer: decodeName(name, names), - serial_hex: integerHex(serial.content, names), + serial_hex: integerHex( + serial.content, + names, + "SignerInfo issuerAndSerialNumber serialNumber", + ), subject_key_identifier_hex: null, }; } diff --git a/src/native/pe/PeCertificateParser.test.ts b/src/native/pe/PeCertificateParser.test.ts index 9b6828e812..20bb591eda 100644 --- a/src/native/pe/PeCertificateParser.test.ts +++ b/src/native/pe/PeCertificateParser.test.ts @@ -515,6 +515,92 @@ it("keeps subjectKeyIdentifier bytes on the signer", () => { ).toThrow(/decoded budget/u); }); +// X.690 8.3.1 and 8.3.2 apply under BER as well as DER. +const malformedSerials = [ + [], + [0x00, 0x00], + [0x00, 0x7f], + [0x00, 0x00, 0x80], + [0xff, 0x80], + [0xff, 0xff], +] as const; +const minimalSerials = [ + [[0x00], "00"], + [[0x7f], "7f"], + [[0x00, 0x80], "80"], + [[0xff], "ff"], + [[0xff, 0x7f], "ff7f"], + [[0x11, 0x22, 0x33, 0x44], "11223344"], +] as const; +const serialName = derSeq( + derSet( + derSeq( + der(0x06, Buffer.from([0x55, 0x04, 0x03])), + der(0x0c, Buffer.from("Fixture")), + ), + ), +); + +it("rejects empty and non-minimal certificate serialNumber INTEGERs", () => { + const certificate = (serial: readonly number[]) => + readAsn1( + derSeq( + derSeq( + der(0x02, Buffer.from(serial)), + SHA256_ALG, + serialName, + derSeq( + der(0x17, Buffer.from("250101000000Z")), + der(0x17, Buffer.from("260101000000Z")), + ), + serialName, + derSeq(SHA256_ALG, der(0x03, Buffer.from([0]))), + ), + SHA256_ALG, + der(0x03, Buffer.from([0])), + ), + ); + for (const serial of malformedSerials) + expect( + () => parseCertificate(certificate(serial)), + JSON.stringify(serial), + ).toThrow(/TBSCertificate serialNumber INTEGER is (empty|not minimally)/u); + for (const [serial, hex] of minimalSerials) + expect(parseCertificate(certificate(serial)).serial_hex).toBe(hex); +}); + +it("rejects empty and non-minimal SignerInfo serialNumber INTEGERs", () => { + const payload = (serial: readonly number[]) => + contentInfo( + derSeq( + derInt(1), + derSet(SHA256_ALG), + derSeq(DATA_OID), + derSet( + derSeq( + derInt(1), + derSeq(serialName, der(0x02, Buffer.from(serial))), + SHA256_ALG, + SHA256_ALG, + der(0x04, Buffer.from([1])), + ), + ), + ), + ); + for (const serial of malformedSerials) + expect( + () => parse(peCertificateFixture(false, payload(serial))), + JSON.stringify(serial), + ).toThrow( + /SignerInfo issuerAndSerialNumber serialNumber INTEGER is (empty|not minimally)/u, + ); + for (const [serial, hex] of minimalSerials) + expect( + parse(peCertificateFixture(false, payload(serial))).entries[0]?.pkcs7 + ?.signers[0]?.serial_hex, + ).toBe(hex); +}); + it("fails a truncated issuerAndSerialNumber instead of dropping the signer", () => { const signer = derSeq( derInt(1), From 6c3c5ebc69c408f938ec200b58c8876548fed2c6 Mon Sep 17 00:00:00 2001 From: RRFZ <247457381+Plastic-time@users.noreply.github.com> Date: Wed, 4 Nov 2026 03:47:31 -0500 Subject: [PATCH 13/94] fix(rust): state inflate limits on the consumed zlib stream (#8835) --- src/rust/RustBinaryProvider.ts | 6 +++++- tests/acceptance/rustBinaryCli.test.ts | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/rust/RustBinaryProvider.ts b/src/rust/RustBinaryProvider.ts index c80443d4c7..5a90462c0e 100644 --- a/src/rust/RustBinaryProvider.ts +++ b/src/rust/RustBinaryProvider.ts @@ -161,12 +161,16 @@ const outputLimit = (path: string) => fixedAllowanceOptions("The Rust complete-report allowance", "report"), ); +/** Resource-guard guidance delivered with every Rust inspection report. */ +export const RUST_RESOURCE_GUARD_LIMITATION = + "Resource guards bound the selected file to 256 MiB, the compressed dependency stream to 1 MiB, inflated JSON to the smaller of 8 MiB and 64 times the consumed compressed-stream length, and the complete report to 8 MiB. Package and dependency counts are admitted against that output budget before schema cloning, graph objects, or reverse-edge allocation."; + const limitationsFor = ( image: ReturnType, ): string[] => [ "Embedded dependency metadata is file content. It does not authenticate package names, versions, sources, or the compiler that produced the binary.", "Only the .dep-v0 dependency section and read-only compiler path strings are inspected. Functions, types, and runtime behavior are not recovered.", - "Resource guards bound the selected file to 256 MiB, a dependency section to 1 MiB, inflated JSON to the smaller of 8 MiB and 64 times the section size, and the complete report to 8 MiB. Package and dependency counts are admitted against that output budget before schema cloning, graph objects, or reverse-edge allocation.", + RUST_RESOURCE_GUARD_LIMITATION, ...(image.section_names_limitation === undefined ? [] : [image.section_names_limitation]), diff --git a/tests/acceptance/rustBinaryCli.test.ts b/tests/acceptance/rustBinaryCli.test.ts index 0658501bb0..9d461b36bb 100644 --- a/tests/acceptance/rustBinaryCli.test.ts +++ b/tests/acceptance/rustBinaryCli.test.ts @@ -6,6 +6,7 @@ import { pathToFileURL } from "node:url"; import { join } from "node:path"; import { expect } from "vitest"; import { toolContract } from "../../src/contracts/toolContracts.js"; +import { RUST_RESOURCE_GUARD_LIMITATION } from "../../src/rust/RustBinaryProvider.js"; import { machoImageWithUnknownCommand } from "../../src/artifacts/apple/MachoImage.fixture.js"; import { compressRustElfSectionNames, @@ -465,6 +466,9 @@ cliTest( expect(metadata?.uncompressed_bytes).toBeLessThanOrEqual( (metadata?.compressed_bytes ?? 0) * 64, ); + // Padding separates the section size from the consumed stream that the + // inflate guidance names. + expect(evidence.limitations).toContain(RUST_RESOURCE_GUARD_LIMITATION); expect(evidence.locations).toContainEqual({ kind: "file-offset-range", start: metadata?.file_offset, From 47cedcc93aa639fb8445ab344308a6837d2b9d5a Mon Sep 17 00:00:00 2001 From: Abduallah Jamal <60349056+abduallahJamal@users.noreply.github.com> Date: Wed, 4 Nov 2026 10:47:46 +0200 Subject: [PATCH 14/94] fix(cli): reject empty artifact command targets before resolution (#8834) --- src/cli/artifactCommands.ts | 19 ++- .../cli/emptyArtifactBundleTargetCli.test.ts | 119 ++++++++++++++++++ .../cli/emptyLocalTargetAdmission.test.ts | 16 ++- 3 files changed, 147 insertions(+), 7 deletions(-) create mode 100644 tests/boundary/cli/emptyArtifactBundleTargetCli.test.ts diff --git a/src/cli/artifactCommands.ts b/src/cli/artifactCommands.ts index 79e2b9b6ae..2431d916e3 100644 --- a/src/cli/artifactCommands.ts +++ b/src/cli/artifactCommands.ts @@ -13,6 +13,7 @@ import { withCommandCancellation } from "./commandCancellation.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { ARTIFACT_GRAPH_PROVIDER } from "../domain/investigationProviders.js"; import { keyedArchiveInputSchema } from "../domain/apple/keyedArchive.js"; +import { localPathStringSchema } from "../domain/localPath.js"; import { z } from "incur"; import type { DirectAnalysis } from "../composition/directAnalysis.js"; @@ -43,7 +44,9 @@ export const registerArtifactCommands = ( description: "Inspect a Foundation keyed archive as an object-reference graph", args: z.object({ - path: z.string().describe("Foundation plist archive or Apple app bundle"), + path: localPathStringSchema.describe( + "Foundation plist archive or Apple app bundle", + ), archive: keyedArchiveInputSchema.shape.path.describe( "Relative archive path when the target is a bundle", ), @@ -87,7 +90,9 @@ const registerDylibResolutionCommand = ( description: "Trace dyld load-path resolution for a Mach-O or every executable in an app bundle", args: z.object({ - path: z.string().describe("Mach-O file or Apple .app bundle path"), + path: localPathStringSchema.describe( + "Mach-O file or Apple .app bundle path", + ), }), options: z.object({ root: z @@ -125,7 +130,9 @@ const registerAssetCatalogCommand = ( ): void => { cli.command(CLI_COMMANDS.inspectAssetCatalog, { description: "Inspect compiled Apple asset catalog metadata", - args: z.object({ path: z.string().describe("Apple .app bundle path") }), + args: z.object({ + path: localPathStringSchema.describe("Apple .app bundle path"), + }), options: z.object({ offset: z .number() @@ -162,7 +169,7 @@ const registerInterfaceBuilderCommand = ( description: "Decode compiled storyboard and nib archives from an app bundle", args: z.object({ - path: z.string().describe("Apple .app bundle path"), + path: localPathStringSchema.describe("Apple .app bundle path"), }), options: z.object({ maxDocuments: z @@ -257,7 +264,7 @@ const registerExtractionCommand = ( description: "Extract regular contents of supported artifacts; DMG extraction is unavailable", args: z.object({ - path: z.string().describe("Application or package path"), + path: localPathStringSchema.describe("Application or package path"), }), options: z.object({ integrityPolicy: z @@ -293,7 +300,7 @@ const registerInspectionCommand = ( description: "Inspect one artifact and return all available observations and next probes", args: z.object({ - path: z.string().min(1).describe("Application or package path"), + path: localPathStringSchema.describe("Application or package path"), }), options: z.object({ ...analysisEvidenceDeliveryOptions, diff --git a/tests/boundary/cli/emptyArtifactBundleTargetCli.test.ts b/tests/boundary/cli/emptyArtifactBundleTargetCli.test.ts new file mode 100644 index 0000000000..50b8aeac86 --- /dev/null +++ b/tests/boundary/cli/emptyArtifactBundleTargetCli.test.ts @@ -0,0 +1,119 @@ +import { mkdir, readdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { build } from "plist"; +import { expect } from "vitest"; + +import { thinMach } from "../../../src/domain/binaryTarget.fixture.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const createWorkingBundleFixture = async (prefix: string) => { + const root = await createTestTempDirectory(prefix); + const bundle = join(root, "Example.app"); + const temp = join(root, "temp"); + const contents = join(bundle, "Contents"); + await mkdir(join(contents, "MacOS"), { recursive: true }); + await mkdir(join(contents, "Resources"), { recursive: true }); + await mkdir(temp, { recursive: true }); + await writeFile( + join(contents, "Info.plist"), + build({ CFBundleExecutable: "Example" }), + ); + await writeFile( + join(contents, "MacOS", "Example"), + thinMach(0xcffaedfe, 0x0100000c), + ); + await writeFile( + join(contents, "Resources", "Archive.plist"), + build({ + $archiver: "NSKeyedArchiver", + $version: 100000, + $objects: ["$null", "resource-only-value"], + $top: { root: { CF$UID: 1 } }, + }), + ); + const environment = { + HOME: join(root, "home"), + USERPROFILE: join(root, "home"), + TMPDIR: temp, + TMP: temp, + TEMP: temp, + REA_LOG_LEVEL: "silent", + }; + return { bundle, temp, environment }; +}; + +cliTest( + "compiled CLI rejects empty inspect-keyed-archive target in a bundle cwd while preserving explicit dot", + async ({ cli }) => { + const { bundle, environment } = await createWorkingBundleFixture( + "rea-empty-keyed-bundle-", + ); + const archive = "Contents/Resources/Archive.plist"; + const empty = await cli.run({ + arguments: ["inspect-keyed-archive", "", archive, "--json"], + cwd: bundle, + environment, + }); + expect(empty.exitCode).not.toBe(0); + expect(empty.json).toMatchObject({ + code: "VALIDATION_ERROR", + fieldErrors: [ + expect.objectContaining({ path: "path", code: "too_small" }), + ], + }); + + const explicitDot = await cli.run({ + arguments: ["inspect-keyed-archive", ".", archive, "--json"], + cwd: bundle, + environment, + }); + expect(explicitDot.exitCode, explicitDot.stderr).toBe(0); + const evidence = parseEvidence(explicitDot.json); + expect(evidence.parameters).toMatchObject({ input_path: "." }); + expect(evidence.normalized_result).toMatchObject({ + total_objects: 2, + archive_path: archive, + }); + }, +); + +cliTest( + "compiled CLI rejects empty extract-artifact and bundle artifact targets before touching the working directory", + async ({ cli }) => { + const { bundle, temp, environment } = await createWorkingBundleFixture( + "rea-empty-artifact-bundle-", + ); + for (const command of [ + "extract-artifact", + "decode-interface-builder", + "inspect-asset-catalog", + "trace-dylib-resolution", + ]) { + const empty = await cli.run({ + arguments: [command, "", "--json"], + cwd: bundle, + environment, + }); + expect(empty.exitCode, command).not.toBe(0); + expect(empty.json).toMatchObject({ + code: "VALIDATION_ERROR", + fieldErrors: [ + expect.objectContaining({ path: "path", code: "too_small" }), + ], + }); + } + expect(await readdir(temp)).toEqual([]); + + const explicitExtract = await cli.run({ + arguments: ["extract-artifact", ".", "--json"], + cwd: bundle, + environment, + }); + expect(explicitExtract.exitCode, explicitExtract.stderr).toBe(0); + expect(parseEvidence(explicitExtract.json).parameters).toMatchObject({ + input_path: ".", + }); + }, +); diff --git a/tests/boundary/cli/emptyLocalTargetAdmission.test.ts b/tests/boundary/cli/emptyLocalTargetAdmission.test.ts index 4e28416a72..9d288516da 100644 --- a/tests/boundary/cli/emptyLocalTargetAdmission.test.ts +++ b/tests/boundary/cli/emptyLocalTargetAdmission.test.ts @@ -1,5 +1,6 @@ import { Cli } from "incur"; import { expect, it, vi } from "vitest"; +import { registerArtifactCommands } from "../../../src/cli/artifactCommands.js"; import { registerCoreBinaryCommands } from "../../../src/cli/coreBinaryCommands.js"; import { registerCoreNativeCommands } from "../../../src/cli/coreNativeCommands.js"; import { registerManagedCommands } from "../../../src/cli/managedCommands.js"; @@ -16,6 +17,7 @@ const registry = () => { const runDirectAnalysis = run(); const runProviderAnalysis = run(); const cli = Cli.create("empty-target-admission", { sync: false }); + registerArtifactCommands(cli, silentLogger, runProviderAnalysis); registerCoreBinaryCommands(cli, silentLogger, runDirectAnalysis, {}); registerCoreNativeCommands(cli, silentLogger, runDirectAnalysis); registerManagedCommands(cli, silentLogger, runProviderAnalysis); @@ -44,6 +46,12 @@ const registry = () => { // Each command with its other required arguments; the target comes first. const commands: readonly (readonly string[])[] = [ + ["inspect-artifact"], + ["extract-artifact"], + ["inspect-keyed-archive"], + ["trace-dylib-resolution"], + ["inspect-asset-catalog"], + ["decode-interface-builder"], ["inspect"], ["decompile", "main"], ["xrefs", "0x1000"], @@ -83,7 +91,7 @@ const commands: readonly (readonly string[])[] = [ ]; it.each(commands)( - "rejects an empty %s target before resolution and admits explicit .", + "rejects an empty or NUL %s target before resolution and admits explicit .", async (command, ...rest) => { const { serve, runs } = registry(); // docs/cli.md: an empty string never selects the working directory. @@ -93,6 +101,12 @@ it.each(commands)( expect.objectContaining({ path: "path", code: "too_small" }), ], }); + expect(await serve([command, "target\x00name", ...rest])).toMatchObject({ + code: "VALIDATION_ERROR", + fieldErrors: [ + expect.objectContaining({ path: "path", code: "invalid_format" }), + ], + }); for (const run of runs) expect(run).not.toHaveBeenCalled(); await serve([command, ".", ...rest]); expect(runs.flatMap((run) => run.mock.calls.map(([path]) => path))).toEqual( From 578e4bb71ae8ff66fdbc7f3ade62638fb965528d Mon Sep 17 00:00:00 2001 From: SathishM Date: Wed, 4 Nov 2026 16:32:32 +0530 Subject: [PATCH 15/94] docs(installation): list every engine accepted by doctor --provider (#4767) The One analysis engine readiness row offered ghidra (or hopper, ida) while the CLI accepts hopper, ghidra, ida, jadx and kuna for doctor --provider. Add the missing IDs and keep the table aligned. Related to #4624: docs/cli.md carries the same stale enumeration and is updated separately in #4651. --- docs/installation.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index c788ae6168..46b282f8bf 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -378,12 +378,12 @@ a particular caller-supplied input will be valid. See Select the part you want to check: -| Task | Readiness check | -| ----------------------------------- | ------------------------------------------------------------------------ | -| Static JavaScript/Electron analysis | Run `rea analyze-javascript-application PATH --json` directly. | -| One analysis engine | `rea doctor --provider ghidra --json` (or `hopper`, `ida`, `kuna`) | -| One agent registration | `rea doctor --client codex --json` (see [client IDs](#supported-agents)) | -| Installed workflow instructions | `rea doctor --skill --json` | +| Task | Readiness check | +| ----------------------------------- | -------------------------------------------------------------------------- | +| Static JavaScript/Electron analysis | Run `rea analyze-javascript-application PATH --json` directly. | +| One analysis engine | `rea doctor --provider ghidra --json` (or `hopper`, `ida`, `jadx`, `kuna`) | +| One agent registration | `rea doctor --client codex --json` (see [client IDs](#supported-agents)) | +| Installed workflow instructions | `rea doctor --skill --json` | A scoped report has `scope.mode: "explicit"`. Its `scope_checks` determine `healthy` and the exit status. Other checks appear in `informational_checks`; From 6ba69cdcc1f65f61e49a429c75582164f494575c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Andrade?= <95101505+viniciusdsandrade@users.noreply.github.com> Date: Wed, 4 Nov 2026 08:02:54 -0300 Subject: [PATCH 16/94] docs(cli): document standard input for JSON request arguments (#7139) An argument that accepts inline JSON or a JSON file path also reads the request from standard input when it is "-". The behavior shipped without documentation. Describe it next to the request-file example, with its UTF-8, byte-order mark and 8 MiB limits. Fixes #7138 --- docs/cli.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/cli.md b/docs/cli.md index 76a3e5ef5c..659bdc635b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -39,6 +39,17 @@ pass an absolute path. REA reads and authenticates the file, then projects it like retained Evidence, so a large file is bounded by the file reader rather than charged as inline input. +An argument that accepts inline JSON or a JSON file path also accepts `-`, which +reads the request from standard input, so a generated request needs no +temporary file: + +```bash +jq -nc '{source: {kind: "evidence-file", path: "app-evidence.json"}, view: {kind: "summary"}}' | rea inspect-analysis-view - +``` + +Standard input is read once as a single UTF-8 JSON document without a +byte-order mark, and it is limited to 8 MiB. Put a larger request in a file. + Local file inspection commands reject empty paths and NUL bytes before resolving relative inputs, auxiliary files or output destinations. An empty string never selects the working directory; use an explicit `.` when selecting that directory From 67552d56cce45a4fedffa544f9f6e3bbcc418fcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A4=80=ED=98=B8?= <101695482+chordpli@users.noreply.github.com> Date: Wed, 4 Nov 2026 20:03:56 +0900 Subject: [PATCH 17/94] docs(limits): say setup forwards the selected MCP response budget (#8342) The design guide said the setup allowlist does not copy REA_MCP_MAX_RESPONSE_BYTES, but SELECTED_REGISTRATION_SETTINGS includes it and setup forwards an explicit selection, as the client guide states. Fixes #7420 --- docs/design/limits-and-recovery.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/design/limits-and-recovery.md b/docs/design/limits-and-recovery.md index 9eccda38fb..c6ea9dde98 100644 --- a/docs/design/limits-and-recovery.md +++ b/docs/design/limits-and-recovery.md @@ -121,9 +121,9 @@ launch environment affects the next REA process. is the byte-budget shape: a decimal safe integer with a 10 MiB floor (`MINIMUM_RESPONSE_BYTES`), the same shared schema, and no silent fallback. A change needs a REA restart and a matching client receive buffer. It does not -raise REA's reserved string limit or Node's string ceiling. The setup -allowlist does not copy this setting. A new headroom setting should, following -the regex deadline. +raise REA's reserved string limit or Node's string ceiling. Like the regex +deadline, it is in `SELECTED_REGISTRATION_SETTINGS`, so setup forwards an +explicit selection into the client registration. A per-call budget can have a runtime default without having an environment setting. JavaScript `max_heap_mb` From 52ff3123cbe07afbbadb3c5d24955cbba2d1835d Mon Sep 17 00:00:00 2001 From: RRFZ <247457381+Plastic-time@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:04:57 -0500 Subject: [PATCH 18/94] docs(mcp): list every tool that omits the untrusted-content marker (#8837) --- docs/mcp-contracts.md | 10 ++++++---- .../contracts/targetContentProvenance.test.ts | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 6ac4ac4411..2bc4680ed6 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -500,10 +500,12 @@ result's `_meta`, whether the call succeeded or failed: ``` The marker is declared once per tool contract (`returnsTargetContent`) and is -on by default: `export_evidence_bundle`, `record_unknown`, `search_guidance`, -and `read_guidance_section` omit it. The first two contain only REA session -state or caller-supplied values; the guidance tools return REA's own installed -documentation, not text from an analysis target. Tools whose results or errors +on by default: `inspect_analysis_settings`, `update_analysis_settings`, +`export_evidence_bundle`, `record_unknown`, `guided_workflows`, +`search_guidance`, and `read_guidance_section` omit it. The first four contain +only REA session state or caller-supplied values; the workflow and guidance +tools return REA's own installed workflows and documentation, not text from an +analysis target. Tools whose results or errors can include provider, launcher, or device output stay marked, including `binary_session` (provider failure reasons and availability diagnostics), `close_binary` (provider cleanup diff --git a/tests/conformance/contracts/targetContentProvenance.test.ts b/tests/conformance/contracts/targetContentProvenance.test.ts index 3041892013..58ec19d16d 100644 --- a/tests/conformance/contracts/targetContentProvenance.test.ts +++ b/tests/conformance/contracts/targetContentProvenance.test.ts @@ -1,3 +1,6 @@ +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; + import { describe, expect, it } from "vitest"; import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js"; @@ -28,6 +31,17 @@ describe("target-content provenance audit", () => { ]); }); + it("documents exactly the audited content-free exceptions", async () => { + const guide = await readFile(resolve("docs/mcp-contracts.md"), "utf8"); + const exceptions = /on by default: ([^.]*?) omit it\./su.exec(guide)?.[1]; + expect(exceptions).toBeDefined(); + expect( + [...(exceptions ?? "").matchAll(/`([a-z_]+)`/gu)] + .map(([, name]) => name) + .sort(), + ).toEqual([...TARGET_CONTENT_FREE_TOOLS].sort()); + }); + it.each([ "procedure_pseudo_code", "list_strings", From 1df7ac9320cb8cfbabb5fa3b2ca02f3a86b2c376 Mon Sep 17 00:00:00 2001 From: Entrpi <133157614+Entrpi@users.noreply.github.com> Date: Wed, 4 Nov 2026 22:09:33 +1100 Subject: [PATCH 19/94] fix(ghidra): update the host no-return probe for the current bridge (#8462) The host no-return verifier, scripts/verify-real-ghidra-noreturn.mjs, fails before its Evidence checks. ReaNoReturnProbe calls private bridge methods through reflection, and two bridge changes left the probe behind: - d4bd65c53 added a Function parameter to noReturnCallLimitations, and 225f26a5f renamed the method to functionBoundaryLimitations. The probe still looked up noReturnCallLimitations(Iterable) and stopped with NoSuchMethodException. - 01a1b3fda made procedureReferences return a function observation envelope. The probe read references and unresolved_calls from the top level of that envelope instead of from its value. Pass each function and its instructions to functionBoundaryLimitations, and read procedureReferences results from the envelope value. The require checks are unchanged. CI does not run this verifier. These edits are the probe repair from robotaro's open draft that treats demangled C++ runtime imports as returning, copied line for line, so either change merges cleanly after the other. Co-authored-by: robotaro <1141230+robotaro@users.noreply.github.com> --- .../conformance/ghidra/ReaNoReturnProbe.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/conformance/ghidra/ReaNoReturnProbe.java b/tests/conformance/ghidra/ReaNoReturnProbe.java index 19f98f346a..c5ca14817f 100644 --- a/tests/conformance/ghidra/ReaNoReturnProbe.java +++ b/tests/conformance/ghidra/ReaNoReturnProbe.java @@ -123,7 +123,8 @@ public class ReaNoReturnProbe extends GhidraScript { query.addProperty("direction", "outgoing"); Method read = bridge.getClass().getDeclaredMethod("procedureReferences", JsonObject.class); read.setAccessible(true); - JsonObject output = (JsonObject) read.invoke(bridge, query); + // procedureReferences returns a function observation envelope. + JsonObject output = ((JsonObject) read.invoke(bridge, query)).getAsJsonObject("value"); require("interior_reference_keeps_known_procedure", output.getAsJsonArray("references").asList().stream().anyMatch( edge -> !edge.getAsJsonObject().get("target_procedure").isJsonNull() && edge.getAsJsonObject().getAsJsonObject("target_procedure").get("name").getAsString().equals(target.getName()))); @@ -185,13 +186,13 @@ public class ReaNoReturnProbe extends GhidraScript { query.addProperty("direction", "outgoing"); Method edges = bridge.getClass().getDeclaredMethod("procedureReferences", JsonObject.class); edges.setAccessible(true); - JsonObject known = (JsonObject) edges.invoke(bridge, query); + JsonObject known = ((JsonObject) edges.invoke(bridge, query)).getAsJsonObject("value"); require("non_call_known_reference_is_not_unresolved", known.getAsJsonArray("unresolved_calls").isEmpty()); references.delete(added); Address unknown = currentProgram.getMemory().getMaxAddress().add(0x1000); added = references.addMemoryReference(source.getAddress(), unknown, RefType.UNCONDITIONAL_CALL, SourceType.USER_DEFINED, -1); - JsonObject unresolved = (JsonObject) edges.invoke(bridge, query); + JsonObject unresolved = ((JsonObject) edges.invoke(bridge, query)).getAsJsonObject("value"); require("non_call_unknown_reference_keeps_site", unresolved.getAsJsonArray("unresolved_calls").size() == 1); } finally { references.delete(added); @@ -253,14 +254,18 @@ public class ReaNoReturnProbe extends GhidraScript { !adjacentCaller.getBody().contains(adjacentEntry)); GhidraScript bridge = script("ReaGhidraBridge.java", new File(directory, "ReaGhidraBridge.java")); - Method warnings = bridge.getClass().getDeclaredMethod("noReturnCallLimitations", Iterable.class); + Method warnings = bridge.getClass().getDeclaredMethod( + "functionBoundaryLimitations", Function.class, Iterable.class); warnings.setAccessible(true); - Object reported = warnings.invoke(bridge, currentProgram.getListing().getInstructions(function("rea_unknown").getBody(), true)); + Object reported = warnings.invoke(bridge, function("rea_unknown"), + currentProgram.getListing().getInstructions(function("rea_unknown").getBody(), true)); require("unknown_boundary_named", reported instanceof List values && values.stream().anyMatch(value -> value.toString().contains("rea_unknown_stop") && value.toString().contains("fallthrough"))); - Object healthy = warnings.invoke(bridge, currentProgram.getListing().getInstructions(function("rea_tls").getBody(), true)); + Object healthy = warnings.invoke(bridge, function("rea_tls"), + currentProgram.getListing().getInstructions(function("rea_tls").getBody(), true)); require("healthy_boundary_no_warning", healthy instanceof List values && values.isEmpty()); - Object guarded = warnings.invoke(bridge, currentProgram.getListing().getInstructions(adjacentCaller.getBody(), true)); + Object guarded = warnings.invoke(bridge, adjacentCaller, + currentProgram.getListing().getInstructions(adjacentCaller.getBody(), true)); require("unrepaired_boundary_named", guarded instanceof List values && values.stream().anyMatch(value -> value.toString().contains("malloc"))); Method initialize = bridge.getClass().getDeclaredMethod("initializeDecompiler"); From f8ec48095a2c4c1d2eb3836491c9ffcd625d7911 Mon Sep 17 00:00:00 2001 From: hhang <17309584214@163.com> Date: Wed, 4 Nov 2026 19:10:35 +0800 Subject: [PATCH 20/94] fix(cli): guide delimited analysis view exports (#8730) Co-authored-by: hhang <169027046+xu91102@users.noreply.github.com> --- docs/cli.md | 5 ++++- src/cli/analysisViewCommands.ts | 2 +- .../boundary/cli/delimitedAnalysisViewCli.test.ts | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 659bdc635b..bcb344daaf 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1196,9 +1196,12 @@ treated as malformed. exact array path with `--filter-output` and provide comma-separated direct scalar columns with `--columns`; column order is the output column order: +For an artifact inventory request that selects the `occurrences` collection +and includes `logical_path` and `artifact_id` in its view fields: + ```bash rea inspect-analysis-view request.json --format csv \ - --filter-output normalized_result.rows --columns name,address + --filter-output normalized_result.items --columns logical_path,artifact_id ``` The first line is a REA metadata record, diff --git a/src/cli/analysisViewCommands.ts b/src/cli/analysisViewCommands.ts index 427ab6709b..90f8b86947 100644 --- a/src/cli/analysisViewCommands.ts +++ b/src/cli/analysisViewCommands.ts @@ -73,7 +73,7 @@ export const registerAnalysisViewCommands = ( }); cli.command(CLI_COMMANDS.inspectAnalysisView, { description: - "Project a selected view of completed Evidence JSON for a supported analysis operation", + "Project a selected view of completed Evidence JSON for a supported analysis operation. For inventory rows, use --format csv or --format tsv with --filter-output normalized_result.items and --columns logical_path,artifact_id (if those fields are included in the view).", args: z.object({ inputJson: z.string().describe("Inline workflow JSON or JSON file path"), }), diff --git a/tests/boundary/cli/delimitedAnalysisViewCli.test.ts b/tests/boundary/cli/delimitedAnalysisViewCli.test.ts index f83202a18a..e2f8e377b6 100644 --- a/tests/boundary/cli/delimitedAnalysisViewCli.test.ts +++ b/tests/boundary/cli/delimitedAnalysisViewCli.test.ts @@ -7,6 +7,20 @@ import { ARTIFACT_COMPARISON_EXAMPLE } from "../../../src/contracts/artifactComp import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; +cliTest( + "shows how to export inventory rows in command help", + async ({ cli }) => { + const result = await cli.run({ + arguments: ["inspect-analysis-view", "--help"], + }); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("--format csv"); + expect(result.stdout).toContain("--format tsv"); + expect(result.stdout).toContain("--filter-output normalized_result.items"); + expect(result.stdout).toContain("--columns logical_path,artifact_id"); + }, +); + cliTest( "exports selected artifact inventory columns through the public CLI", async ({ cli }) => { From 007e181f680250f8b2325c6e0c2d993058d0dd2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E4=BD=B3=E6=B5=A9Alben?= <35291372+XiaoBinGan@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:11:35 +0800 Subject: [PATCH 21/94] fix(javascript): require exactly one fork IPC channel (#8017) * fix(javascript): require exactly one fork IPC channel Node's fork() rejects an explicit stdio array that carries more than one IPC channel, and the throw is synchronous, before any child process exists: if (ArrayPrototypeFilter(options.stdio, ...'ipc').length > 1) throw new ERR_IPC_ONE_PIPE() The analysis already dropped an array with no "ipc" (#7992) but still reported a resolved fork for an array carrying it twice. Measured on Node 24.22.1: fork({stdio: ["ipc", "ipc"]}) -> THROWS ERR_IPC_ONE_PIPE fork({stdio: ["pipe", "ipc", "ipc"]}) -> THROWS ERR_IPC_ONE_PIPE fork({stdio: ["ipc"]}) -> SPAWNED connected=true fork({stdio: ["pipe", "ipc"]}) -> SPAWNED connected=true fork({stdio: ["pipe", "pipe", "pipe", "ipc"]})-> SPAWNED connected=true So the required count is exactly one, not at least one. Count the resolved "ipc" literals and reject the candidate unless the count is one; a spread or an unreadable element may still change the count and is left alone, as before. * chore: retrigger title and readiness checks after the title was corrected --- ...riptSemanticAnalysis.fork_stdio_ipc.test.ts | 12 ++++++++++++ .../javascriptSemanticChildProcesses.ts | 18 ++++++++++-------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/domain/javascript/javascriptSemanticAnalysis.fork_stdio_ipc.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.fork_stdio_ipc.test.ts index 0d3e845a3d..fb9e369ec8 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.fork_stdio_ipc.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.fork_stdio_ipc.test.ts @@ -37,6 +37,18 @@ it.each([ ]); }); +/** + * Exactly one channel is required: naming `"ipc"` twice throws ERR_IPC_ONE_PIPE + * for the same synchronous reason, so those calls are not spawns either. + */ +it.each([ + ["ipc twice", `["ipc", "ipc"]`], + ["three ipc entries", `["ipc", "ipc", "ipc"]`], + ["a pipe between two ipc entries", `["pipe", "ipc", "ipc"]`], +] as const)("emits no fork for a stdio array with %s", (_name, stdio) => { + expect(spawnsFor(`fork("worker.js", { stdio: ${stdio} })`)).toEqual([]); +}); + it.each([ ["a string stdio", `fork("worker.js", { stdio: "pipe" })`], ["omitted stdio", `fork("worker.js", {})`], diff --git a/src/domain/javascript/javascriptSemanticChildProcesses.ts b/src/domain/javascript/javascriptSemanticChildProcesses.ts index ea8c872e19..ea7fb69ab9 100644 --- a/src/domain/javascript/javascriptSemanticChildProcesses.ts +++ b/src/domain/javascript/javascriptSemanticChildProcesses.ts @@ -450,12 +450,13 @@ const stdioMode = ( }; /** - * Node's `fork` requires an explicit `stdio` array to carry the IPC channel. An - * array that provably omits `"ipc"` throws ERR_CHILD_PROCESS_IPC_REQUIRED - * synchronously, before any child process exists, so it is not a spawn at all. - * A string `stdio` is normalised with `"ipc"` appended, so only the array form - * can omit it; a spread or unreadable element may still supply the channel and - * is therefore left alone. + * Node's `fork` requires an explicit `stdio` array to carry exactly one IPC + * channel. An array that provably omits `"ipc"` throws + * ERR_CHILD_PROCESS_IPC_REQUIRED, and one that names `"ipc"` twice throws + * ERR_IPC_ONE_PIPE; both throws are synchronous, before any child process + * exists, so such a call is not a spawn at all. A string `stdio` is normalised + * with a single `"ipc"` appended, so only the array form can be wrong; a spread + * or unreadable element may still change the count and is therefore left alone. */ const forkRejectsStdioArray = ( candidate: SpawnCandidate, @@ -477,10 +478,11 @@ const forkRejectsStdioArray = ( ); if (literals.some((literal) => literal === undefined || !literal.found)) return false; - return !literals.some( + const ipcCount = literals.filter( (literal) => literal !== undefined && literal.found && literal.value === "ipc", - ); + ).length; + return ipcCount !== 1; }; /** From a56cc93cabe81242dcadaee85a129a14fc05bd2a Mon Sep 17 00:00:00 2001 From: Beqa <91750923+BeqaKs@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:13:53 +0400 Subject: [PATCH 22/94] test(desktop): cover ENOTDIR intermediate path refusal (#8692) * test(desktop): cover ENOTDIR intermediate path refusal Lock the desktop consumer projection for selections whose intermediate component is a regular file, matching inspect-artifact's unavailable-target refusal after directory source spelling validation. Fixes #7424 * style: oxfmt ArtifactSourceResolution test --- .../ArtifactSourceResolution.test.ts | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/artifacts/ArtifactSourceResolution.test.ts b/src/artifacts/ArtifactSourceResolution.test.ts index 745705b4aa..7101a31382 100644 --- a/src/artifacts/ArtifactSourceResolution.test.ts +++ b/src/artifacts/ArtifactSourceResolution.test.ts @@ -8,6 +8,9 @@ import { assertArtifactFileSelectedSource } from "./ArtifactFileSource.js"; import { assertArtifactSelectedSource } from "./ArtifactSelectedSource.js"; import { readStableArtifact } from "./readStableArtifact.js"; import { DirectoryArtifactReader } from "./DirectoryArtifactReader.js"; +import { createDesktopApplicationService } from "../composition/desktop.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { desktopApplicationSchema } from "../domain/desktopApplication.js"; it.skipIf(process.platform === "win32")( "admits a caller-selected directory symlink without changing its target identity", @@ -46,6 +49,54 @@ it.skipIf(process.platform === "win32")( }, ); +it.skipIf(process.platform === "win32")( + "rejects desktop inspection when the selected spelling traverses a regular file", + async () => { + const root = await createTestTempDirectory( + "rea-desktop-enotdir-intermediate-", + ); + const app = join(root, "Demo.app"); + await mkdir(app); + await writeFile( + join(app, "Info.plist"), + ` + +CFBundleIdentifiercom.example.demo`, + ); + await writeFile(join(app, "marker.txt"), "marker\n"); + await writeFile(join(root, "regular"), "regular file\n"); + const service = createDesktopApplicationService(); + try { + const invalid = await service.inspect({ + path: `${root}/regular/../Demo.app`, + max_entries: 32, + max_probe_bytes: 4096, + }); + expect(invalid.ok).toBe(false); + if (invalid.ok) throw new Error("expected ENOTDIR refusal"); + expect(projectAnalysisError(invalid.error)).toMatchObject({ + code: "target_unavailable", + category: "unavailable", + }); + const valid = await service.inspect({ + path: app, + max_entries: 32, + max_probe_bytes: 4096, + }); + expect(valid.ok).toBe(true); + if (!valid.ok) throw valid.error; + expect( + desktopApplicationSchema + .parse(valid.value.normalized_result) + .files.map((file) => file.path) + .sort(), + ).toEqual(["Info.plist", "marker.txt"]); + } finally { + await service.close(); + } + }, +); + it.skipIf(process.platform === "win32")( "reports a replaced ancestor symlink loop as a changed admitted source", async () => { From 5b953e13c4451df3afcaab7724c06db5eedf58f7 Mon Sep 17 00:00:00 2001 From: Christopher Pruijsen Date: Wed, 4 Nov 2026 11:14:52 +0000 Subject: [PATCH 23/94] fix(mcp): keep detail guidance in compact input schemas (#8034) Compact input schemas are the default advertisement since #5975, and the profile drops every property description. On 23 of the 31 tools with a guided `detail` option, including inspect_artifact and inspect_macho, `detail` is advertised as a bare complete|summary enum, so callers take the complete default. On a universal macOS app that reply was about 699k characters for inspect_artifact and 336k for inspect_macho, against about 4.4k and 3.2k with summary. Keep the canonical description of a root `detail` property in the inline compact input form. The reduced form already keeps first-level guidance, and compact output schemas are unchanged. Advertised compact input schemas grow from 127,616 to 133,358 bytes across 233 tools; the largest schema stays at 4,107 bytes, within the 13,312-byte budget. The new boundary test compares `detail` guidance between the full and compact tools/list and fails before the change. --- docs/mcp-contracts.md | 4 ++- src/contracts/compactInputSchema.ts | 33 ++++++++++++++++++- .../mcp/compactInputSchemaMcp.test.ts | 27 +++++++++++++++ 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 2bc4680ed6..b4bf29bce2 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -1707,7 +1707,9 @@ JSON Schema: - Annotation prose leaves the advertised form: nested property descriptions, literal examples, defaults, and titles are dropped. The root keeps its - description because it states the accepted input groups. + description because it states the accepted input groups. A root `detail` + property keeps its description because it selects between complete Evidence + and a bounded summary. - Schema-local references are inlined within the budget. Kimi Code expands references before sending provider requests, so sharing definitions cannot establish the provider-facing size. Expansion stops at the budget before diff --git a/src/contracts/compactInputSchema.ts b/src/contracts/compactInputSchema.ts index e7ec54ae53..d45329674f 100644 --- a/src/contracts/compactInputSchema.ts +++ b/src/contracts/compactInputSchema.ts @@ -2,6 +2,7 @@ import { COMPACT_SCHEMA_MAX_DEPTH, hasGatewayIncompatibleShape, inlineWithinBudget, + isSchemaObject, reduceToShallowProperties, schemaDepth, serializedBytes, @@ -28,6 +29,34 @@ export interface CompactedInputSchema { const INPUT_REDUCED_ANNOTATION = "Advertised in reduced form by the compact input schema profile; the server still validates complete canonical inputs."; +// `detail` selects between complete Evidence and a bounded summary. Without +// its guidance, callers default to complete replies that can exceed context. +const retainedRootPropertyDescriptions = ["detail"]; + +/** Restore the description of root properties whose choice bounds the reply. */ +const retainRootPropertyGuidance = ( + schema: Record, + stripped: Record, +): Record => { + if ( + !isSchemaObject(schema.properties) || + !isSchemaObject(stripped.properties) + ) + return stripped; + const properties = { ...stripped.properties }; + for (const name of retainedRootPropertyDescriptions) { + const source = schema.properties[name]; + const kept = properties[name]; + if ( + isSchemaObject(source) && + isSchemaObject(kept) && + typeof source.description === "string" + ) + properties[name] = { ...kept, description: source.description }; + } + return { ...stripped, properties }; +}; + /** * Render an advertised input schema within a per-schema byte budget for * providers that validate the literal serialized schema size. Validation @@ -41,7 +70,9 @@ export const compactAdvertisedInputSchema = ( ): CompactedInputSchema => { const expanded = inlineWithinBudget(stripAnnotations(schema), budgetBytes); const inline = - expanded === undefined ? undefined : typeArrayItemUnions(expanded); + expanded === undefined + ? undefined + : typeArrayItemUnions(retainRootPropertyGuidance(schema, expanded)); if ( inline !== undefined && serializedBytes(inline) <= budgetBytes && diff --git a/tests/boundary/mcp/compactInputSchemaMcp.test.ts b/tests/boundary/mcp/compactInputSchemaMcp.test.ts index 8b916cca73..87a3d53554 100644 --- a/tests/boundary/mcp/compactInputSchemaMcp.test.ts +++ b/tests/boundary/mcp/compactInputSchemaMcp.test.ts @@ -644,3 +644,30 @@ it.each(["full", "compact"] as const)( expect(calls).toBe(1); }, ); + +it("keeps root detail guidance so callers can choose bounded summaries", async () => { + const full = fullServer(); + const compact = compactServer(); + registerCatalog(full); + registerCatalog(compact); + const [fullTools, compactTools] = await Promise.all([ + listToolsThroughTransport(full), + listToolsThroughTransport(compact), + ]); + const detailGuidance = (tools: readonly AdvertisedTool[]) => + Object.fromEntries( + tools.flatMap(({ name, inputSchema }) => { + const detail = ( + inputSchema.properties as Record> + )?.detail; + return typeof detail?.description === "string" + ? [[name, detail.description]] + : []; + }), + ); + const expected = detailGuidance(fullTools); + expect(Object.keys(expected)).toEqual( + expect.arrayContaining(["inspect_artifact", "inspect_macho"]), + ); + expect(detailGuidance(compactTools)).toEqual(expected); +}); From 4910e149ad9b2d8e3c3d0be62332b633ac418889 Mon Sep 17 00:00:00 2001 From: Beqa <91750923+BeqaKs@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:22:36 +0400 Subject: [PATCH 24/94] docs(setup): describe FreeBuff as published setup support (#7246) Setup against npm latest already accepts --client freebuff. Replace the stale proposed-integration wording while keeping the unverified native client loading caveat. --- docs/client-configuration.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/client-configuration.md b/docs/client-configuration.md index 87f55ed530..54b5e27493 100644 --- a/docs/client-configuration.md +++ b/docs/client-configuration.md @@ -632,21 +632,20 @@ writing a registration ZCode cannot load. ## FreeBuff -For FreeBuff, the proposed integration writes `mcpServers.rea` into the user-scope -`~/.agents/mcp.json` shared-agent file. Entries carry `command`, `args`, and -`env`; stdio is the default transport. The proposed personal skill location is -`~/.agents/skills`, so setup installs its skill copy there. Because the same +For FreeBuff, `rea setup --client freebuff` writes `mcpServers.rea` into the +user-scope `~/.agents/mcp.json` shared-agent file. Entries carry `command`, +`args`, and `env`; stdio is the default transport. The personal skill location +is `~/.agents/skills`, so setup installs its skill copy there. Because the same file doubles as ZCode's user-scope fallback (see above), a `freebuff` -registration is also in the file ZCode uses until ZCode defines its own native server. -Setup and doctor detect FreeBuff from its own profile directory -(`~/.config/freebuff-desktop`), not from the shared `.agents` folder that -REA's skill installs create for every client. +registration is also in the file ZCode uses until ZCode defines its own native +server. Setup and doctor detect FreeBuff from its own profile directory +(`~/.config/freebuff-desktop`), not from the shared `.agents` folder that REA's +skill installs create for every client. -The configuration write and personal skill locations are proposed integration -boundaries. FreeBuff MCP loading and skill discovery have not been verified -with an installed native FreeBuff client/version; filesystem tests cover REA's -editing, discovery, and ownership behavior only. Automatic applicability -requires the FreeBuff profile directory even when the shared MCP file exists. +FreeBuff MCP loading and skill discovery have not been verified with an +installed native FreeBuff client/version; filesystem tests cover REA's editing, +discovery, and ownership behavior only. Automatic applicability requires the +FreeBuff profile directory even when the shared MCP file exists. When uninstall cannot establish FreeBuff or another shared-storage reader from product discovery, a malformed or unreadable shared MCP file is retained unchanged and does not stop other REA removals, including `--purge-data`. When @@ -654,8 +653,9 @@ FreeBuff or ZCode product applicability is established or unresolved, the same malformed shared file stops uninstall before removals. See [Diagnose, update, and remove](installation.md#diagnose-update-and-remove). -FreeBuff setup support requires `rea-agents@6.8.0` or later. Older versions -reject `rea setup --client freebuff` as an unsupported client. See the +FreeBuff setup support requires `rea-agents@6.8.0` or later (including the +current npm `latest` release). Older versions reject +`rea setup --client freebuff` as an unsupported client. See the [installation guide](installation.md#supported-agents) for client availability. ## DeepSeek Harness From 703c11c90fb5a24dd9f511b1ad5eede336df6885 Mon Sep 17 00:00:00 2001 From: Tawhidul Islam Date: Wed, 4 Nov 2026 17:23:52 +0600 Subject: [PATCH 25/94] fix(javascript): recover configuration file reads from fs/promises (#8446) * fix(javascript): recover configuration file reads from fs/promises * fix(javascript): disallow nonexistent readFileSync on fs/promises --------- Co-authored-by: Tawhidul Islam <276017696+FhDBro@users.noreply.github.com> --- ...tSemanticAnalysis.builtin_defaults.test.ts | 55 +++++++++++++++++++ .../javascriptSemanticDataEffects.ts | 15 +++-- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/src/domain/javascript/javascriptSemanticAnalysis.builtin_defaults.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.builtin_defaults.test.ts index 5adc82e75a..38741665e0 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.builtin_defaults.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.builtin_defaults.test.ts @@ -677,3 +677,58 @@ it("retains an earlier captured function after its object method is replaced", ( { kind: "acquire", method: "open" }, ]); }); + +it.each([ + "import { readFile } from 'node:fs/promises'; readFile('config.json');", + "import fs from 'node:fs/promises'; fs.readFile('config.json');", + "import fs from 'node:fs'; fs.promises.readFile('config.json');", + "const { readFile } = require('fs/promises'); readFile('config.json');", + "const fsp = require('node:fs/promises'); fsp.readFile('config.json');", + "const fs = require('node:fs'); fs.promises.readFile('config.json');", +])( + "recovers configuration file reads from promise-based fs APIs: %s", + (source) => { + const ir = analyzeCommonJsSemanticFixture(source); + expect(ir.configurationOperations).toMatchObject([ + { + kind: "file", + key: "config.json", + resolution: "complete", + }, + ]); + }, +); + +it("recovers partial configuration file reads from promise-based fs APIs with dynamic path", () => { + const ir = analyzeCommonJsSemanticFixture( + "import { readFile } from 'node:fs/promises'; readFile(getConfigPath());", + ); + expect(ir.configurationOperations).toMatchObject([ + { + kind: "file", + key: null, + resolution: "partial", + }, + ]); +}); + +it("invalidates promise-based configuration file reads after mutation", () => { + const ir = analyzeCommonJsSemanticFixture( + "const fs = require('fs'); fs.promises.readFile = custom; const fsp = require('fs/promises'); fsp.readFile('wrong');", + ); + expect(ir.configurationOperations).toEqual([]); +}); + +it.each([ + "const { readFileSync } = require('fs/promises'); readFileSync('config.json');", + "const { readFileSync } = require('node:fs/promises'); readFileSync('config.json');", + "import { readFileSync } from 'node:fs/promises'; readFileSync('config.json');", + "import fs from 'node:fs/promises'; fs.readFileSync('config.json');", + "const fs = require('node:fs'); fs.promises.readFileSync('config.json');", +])( + "does not treat nonexistent readFileSync on promise-based fs as configuration read: %s", + (source) => { + const ir = analyzeCommonJsSemanticFixture(source); + expect(ir.configurationOperations).toEqual([]); + }, +); diff --git a/src/domain/javascript/javascriptSemanticDataEffects.ts b/src/domain/javascript/javascriptSemanticDataEffects.ts index f2bfa6ee88..7e8e8163b3 100644 --- a/src/domain/javascript/javascriptSemanticDataEffects.ts +++ b/src/domain/javascript/javascriptSemanticDataEffects.ts @@ -123,11 +123,16 @@ const configurationDetails = ( } if (!t.isCallExpression(node) && !t.isOptionalCallExpression(node)) return null; - const fileMethod = builtinMethod(node.callee, context.state, [ - "fs", - "node:fs", - ]); - return fileMethod === "readFile" || fileMethod === "readFileSync" + const fsMethod = builtinMethod(node.callee, context.state, ["fs", "node:fs"]); + const isFileRead = + fsMethod === "readFile" || + fsMethod === "readFileSync" || + (fsMethod === null && + builtinMethod(node.callee, context.state, [ + "fs/promises", + "node:fs/promises", + ]) === "readFile"); + return isFileRead ? { kind: "file", key: literalString(node.arguments[0]), From 810cebcba07d1a793b8657a7f3a5c14fe5995206 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:28:15 +0800 Subject: [PATCH 26/94] fix(ghidra): explain read-only project edit refusals (#8413) * fix(ghidra): explain read-only project edit refusals * test: follow provider guidance ownership --- src/application/AnalysisProvider.ts | 4 +- .../binary/BinarySessionExecution.ts | 8 +++ src/ghidra/GhidraProviderCapabilities.ts | 4 ++ .../boundary/mcp/providerSelectionMcp.test.ts | 65 +++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) diff --git a/src/application/AnalysisProvider.ts b/src/application/AnalysisProvider.ts index 74d41d2d81..6940e3c421 100644 --- a/src/application/AnalysisProvider.ts +++ b/src/application/AnalysisProvider.ts @@ -258,8 +258,10 @@ type CapabilityAvailability = readonly available: false; readonly reason: string; readonly availabilityCode?: ProviderRejectionCode; + /** Provider-owned explanation and recovery for an attempted operation. */ + readonly userMessage?: string; /** - * Repair instruction from the readiness owner that made this operation + * Recovery instruction from the owner that made this operation * unavailable. Static capability facts omit it. */ readonly remediation?: string; diff --git a/src/application/binary/BinarySessionExecution.ts b/src/application/binary/BinarySessionExecution.ts index 37c1f2551b..20d4dd063d 100644 --- a/src/application/binary/BinarySessionExecution.ts +++ b/src/application/binary/BinarySessionExecution.ts @@ -76,6 +76,14 @@ export const prepareSessionExecution = ( active.route.binding?.identity.id ?? active.route.identity.id, operation, capability?.reason ?? "operation is not declared by this provider", + { + ...(capability?.userMessage === undefined + ? {} + : { userMessage: capability.userMessage }), + ...(capability?.remediation === undefined + ? {} + : { remediationAction: capability.remediation }), + }, ), ); } diff --git a/src/ghidra/GhidraProviderCapabilities.ts b/src/ghidra/GhidraProviderCapabilities.ts index cccfefe7bb..4c57f3264b 100644 --- a/src/ghidra/GhidraProviderCapabilities.ts +++ b/src/ghidra/GhidraProviderCapabilities.ts @@ -276,6 +276,10 @@ export const existingProjectCapabilities = ( available: false, reason: "Existing Ghidra project sessions are read-only.", availabilityCode: "open_options_invalid", + userMessage: + "This Ghidra project session is read-only; the requested edit was not applied.", + remediation: + "Continue inspecting the selected project. To make session-only edits, start a separate REA session with REA_GHIDRA_PROJECT_PATH and REA_GHIDRA_PROGRAM_NAME unset, then open a fresh Ghidra import. It is a separate ephemeral analysis and does not preserve the selected project's state.", }; return { ...capability, limitations }; }); diff --git a/tests/boundary/mcp/providerSelectionMcp.test.ts b/tests/boundary/mcp/providerSelectionMcp.test.ts index a821fb5f30..7faed129b8 100644 --- a/tests/boundary/mcp/providerSelectionMcp.test.ts +++ b/tests/boundary/mcp/providerSelectionMcp.test.ts @@ -28,6 +28,7 @@ import { err, ok } from "../../../src/domain/result.js"; import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js"; import { ProviderAdapterError } from "../../../src/domain/providerAdapterError.js"; import { ProviderCleanupError } from "../../../src/domain/providerCleanupError.js"; +import { existingProjectCapabilities } from "../../../src/ghidra/GhidraProviderCapabilities.js"; import { silentLogger } from "../../../src/logger.js"; import { createServer } from "../../../src/server/createServer.js"; import { BINARY_SESSION_SUMMARY_OMITTED_FIELDS } from "../../../src/contracts/sessionStatusContract.js"; @@ -782,6 +783,70 @@ const structured = (result: CallToolResult): Record => ? parseMcpToolError(result) : z.record(z.string(), z.unknown()).parse(result.structuredContent); +it("preserves a selected Ghidra project's read-only guidance when an edit is requested", async () => { + const directory = await createTestTempDirectory("rea-project-refusal-mcp-"); + const target = join(directory, "fixture.hop"); + await writeFile(target, "fixture"); + const starts: string[] = []; + const base = candidate("ghidra", starts); + const projectCapabilities = existingProjectCapabilities("linux"); + const annotation = projectCapabilities.find( + (entry) => entry.operation === "annotate_native_function", + ); + if (annotation === undefined || annotation.available) + throw new Error("Expected read-only annotation capability"); + expect(annotation.userMessage).toBeDefined(); + expect(annotation.remediation).toBeDefined(); + const provider = { + ...base, + capabilities: () => + projectCapabilities.map((entry) => ({ + ...entry, + provider: base.identity(), + })), + }; + const session = composeBinarySession( + new AnalysisProviderRegistry([provider]), + ); + const server = createServer( + { kind: "session", session }, + { logger: silentLogger }, + ); + const mcp = new Client({ name: "project-refusal", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + resources.push(mcp, server, { close: () => session.close() }); + await server.connect(serverTransport); + await mcp.connect(clientTransport); + expect( + (await mcp.callTool({ name: "open_binary", arguments: { path: target } })) + .isError, + ).not.toBe(true); + + const refused = await mcp.callTool({ + name: "annotate_native_function", + arguments: { procedure: "fixture", comment: "must not be applied" }, + }); + expect(refused.isError).toBe(true); + expect(parseMcpToolError(refused)).toMatchObject({ + error: { + code: "capability_unavailable", + message: annotation.userMessage, + details: { reason: "Existing Ghidra project sessions are read-only." }, + remediation: { action: annotation.remediation }, + }, + }); + const status = await sessionStatusOf(mcp, { detail: "complete" }); + const availability = status.parsed.tool_availability.groups.find((group) => + group.names.includes("annotate_native_function"), + ); + expect(availability).toMatchObject({ + available: false, + remediation: annotation.remediation, + }); + expect(starts).toEqual([]); +}); + it("retains attributed mixed profile cleanup through registered MCP calls and partial retry", async () => { directory = await createTestTempDirectory("rea-profile-completion-mcp-"); const path = join(directory, "fixture.hop"); From d857b61bc56cf23b4081095078b095d31d8bf4bb Mon Sep 17 00:00:00 2001 From: Dragolone <160818051+Dragolone@users.noreply.github.com> Date: Wed, 4 Nov 2026 22:28:59 +1100 Subject: [PATCH 27/94] fix(android): ignore action-less filters in activity export defaults (#8030) Android's package parser discards an activity, activity-alias or receiver intent filter that declares no action, so such a filter neither makes the component exported by default nor triggers the API 31 explicit-export requirement. Services keep action-less filters. The manifest inventory counted every declared filter, reporting exported_default true (or an explicit-export validation error) for components Android does not export. intent_filter_count still reports every declared filter. --- docs/android-analysis.md | 3 ++ .../AndroidManifestComponentDetails.ts | 18 +++++++ src/android/AndroidManifestComponents.ts | 13 +++-- .../android/androidManifestComponents.test.ts | 54 +++++++++++++++++++ 4 files changed, 85 insertions(+), 3 deletions(-) diff --git a/docs/android-analysis.md b/docs/android-analysis.md index 4f42dd2fc0..932e5f4fa8 100644 --- a/docs/android-analysis.md +++ b/docs/android-analysis.md @@ -73,6 +73,9 @@ spellings; the original SDK declaration strings remain unchanged. it remains null for unknown provider target SDKs, nonliteral declarations, or components with filters targeting API 31 or later that omit the now-required explicit declaration. A limitation records the latter validation error. +As in Android's package parser, an activity, alias or receiver filter without +an `` is discarded and does not count toward either rule; a service +keeps such a filter. `intent_filter_count` still counts every declared filter. Provider authorities and all inventory dimensions are bounded. `status: "partial"` and `limitations` identify omitted declarations or values; a capacity limitation appears only when an actual collection limit was reached. diff --git a/src/android/AndroidManifestComponentDetails.ts b/src/android/AndroidManifestComponentDetails.ts index a3f1d7ebdc..3f5ecdbf5d 100644 --- a/src/android/AndroidManifestComponentDetails.ts +++ b/src/android/AndroidManifestComponentDetails.ts @@ -32,6 +32,20 @@ const projectedDataAttributes = new Set([ const literalBoolean = (value: string | null): boolean | null => value === "true" ? true : value === "false" ? false : null; +/** Whether a filter names an action, which Android requires to keep an activity filter. */ +const declaresAction = (filter: Element): boolean => { + for (let index = 0; index < filter.childNodes.length; index += 1) { + const member = filter.childNodes.item(index); + if ( + isManifestElement(member) && + member.nodeName === "action" && + androidAttribute(member, "name") !== null + ) + return true; + } + return false; +}; + /** One decoded-manifest lifetime owns nested declarations and cumulative copy budgets. */ export const createAndroidManifestDetailsReader = ( source: string, @@ -99,6 +113,7 @@ export const createAndroidManifestDetailsReader = ( let legacyValuesRemaining = limits.intentValues; let legacyTruncated = false; let filterCount = 0; + let actionFilter = false; for (let index = 0; index < component.childNodes.length; index += 1) { const child = component.childNodes.item(index); if (!isManifestElement(child)) continue; @@ -111,6 +126,8 @@ export const createAndroidManifestDetailsReader = ( continue; } if (child.nodeName !== "intent-filter") continue; + // Checked past the filter bound too; one action-bearing filter suffices. + actionFilter ||= declaresAction(child); const filterIndex = filterCount++; if (filterIndex >= limits.intentFilters) { partial = true; @@ -237,6 +254,7 @@ export const createAndroidManifestDetailsReader = ( actions, categories, filterCount, + actionFilter, legacyTruncated, partial, detailLimited, diff --git a/src/android/AndroidManifestComponents.ts b/src/android/AndroidManifestComponents.ts index 92fbc7be63..03f63f3d46 100644 --- a/src/android/AndroidManifestComponents.ts +++ b/src/android/AndroidManifestComponents.ts @@ -454,11 +454,18 @@ export const parseAndroidManifestComponents = ( : null; const exportedNonliteral = declaredExported !== null && exportedLiteral === null; + // Android discards an activity, alias or receiver filter that declares + // no action, while a service keeps it. Only kept filters select the + // export default and the API 31 explicit-export requirement. + const keptFilters = + component.nodeName === "service" + ? details.filterCount > 0 + : details.actionFilter; const explicitExportRequired = declaredExported === null && targetSdkLevel !== null && targetSdkLevel >= 31 && - details.filterCount > 0 && + keptFilters && (component.nodeName === "activity" || component.nodeName === "activity-alias" || component.nodeName === "service" || @@ -466,7 +473,7 @@ export const parseAndroidManifestComponents = ( const filteredExportUnknown = declaredExported === null && targetSdkLevel === null && - details.filterCount > 0 && + keptFilters && (component.nodeName === "activity" || component.nodeName === "activity-alias" || component.nodeName === "service" || @@ -491,7 +498,7 @@ export const parseAndroidManifestComponents = ( ? null : component.nodeName === "provider" ? providerDefault - : details.filterCount > 0, + : keptFilters, permission, permission_source: effectivePermissionSource, intent_filter_count: details.filterCount, diff --git a/tests/boundary/android/androidManifestComponents.test.ts b/tests/boundary/android/androidManifestComponents.test.ts index 9d5f64257f..618169cfdd 100644 --- a/tests/boundary/android/androidManifestComponents.test.ts +++ b/tests/boundary/android/androidManifestComponents.test.ts @@ -689,6 +689,60 @@ it("leaves target-31-or-later filtered components without an inferred export def ).toMatchObject({ exported_default: null }); }); +it("ignores action-less activity, alias and receiver filters for export defaults", () => { + // Android's package parser drops these filters; a service keeps them. + const manifest = [ + '', + '', + '', + '', + '', + '', + "", + ].join(""); + const defaults = (sdk: string) => + parseAndroidManifestComponents(manifest, "complete", sdk).items.map( + (item) => [item.intent_filter_count, item.exported_default], + ); + expect(defaults("30")).toEqual([ + [1, false], + [1, false], + [1, false], + [1, true], + [2, true], + ]); + const modern = parseAndroidManifestComponents(manifest, "complete", "31"); + expect(modern.items.map((item) => item.exported_default)).toEqual([ + false, + false, + false, + null, + null, + ]); + expect(modern.limitations.join(" ")).toContain( + "2 component(s) with intent filters target API 31", + ); + expect(defaults("Preview").map(([, value]) => value)).toEqual([ + false, + false, + false, + null, + null, + ]); + const filters = "".repeat( + ANDROID_MANIFEST_LIMITS.intentFilters, + ); + const late = parseAndroidManifestComponents( + `${filters}`, + "complete", + "30", + ); + expect(late.items[0]).toMatchObject({ + intent_filter_count: ANDROID_MANIFEST_LIMITS.intentFilters + 1, + exported_default: true, + }); +}); + it("ignores same-local-name attributes from other namespaces", () => { const manifest = [ '', From d0d31fcf21e43761e251922c99552c3b2e740222 Mon Sep 17 00:00:00 2001 From: highmachs Date: Wed, 4 Nov 2026 16:59:44 +0530 Subject: [PATCH 28/94] fix(browser): accept 3-digit HTTP response status codes up to 999 (#8075) --- src/browser/PlaywrightScenarioEvents.test.ts | 58 ++++++++++++++++++++ src/domain/browserScenarioCapture.test.ts | 27 +++++++++ src/domain/browserScenarioCaptureValues.ts | 2 +- src/domain/browserScenarioValues.ts | 2 +- 4 files changed, 87 insertions(+), 2 deletions(-) diff --git a/src/browser/PlaywrightScenarioEvents.test.ts b/src/browser/PlaywrightScenarioEvents.test.ts index a139c96f39..659b6e1498 100644 --- a/src/browser/PlaywrightScenarioEvents.test.ts +++ b/src/browser/PlaywrightScenarioEvents.test.ts @@ -283,6 +283,64 @@ describe("PlaywrightScenarioEvents", () => { }); }); +it("retains network response events with HTTP status codes above 599 without error", () => { + const contextListeners = new Map void>(); + const context = { + on: (name: string, listener: (value: unknown) => void) => { + contextListeners.set(name, listener); + }, + off: () => {}, + } as unknown as BrowserContext; + const page = { + url: () => "https://app.example.test/", + context: () => context, + on: () => {}, + } as unknown as Page; + const scenario = browserScenarioSchema.parse({ + browser: { mode: "launch", executable_path: "/opt/chromium" }, + start_url: { url: "https://app.example.test/" }, + actions: [{ step_id: "wait", action: "wait_for_timeout", duration_ms: 1 }], + capture: { events: ["network"] }, + }); + const secrets = resolveScenarioSecrets(scenario, {}); + const events = new PlaywrightScenarioEvents({ + page, + context, + enabled: new Set(["network"]), + secrets, + }); + const request = { + url: () => "https://app.example.test/api/check", + method: () => "GET", + resourceType: () => "xhr", + postDataBuffer: () => null, + redirectedFrom: () => null, + failure: () => null, + headers: () => ({}), + headersArray: async () => [], + frame: () => ({ page: () => page }), + } as unknown as Request; + const response = { + request: () => request, + url: () => request.url(), + status: () => 999, + headers: () => ({ "content-type": "text/plain" }), + headersArray: async () => [{ name: "content-type", value: "text/plain" }], + body: async () => Buffer.from("denied"), + } as unknown as Response; + + contextListeners.get("request")?.(request); + expect(() => contextListeners.get("response")?.(response)).not.toThrow(); + const items = events.result().items; + const responseEvent = items.find((e) => e.kind === "response"); + expect(responseEvent).toBeDefined(); + expect(responseEvent).toMatchObject({ + kind: "response", + status: 999, + url: { url: "https://app.example.test/api/check" }, + }); +}); + it("retains ordinary event volumes and reports event-count exhaustion", () => { const listeners = new Map void)[]>(); const page = { diff --git a/src/domain/browserScenarioCapture.test.ts b/src/domain/browserScenarioCapture.test.ts index fa1dcb743b..293b7003cc 100644 --- a/src/domain/browserScenarioCapture.test.ts +++ b/src/domain/browserScenarioCapture.test.ts @@ -70,6 +70,33 @@ it("parses each network outcome without contradictory status or failure data", ( ).toBe(false); }); +it("accepts three-digit HTTP response status codes including statuses above 599", () => { + expect( + browserScenarioEventSchema.safeParse({ + ...networkEvent, + kind: "response", + status: 999, + failure: null, + }).success, + ).toBe(true); + expect( + browserScenarioEventSchema.safeParse({ + ...networkEvent, + kind: "response", + status: 1000, + failure: null, + }).success, + ).toBe(false); + expect( + browserScenarioEventSchema.safeParse({ + ...networkEvent, + kind: "response", + status: 99, + failure: null, + }).success, + ).toBe(false); +}); + it("keeps WebSocket payload representation tied to its payload type", () => { const frame = { sequence: 1, diff --git a/src/domain/browserScenarioCaptureValues.ts b/src/domain/browserScenarioCaptureValues.ts index 04021df1ca..436c69e53d 100644 --- a/src/domain/browserScenarioCaptureValues.ts +++ b/src/domain/browserScenarioCaptureValues.ts @@ -210,7 +210,7 @@ export const browserScenarioEventSchema = z z.strictObject({ ...networkEventShape, kind: z.literal("response"), - status: z.int().min(100).max(599), + status: z.int().min(100).max(999), failure: z.null(), }), z.strictObject({ diff --git a/src/domain/browserScenarioValues.ts b/src/domain/browserScenarioValues.ts index bf42e54f6c..e6edf5fbfc 100644 --- a/src/domain/browserScenarioValues.ts +++ b/src/domain/browserScenarioValues.ts @@ -437,7 +437,7 @@ export const browserScenarioActionSchema = z.discriminatedUnion("action", [ z .int() .min(100) - .max(599) + .max(999) .refine((status) => status < 300 || status > 399, { message: "Request steps follow no redirects, so a 3xx status cannot complete one", From 1861059f8b37dadd9a84f716fef67611750c6422 Mon Sep 17 00:00:00 2001 From: Adolanium <94890352+Adolanium@users.noreply.github.com> Date: Wed, 4 Nov 2026 13:30:31 +0200 Subject: [PATCH 29/94] fix(wasm): name the compare_wasm_interfaces side of rejected Evidence (#7302) * fix(wasm): name the compare_wasm_interfaces side of rejected Evidence compare_wasm_interfaces parsed both sides inside one try and reported any rejection, such as tampered content, a different operation or a rebound subject, as an input issue at path [], which names no argument. Each side is now parsed separately and a rejection is reported at [left] or [right], as the function, native and managed comparers already do. * test(wasm): cover both comparison sides through the MCP boundary --- .../WasmInterfaceComparisonService.test.ts | 34 +++++++++++++ .../wasm/WasmInterfaceComparisonService.ts | 16 ++++-- .../derivedOperationFailures.test.ts | 51 +++++++++++++++++++ 3 files changed, 96 insertions(+), 5 deletions(-) diff --git a/src/application/wasm/WasmInterfaceComparisonService.test.ts b/src/application/wasm/WasmInterfaceComparisonService.test.ts index 14cd133185..138f9de6ed 100644 --- a/src/application/wasm/WasmInterfaceComparisonService.test.ts +++ b/src/application/wasm/WasmInterfaceComparisonService.test.ts @@ -168,6 +168,40 @@ it("requires the producer's file subject format even for authentic Evidence", as ).toMatchObject({ ok: false, error: { _tag: "AnalysisInputError" } }); }); +it.each(["left", "right"] as const)( + "names the %s argument whose Evidence is rejected", + async (side) => { + const tampered = JSON.parse(JSON.stringify(evidence(facts("f")))); + tampered.normalized_result.interface.entries[0].name = "tampered"; + const report = wasmArtifactFixture(); + const otherSelection = createEvidence( + { + path: report.artifact.path, + sha256: report.artifact.sha256, + format: "file", + }, + { id: "fixture", name: "typed fixture", version: "1" }, + { + operation: "inspect_wasm_artifact", + parameters: { path: "/artifacts/other.wasm" }, + result: report, + confidence: "observed", + }, + ); + for (const rejected of [tampered, otherSelection]) + expect( + await compareWasmInterfaces({ + left: side === "left" ? rejected : evidence(facts("f")), + right: side === "right" ? rejected : evidence(facts("f")), + scope: "both", + }), + ).toMatchObject({ + ok: false, + error: { issues: [{ path: [side], reason: "invalid_value" }] }, + }); + }, +); + it("resolves exact retained references and permits cancellation between comparison batches", async () => { const left = evidence(facts("f")); const right = evidence(facts("f", "i64")); diff --git a/src/application/wasm/WasmInterfaceComparisonService.ts b/src/application/wasm/WasmInterfaceComparisonService.ts index a71b6e5491..d7689cdbe1 100644 --- a/src/application/wasm/WasmInterfaceComparisonService.ts +++ b/src/application/wasm/WasmInterfaceComparisonService.ts @@ -40,16 +40,19 @@ import { const OPERATION = "compare_wasm_interfaces"; const parseInspection = ( raw: unknown, + side: "left" | "right", ): Result<{ evidence: Evidence; result: WasmArtifact }, AnalysisInputError> => { const authenticated = safeParseEvidence(raw); - if (!authenticated.ok) return inspectionInputFailure(authenticated.error); + if (!authenticated.ok) + return inspectionInputFailure(authenticated.error, side); const evidence = authenticated.value; if (evidence.operation !== "inspect_wasm_artifact") return inspectionInputFailure( new TypeError("Expected complete inspect_wasm_artifact Evidence"), + side, ); const parsed = wasmArtifactSchema.safeParse(evidence.normalized_result); - if (!parsed.success) return inspectionInputFailure(parsed.error); + if (!parsed.success) return inspectionInputFailure(parsed.error, side); const result = parsed.data; if ( !wasmArtifactMatchesSubject( @@ -67,16 +70,19 @@ const parseInspection = ( new TypeError( "WASM Evidence subject/selection does not match normalized artifact identity", ), + side, ); return ok({ evidence, result }); }; +/** Attribute a rejected side to the input argument that supplied it. */ const inspectionInputFailure = ( cause: Error, + side: "left" | "right", ): Result => err( new AnalysisInputError(OPERATION, { cause }, [ - { path: [], reason: "invalid_value", message: cause.message }, + { path: [side], reason: "invalid_value", message: cause.message }, ]), ); @@ -107,9 +113,9 @@ export const compareWasmInterfaces = async ( try { await setImmediate(); assertActive(); - const parsedLeft = parseInspection(resolved.value.input.left); + const parsedLeft = parseInspection(resolved.value.input.left, "left"); if (!parsedLeft.ok) return parsedLeft; - const parsedRight = parseInspection(resolved.value.input.right); + const parsedRight = parseInspection(resolved.value.input.right, "right"); if (!parsedRight.ok) return parsedRight; const left = parsedLeft.value; const right = parsedRight.value; diff --git a/tests/process-global/derivedOperationFailures.test.ts b/tests/process-global/derivedOperationFailures.test.ts index 3312d4ffd1..c12e751bdd 100644 --- a/tests/process-global/derivedOperationFailures.test.ts +++ b/tests/process-global/derivedOperationFailures.test.ts @@ -33,6 +33,7 @@ import { OwnedFileHandle } from "../../src/filesystem/OwnedFileHandle.js"; import { primaryAnalysisError } from "../../src/domain/analysisErrorBase.js"; import { buildManagedPeFixture } from "../../src/dotnet/ManagedPe.fixture.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { wasmArtifactFixture } from "../fixtures/wasm/artifact.js"; import { ProcessCaptureError } from "../../src/process/capture/ProcessCaptureError.js"; import { reconstructionObligationLedgerSchema } from "../../src/domain/reconstructionObligationLedgerSchemas.js"; @@ -919,3 +920,53 @@ describe("native matching failure ownership through the public SDK", () => { }, ); }); + +describe("compare_wasm_interfaces Evidence attribution", () => { + const inspection = (selectedPath?: string) => { + const report = wasmArtifactFixture(); + return createEvidence( + { + path: report.artifact.path, + sha256: report.artifact.sha256, + format: "file", + }, + { id: "typed-fixture", name: "Typed fixture", version: "1" }, + { + operation: "inspect_wasm_artifact", + parameters: { path: selectedPath ?? report.artifact.path }, + result: report, + confidence: "observed", + }, + ); + }; + + it.each(["left", "right"] as const)( + "names the %s argument whose authentic Evidence selects another artifact", + async (side) => { + await withSdk(async ({ client, session }) => { + const rejected = inspection("/artifacts/other.wasm"); + const response = await client.callTool({ + name: "compare_wasm_interfaces", + arguments: { + left: side === "left" ? rejected : inspection(), + right: side === "right" ? rejected : inspection(), + scope: "both", + }, + }); + + expect(parseMcpToolError(response).error).toMatchObject({ + code: "invalid_request", + details: { + issues: [ + expect.objectContaining({ + path: [side], + reason: "invalid_value", + }), + ], + }, + }); + expect(session.exportEvidenceBundle().records).toEqual([]); + }); + }, + ); +}); From 1f8084c7db806354ebe571e1490264b128ef92c2 Mon Sep 17 00:00:00 2001 From: Dragolone <160818051+Dragolone@users.noreply.github.com> Date: Wed, 4 Nov 2026 22:31:14 +1100 Subject: [PATCH 30/94] fix(node-sea): bound ELF note scans by the notes a carrier holds (#8031) Artifact classification probes every ELF root for a Node SEA note. Each PT_NOTE program header was scanned on its own, one read per note, so headers that declare the same note bytes multiplied the work: a 295 KB ELF with 4,096 such headers kept inspect-artifact busy for about three and a half minutes. Every note header takes at least 12 bytes, so disjoint tables hold at most one note per 12 file bytes. The scan now stops there. Past that count the remaining tables are unreadable: an ELF without a confirmed blob keeps its native classification, and a confirmed blob fails closed. --- docs/node-sea.md | 8 ++- src/artifacts/NodeSeaArtifactReader.test.ts | 66 ++++++++++++++++++++- src/artifacts/nodeSea/NodeSeaBlob.ts | 2 + src/artifacts/nodeSea/NodeSeaLocation.ts | 27 +++++++-- 4 files changed, 96 insertions(+), 7 deletions(-) diff --git a/docs/node-sea.md b/docs/node-sea.md index 33e9188845..85483d8fff 100644 --- a/docs/node-sea.md +++ b/docs/node-sea.md @@ -82,8 +82,12 @@ override. Asset and argument decoding, padding scans, and inventory construction yield in bounded batches and honor caller cancellation. Asset count and name budgets bound retained metadata independently of stored member body sizes. The PE name probe reads headers from the first 64 KiB of the carrier; a PE whose -headers extend beyond that is left to native classification. Snapshot and -code-cache bytes remain opaque. +headers extend beyond that is left to native classification. ELF note tables +are read for at most one note per 12 carrier bytes in total, the most that +disjoint tables can hold, so program headers that repeat the same notes do not +rescan them once per header. Past that count the remaining tables are +unreadable: an ELF without a confirmed blob keeps its native classification, +and one with a blob fails closed. Snapshot and code-cache bytes remain opaque. ## Verification diff --git a/src/artifacts/NodeSeaArtifactReader.test.ts b/src/artifacts/NodeSeaArtifactReader.test.ts index 21984a69f2..0e6f78d465 100644 --- a/src/artifacts/NodeSeaArtifactReader.test.ts +++ b/src/artifacts/NodeSeaArtifactReader.test.ts @@ -17,6 +17,7 @@ import { extractArtifact } from "./extraction/ArtifactExtraction.js"; import { NODE_SEA_BAD_MAGIC, NODE_SEA_DUPLICATE_BLOB, + NODE_SEA_ELF_NOTES_REPEATED, NODE_SEA_FUSE_CONFLICT, NODE_SEA_FUSE_ABSENT, NODE_SEA_FUSE_UNSET, @@ -32,7 +33,7 @@ import { nodeSeaPe, nodeSeaUniversal, } from "./nodeSea/NodeSea.fixture.js"; -import { elfNoteFixture } from "../native/elf/ElfNotes.fixture.js"; +import { elfNote, elfNoteFixture } from "../native/elf/ElfNotes.fixture.js"; import { peResourceFixture } from "../native/pe/PeResources.fixture.js"; import { locateNodeSea } from "./nodeSea/NodeSeaLocation.js"; import { MAX_INSPECT_PE_FILE_BYTES } from "../domain/native/peResources.js"; @@ -413,6 +414,69 @@ it("keeps one blob when the same bytes are declared twice", async () => { ).toHaveLength(1); }); +/** ELF64 whose `count` PT_NOTE program headers all declare the same notes. */ +const repeatedNoteElf = (count: number, notes: Buffer): Buffer => { + const noteOffset = 64 + count * 56; + const bytes = Buffer.alloc(noteOffset + notes.length); + bytes.writeUInt32LE(0x464c457f, 0); + bytes[4] = 2; + bytes[5] = 1; + bytes[6] = 1; + bytes.writeUInt16LE(3, 0x10); + bytes.writeUInt16LE(62, 0x12); + bytes.writeUInt32LE(1, 0x14); + bytes.writeBigUInt64LE(64n, 0x20); + bytes.writeUInt16LE(64, 0x34); + bytes.writeUInt16LE(56, 0x36); + bytes.writeUInt16LE(count, 0x38); + bytes.writeUInt16LE(64, 0x3a); + for (let index = 0; index < count; index += 1) { + const at = 64 + index * 56; + bytes.writeUInt32LE(4, at); + bytes.writeBigUInt64LE(BigInt(noteOffset), at + 8); + bytes.writeBigUInt64LE(BigInt(notes.length), at + 32); + bytes.writeBigUInt64LE(BigInt(notes.length), at + 40); + bytes.writeBigUInt64LE(4n, at + 48); + } + notes.copy(bytes, noteOffset); + return bytes; +}; + +it("reads repeated PT_NOTE declarations at most once per note the file holds", async () => { + // 4 KiB of zero bytes is 341 empty 12-byte notes. + const empty = Buffer.alloc(4096); + const unconfirmed = repeatedNoteElf(64, empty); + const handle = await open( + await writeCarrier("repeated-notes", unconfirmed), + "r", + ); + const read = vi.spyOn(handle, "read"); + try { + await expect( + locateNodeSea(handle, unconfirmed.length), + ).resolves.toBeUndefined(); + // The prefix, the ELF header and 64 program headers, then the notes. + expect(read.mock.calls.length).toBeLessThanOrEqual( + 2 + 64 + Math.floor(unconfirmed.length / 12), + ); + } finally { + await handle.close(); + } + const blob = buildNodeSeaBlob({ main: script, codePath: "sea.js" }); + const confirmed = repeatedNoteElf( + 64, + Buffer.concat([elfNote("NODE_SEA_BLOB", 0, blob), empty]), + ); + await expect( + scanCanonicalArtifactInventory( + await writeCarrier("repeated-sea-notes", confirmed), + ), + ).rejects.toMatchObject({ + reason: "format", + message: NODE_SEA_ELF_NOTES_REPEATED, + }); +}); + it("extracts the main script and assets without executing them", async () => { const directory = await createTestTempDirectory("rea-node-sea-extract-"); const path = join(directory, "sea.elf"); diff --git a/src/artifacts/nodeSea/NodeSeaBlob.ts b/src/artifacts/nodeSea/NodeSeaBlob.ts index 4c8cd2d0ca..e63da49fd9 100644 --- a/src/artifacts/nodeSea/NodeSeaBlob.ts +++ b/src/artifacts/nodeSea/NodeSeaBlob.ts @@ -43,6 +43,8 @@ export const NODE_SEA_ELF_DESCRIPTOR_TRUNCATED = "Node SEA ELF note descriptor is truncated"; export const NODE_SEA_ELF_TABLE_TRUNCATED = "Node SEA ELF note table is truncated"; +export const NODE_SEA_ELF_NOTES_REPEATED = + "Node SEA ELF note tables declare more notes than the file holds"; export const NODE_SEA_MACHO_COMMANDS_TRUNCATED = "Node SEA Mach-O load commands are truncated"; export const NODE_SEA_MACHO_SEGMENT_TRUNCATED = diff --git a/src/artifacts/nodeSea/NodeSeaLocation.ts b/src/artifacts/nodeSea/NodeSeaLocation.ts index 45095003f3..0fcddec37b 100644 --- a/src/artifacts/nodeSea/NodeSeaLocation.ts +++ b/src/artifacts/nodeSea/NodeSeaLocation.ts @@ -33,6 +33,7 @@ import { NODE_SEA_BLOB_TOO_LARGE, NODE_SEA_DUPLICATE_BLOB, NODE_SEA_ELF_DESCRIPTOR_TRUNCATED, + NODE_SEA_ELF_NOTES_REPEATED, NODE_SEA_ELF_TABLE_TRUNCATED, NODE_SEA_MACHO_COMMANDS_TRUNCATED, NODE_SEA_MACHO_SEGMENT_TRUNCATED, @@ -358,6 +359,10 @@ const locateElfNotes = async ( const minimumSection = elf64 ? 64 : 40; const found: BlobRange[] = []; let opaque = false; + // Every note header occupies at least 12 bytes, so disjoint tables hold at + // most one note per 12 file bytes. Headers that point at the same bytes + // would otherwise rescan them once per declaration. + const scan: NoteScan = { little, notes: Math.floor(carrier.fileSize / 12) }; const programs = await readElfTables(carrier, { offset: phoff, count: phnum, @@ -392,7 +397,7 @@ const locateElfNotes = async ( const scanned = await scanElfNoteTable( carrier, note, - little, + scan, found.length > 0, ); if (scanned === undefined) opaque = true; @@ -440,7 +445,7 @@ const locateElfNotes = async ( const scanned = await scanElfNoteTable( carrier, note, - little, + scan, found.length > 0, ); if (scanned === undefined) opaque = true; @@ -485,6 +490,12 @@ const readElfTables = async ( return entries; }; +/** One carrier's note byte order and the note headers it may still read. */ +interface NoteScan { + readonly little: boolean; + notes: number; +} + const scanElfNoteTable = async ( carrier: Carrier, note: { @@ -492,7 +503,7 @@ const scanElfNoteTable = async ( readonly length: number | undefined; readonly align: number | undefined; }, - little: boolean, + scan: NoteScan, alreadyFound: boolean, ): Promise => { if ( @@ -507,10 +518,18 @@ const scanElfNoteTable = async ( let cursor = 0; while (cursor + 12 <= note.length) { abortIfNeeded(carrier.signal); + // Past the file's note capacity the tables overlap: an unconfirmed + // carrier stays unrecognized and a confirmed one fails closed. + if (scan.notes === 0) { + if (alreadyFound || found.length > 0) + throw formatFailure(NODE_SEA_ELF_NOTES_REPEATED); + return undefined; + } + scan.notes -= 1; const header = await readExact(carrier, note.offset + cursor, 12); if (header === undefined) return failNoteTable(alreadyFound || found.length > 0); - const parsedHeader = readElfNoteHeader(header, little); + const parsedHeader = readElfNoteHeader(header, scan.little); if (!parsedHeader.ok) return failNoteTable(alreadyFound || found.length > 0); const { namesz, descsz } = parsedHeader.value; From f042aaaf90d7528a2589a8b4e55071e8c2cdd013 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare <48523873+rohitg00@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:02:00 +0530 Subject: [PATCH 31/94] fix(gdb): keep sessions usable after an oversized stream record (#8029) * fix(gdb): keep sessions usable after an oversized stream record A single MI console, target or log record over the 16 MiB line limit failed the whole session protocol, so close could not re-synchronize and GDB plus its inferior stayed owned with no recovery path. MI stream records carry only output and c-strings escape newlines, so a raw LF always ends the record. #receive now drops the remainder of an oversized stream record, marks pending command output and retained records truncated, and keeps parsing later lines; oversized result and async records still fail closed as before. Fixes #8023. * fix(gdb): keep earlier records in a chunk with an oversized stream record * test(gdb): drop superseded oversized-record test copies --- docs/reverse-engineering-tools.md | 2 +- src/gdb/GdbSessionManager.ts | 28 +++++++ .../GdbSessionManagerCoverage.test.ts | 79 +++++++++++++++++++ 3 files changed, 108 insertions(+), 1 deletion(-) diff --git a/docs/reverse-engineering-tools.md b/docs/reverse-engineering-tools.md index 10e7ab43cc..58fb66f0e7 100644 --- a/docs/reverse-engineering-tools.md +++ b/docs/reverse-engineering-tools.md @@ -46,7 +46,7 @@ Captured stdout and stderr are retained up to a 16 MiB combined limit; `output_t ## GDB -Persistent GDB sessions, including the MCP workflow and one-shot `rea debug-with-gdb` CLI, are available only on Linux. REA verifies a live inferior's output descriptors through `/proc` before detaching it; it does not claim equivalent descriptor verification on macOS. Windows is also unsupported because owned Job Object cleanup can terminate debugger inferiors. On supported Linux hosts, `start_gdb_session`, `gdb_console`, `gdb_session_status`, and `close_gdb_session` manage a persistent GDB/MI session in MCP. The one-shot CLI requires a caller-selected durable TTY: `rea debug-with-gdb --inferior-tty /dev/pts/7 'file ./program'`. The TTY must remain available after GDB exits so a detached inferior can continue writing. MCP starts GDB with `--nx --quiet --interpreter=mi3`, retries initialization with MI2 if MI3 initialization returns an error and cleanup succeeds, waits for its prompt, and disables auto-loading. Pass optional `inferior_tty` to select the terminal that future inferior runs use; REA configures it before the session becomes ready and checks the exact value with GDB/MI. `gdb_session_status` reports the selected path, whether REA still considers the configuration verified, and whether invalid UTF-8 has been observed in MI bytes (`mi_invalid_utf8`) or stderr (`stderr_invalid_utf8`). Commands that can change the inferior TTY or redirect a run revoke that verification permanently for the session, so close retains GDB ownership if a live inferior remains. The complete built-in inspections `show inferior-tty` and `help inferior-tty` preserve verification; compound commands still use the conservative mutation checks. Closing a session with one live inferior and a verified TTY automatically detaches the target, which can resume execution; MCP clients receive destructive-operation annotations for this behavior. The console remains unrestricted and can run shell commands, change target state, access files, and use network in the REA process user's security context. Raw MI records and console/target/log streams are returned with Evidence; command output is retained up to 16 MiB and explicitly marked when truncated. GDB startup waits 10000 ms and each MI command waits 120000 ms. This release has no runtime override for those deadlines, the 16 MiB command output budget, session capacity, or the pending-command queue. `command_timeout_ms` does not change them. +Persistent GDB sessions, including the MCP workflow and one-shot `rea debug-with-gdb` CLI, are available only on Linux. REA verifies a live inferior's output descriptors through `/proc` before detaching it; it does not claim equivalent descriptor verification on macOS. Windows is also unsupported because owned Job Object cleanup can terminate debugger inferiors. On supported Linux hosts, `start_gdb_session`, `gdb_console`, `gdb_session_status`, and `close_gdb_session` manage a persistent GDB/MI session in MCP. The one-shot CLI requires a caller-selected durable TTY: `rea debug-with-gdb --inferior-tty /dev/pts/7 'file ./program'`. The TTY must remain available after GDB exits so a detached inferior can continue writing. MCP starts GDB with `--nx --quiet --interpreter=mi3`, retries initialization with MI2 if MI3 initialization returns an error and cleanup succeeds, waits for its prompt, and disables auto-loading. Pass optional `inferior_tty` to select the terminal that future inferior runs use; REA configures it before the session becomes ready and checks the exact value with GDB/MI. `gdb_session_status` reports the selected path, whether REA still considers the configuration verified, and whether invalid UTF-8 has been observed in MI bytes (`mi_invalid_utf8`) or stderr (`stderr_invalid_utf8`). Commands that can change the inferior TTY or redirect a run revoke that verification permanently for the session, so close retains GDB ownership if a live inferior remains. The complete built-in inspections `show inferior-tty` and `help inferior-tty` preserve verification; compound commands still use the conservative mutation checks. Closing a session with one live inferior and a verified TTY automatically detaches the target, which can resume execution; MCP clients receive destructive-operation annotations for this behavior. The console remains unrestricted and can run shell commands, change target state, access files, and use network in the REA process user's security context. Raw MI records and console/target/log streams are returned with Evidence; command output is retained up to 16 MiB and explicitly marked when truncated. A single console, target or log record over the 16 MiB MI line limit is dropped and the command is marked truncated, while an oversized result or async record still stops the session. GDB startup waits 10000 ms and each MI command waits 120000 ms. This release has no runtime override for those deadlines, the 16 MiB command output budget, session capacity, or the pending-command queue. `command_timeout_ms` does not change them. With multiple live inferiors, or a live inferior without a verified TTY, manual close requests an explicit disposition and keeps the session ready. Use diff --git a/src/gdb/GdbSessionManager.ts b/src/gdb/GdbSessionManager.ts index 8eb8c3d7fb..23b2372e75 100644 --- a/src/gdb/GdbSessionManager.ts +++ b/src/gdb/GdbSessionManager.ts @@ -520,6 +520,7 @@ interface GdbSession { bufferTail: Buffer | undefined; bufferTailBytes: number; bufferBytes: number; + discardingOversizedStreamRecord: boolean; invalidUtf8Observed: boolean; promptTail: string; ready: boolean; @@ -881,6 +882,7 @@ export class GdbSessionManager { bufferTail: undefined, bufferTailBytes: 0, bufferBytes: 0, + discardingOversizedStreamRecord: false, invalidUtf8Observed: false, promptTail: "", ready: false, @@ -1663,6 +1665,13 @@ export class GdbSessionManager { []; let offset = 0; while (offset < data.byteLength) { + if (session.discardingOversizedStreamRecord) { + const newline = data.indexOf(0x0a, offset); + if (newline < 0) break; + session.discardingOversizedStreamRecord = false; + offset = newline + 1; + continue; + } const newline = data.indexOf(0x0a, offset); const piece = data.subarray( offset, @@ -1672,6 +1681,25 @@ export class GdbSessionManager { const lineBytes = session.bufferBytes - (newline >= 0 && piece.at(-1) === 0x0d ? 1 : 0); if (lineBytes > MAX_MI_LINE_BYTES) { + const firstByte = + session.bufferChunks[0]?.at(0) ?? + session.bufferTail?.at(0) ?? + piece.at(0); + if (firstByte === 0x7e || firstByte === 0x40 || firstByte === 0x26) { + session.bufferChunks.length = 0; + session.bufferTail = undefined; + session.bufferTailBytes = 0; + session.bufferBytes = 0; + for (const pending of session.pending.values()) + pending.outputTruncated = true; + session.recentRecordsTruncated = true; + if (newline < 0) { + session.discardingOversizedStreamRecord = true; + break; + } + offset = newline + 1; + continue; + } this.#failProtocol( session, `GDB emitted an MI line exceeding the ${String(MAX_MI_LINE_BYTES)} byte (16 MiB) protocol limit. This release has no runtime override.`, diff --git a/tests/boundary/providers/GdbSessionManagerCoverage.test.ts b/tests/boundary/providers/GdbSessionManagerCoverage.test.ts index 5ddd2026ed..5d0adde808 100644 --- a/tests/boundary/providers/GdbSessionManagerCoverage.test.ts +++ b/tests/boundary/providers/GdbSessionManagerCoverage.test.ts @@ -3633,6 +3633,85 @@ process.stdin.on('data', (chunk) => { expect(manager.status(started.value.session_id)?.state).toBe("closed"); }); +it("keeps the session usable after an oversized stream record is dropped", async () => { + const fake = createFakeSpawner((child, token, command) => { + if (command === "-gdb-set auto-load off") { + child.stdout.write(`${token}^done\n`); + return; + } + if (command === '-interpreter-exec console "big-stream"') { + child.stdout.write('~"before"\n~"' + "x".repeat(16 * 1024 * 1024 + 1)); + child.stdout.write('"\n' + token + "^done\n(gdb)\n"); + return; + } + if (command === '-interpreter-exec console "after-drop"') { + child.stdout.write(token + "^done\n(gdb)\n"); + return; + } + if (command === "-list-thread-groups --recurse 1") { + child.stdout.write(`${token}^done,groups=[]\n`); + return; + } + if (command === "-gdb-exit") { + child.stdout.write(`${token}^done\n`); + return; + } + child.stdout.write(token + '^error,msg="unmodeled request"\n'); + }); + const manager = managerWithFakeSpawner(fake.spawnProcess); + const started = await expectGdbSessionStarted(manager); + + const executed = await manager.execute( + started.value.session_id, + "big-stream", + ); + + expect(executed.ok).toBe(true); + if (executed.ok) { + expect(executed.value.result.mi).toMatch(/^\d+\^done$/); + expect(executed.value.result.console).toContain("before"); + expect(executed.value.result.completion_status).toBe("complete"); + expect(executed.value.result.output_truncated).toBe(true); + expect(executed.value.evidence.limitations.join(" ")).toContain( + "Retained MI records reached the 16777216 byte or 16384 record cap. This release has no runtime override. Output completeness is unknown.", + ); + } + const second = await manager.execute(started.value.session_id, "after-drop"); + expect(second.ok).toBe(true); + if (second.ok) expect(second.value.result.mi).toMatch(/^\d+\^done$/); + await expect(manager.close(started.value.session_id)).resolves.toMatchObject({ + ok: true, + }); +}); + +it("still stops the session for an oversized result record", async () => { + const manager = await managerFor(` +process.stdout.write('(gdb)\\n'); +let input = ''; +process.stdin.setEncoding('utf8'); +process.stdin.on('data', (chunk) => { + input += chunk; + for (;;) { + const end = input.indexOf('\\n'); + if (end < 0) break; + const line = input.slice(0, end); input = input.slice(end + 1); + const match = /^(\\d+)(.*)/.exec(line); + if (!match) continue; + if (match[2] === '-gdb-set auto-load off') process.stdout.write(match[1] + '^done\\n'); + else if (match[2] === '-interpreter-exec console "bad-value"') process.stdout.write(match[1] + '^done,value="' + 'x'.repeat(16 * 1024 * 1024 + 1) + '"\\n'); + else process.stdout.write(match[1] + '^error,msg="unmodeled request"\\n'); + } +}); +`); + const started = await expectGdbSessionStarted(manager); + + const failed = await manager.execute(started.value.session_id, "bad-value"); + + expect(failed.ok).toBe(false); + if (!failed.ok) + expect(failed.error._tag).toBe("AnalysisCapabilityUnavailableError"); +}); + it("caps the number of MI records retained for a single command", async () => { const recordCount = 16_385; const manager = await managerFor(` From 1dd70f6c2b1c65c48171e931f16ed69d428fa235 Mon Sep 17 00:00:00 2001 From: 2SinceJuly Date: Wed, 4 Nov 2026 19:32:44 +0800 Subject: [PATCH 32/94] fix(cli): seal in-process Evidence before publishing (#8468) * fix(cli): seal in-process Evidence before publishing The CLI rebuilt a plain bundle from records it had already authenticated, so writeEvidenceBundle cloned and authenticated the complete analyzed graph again at the final checkpoint. On a default heap that second copy could exceed available memory after minutes of analysis and discard the whole run. Seal the in-process bundle once so publication can stream the canonical form without the redundant authentication copies. The regression test fails on the previous behavior with resource_constraint under a one-byte heap allowance. Fixes #8411 * fix(cli): preserve the heap check for mutable Evidence Only seal the publication bundle when every record is already immutable. Mutable producer records keep the writer's existing heap check instead of being copied before that check. The regression now covers the safe fallback and lets the shared temporary-directory fixture own cleanup. * test(cli): keep a distinct mutable source in the publication regression Seal the main record and give the mutable source its own evidence ID so the map cannot replace it before the bundle guard runs. --- src/cli/analysisEvidenceDelivery.ts | 26 ++++-- .../analysisEvidenceDelivery.test.ts | 83 +++++++++++++++++++ 2 files changed, 101 insertions(+), 8 deletions(-) create mode 100644 tests/process-global/analysisEvidenceDelivery.test.ts diff --git a/src/cli/analysisEvidenceDelivery.ts b/src/cli/analysisEvidenceDelivery.ts index 1858bd3fa8..2d0cf5d26c 100644 --- a/src/cli/analysisEvidenceDelivery.ts +++ b/src/cli/analysisEvidenceDelivery.ts @@ -6,6 +6,7 @@ import { import type { AnalysisViewRequest } from "../domain/analysisView/analysisView.js"; import { z } from "incur"; import { + isImmutableEvidence, parseEvidence, withEvidenceCallerInputPath, type Evidence, @@ -16,7 +17,10 @@ import type { AnalysisError } from "../domain/analysisErrorBase.js"; import { redactExplicitFailure } from "../domain/explicitSensitiveFailure.js"; import { err, ok, type Result } from "../domain/result.js"; import { AnalysisInputError } from "../domain/analysisErrorCore.js"; -import { createEvidenceBundle } from "../domain/evidenceBundle.js"; +import { + createEvidenceBundle, + createImmutableEvidenceBundle, +} from "../domain/evidenceBundle.js"; import { evidenceDeliveryCommitted, publishEvidenceDelivery, @@ -240,14 +244,20 @@ export const publishCliAnalysisEvidence = async ( : delivered.value; const inspectionSources = analysisEvidenceSources(complete); if (!inspectionSources.ok) return undelivered(inspectionSources.error); + const records = [ + ...new Map( + [...(input.sourceEvidence ?? []), ...inspectionSources.value].map( + (record) => [record.evidence_id, record], + ), + ).values(), + ]; + // Only skip the writer's heap check when every record is already sealed. + // A mutable producer record must retain the existing allocation check. + const bundle = records.every(isImmutableEvidence) + ? createImmutableEvidenceBundle(records) + : createEvidenceBundle(records); const published = await publishEvidenceDelivery({ - bundle: createEvidenceBundle([ - ...new Map( - [...(input.sourceEvidence ?? []), ...inspectionSources.value].map( - (record) => [record.evidence_id, record], - ), - ).values(), - ]), + bundle, complete, delivered: selected, outputPath: absoluteCallerPath(evidenceOutput), diff --git a/tests/process-global/analysisEvidenceDelivery.test.ts b/tests/process-global/analysisEvidenceDelivery.test.ts new file mode 100644 index 0000000000..04fcd59197 --- /dev/null +++ b/tests/process-global/analysisEvidenceDelivery.test.ts @@ -0,0 +1,83 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it, vi } from "vitest"; + +import { publishCliAnalysisEvidence } from "../../src/cli/analysisEvidenceDelivery.js"; +import { createEvidence } from "../../src/domain/evidence.js"; +import { parseEvidenceBundle } from "../../src/domain/evidenceBundle.js"; +import { ok } from "../../src/domain/result.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +vi.mock("node:v8", () => ({ + getHeapStatistics: () => ({ + total_available_size: 1, + heap_size_limit: 1, + }), +})); + +it("publishes CLI Evidence without re-authenticating the complete graph", async () => { + const root = await createTestTempDirectory("rea-cli-evidence-"); + const destination = join(root, "bundle.json"); + const evidence = createEvidence( + undefined, + { id: "fixture", name: "Fixture", version: "1" }, + { + operation: "fixture", + parameters: { payload_bytes: 4_000_000 }, + result: { payload: "x".repeat(4_000_000) }, + }, + { immutable: true }, + ); + + const output = await publishCliAnalysisEvidence(ok(evidence), { + providerId: "fixture", + operation: "fixture", + signal: new AbortController().signal, + detail: "complete", + evidenceOutput: destination, + }); + + expect(output).toMatchObject({ evidence_id: evidence.evidence_id }); + const bundle = parseEvidenceBundle( + JSON.parse(await readFile(destination, "utf8")), + ); + expect(bundle.records.map((record) => record.evidence_id)).toContain( + evidence.evidence_id, + ); +}); + +it("keeps the writer heap check for mutable source Evidence", async () => { + const root = await createTestTempDirectory("rea-cli-mutable-evidence-"); + const destination = join(root, "bundle.json"); + const evidence = createEvidence( + undefined, + { id: "fixture", name: "Fixture", version: "1" }, + { + operation: "fixture", + parameters: { payload_bytes: 1_000 }, + result: { payload: "x".repeat(1_000) }, + }, + { immutable: true }, + ); + const source = createEvidence( + undefined, + { id: "fixture", name: "Fixture", version: "1" }, + { + operation: "source-fixture", + parameters: { payload_bytes: 1_000 }, + result: { payload: "y".repeat(1_000) }, + }, + ); + const mutableSource = JSON.parse(JSON.stringify(source)) as typeof source; + + const output = await publishCliAnalysisEvidence(ok(evidence), { + providerId: "fixture", + operation: "fixture", + signal: new AbortController().signal, + detail: "complete", + evidenceOutput: destination, + sourceEvidence: [mutableSource], + }); + + expect(output).toMatchObject({ code: "resource_constraint" }); +}); From 39172ca2e191763a87bedf889b1f970befd5d96c Mon Sep 17 00:00:00 2001 From: Machen John Date: Wed, 4 Nov 2026 12:33:29 +0100 Subject: [PATCH 33/94] fix(ghidra): keep a changed existing project's refusal typed through launch (#8273) * fix(ghidra): keep a changed existing project's refusal typed through launch When the selected REA_GHIDRA_PROJECT_PATH project changes after open_binary commits its profile, the launcher's verified copy refuses it with AnalysisArtifactChangedError. GhidraHeadlessLauncher wraps that in GhidraLaunchError, and the session error's cause became the wrapper, so the projection missed the existing artifact_changed branch and returned generic execution_failure with rea doctor advice. Read the launcher's primary failure when it is an AnalysisArtifactChangedError, so the refusal reaches callers as artifact_changed with the changed path and the existing reopen guidance. The refusal stays fail-closed: no automatic reopen or fresh-import fallback. Other launch failures keep their current projection. Fixes #8217 * fix(ghidra): describe a project profile mismatch without a timing claim When the owned copy of a selected Ghidra project no longer matches the fingerprint committed by open_binary, copyExistingGhidraProject threw RegularFileChangedError, so the artifact_changed reason said the input "changed while being read". A digest mismatch only shows that the project differs from its committed profile; the change may have happened before the copy started. Throw AnalysisArtifactChangedError directly with a reason that states the mismatch and leaves the time of the change unestablished. Changes observed while files are being read keep their existing reason. --- src/ghidra/GhidraProjectSnapshot.ts | 13 ++- src/ghidra/GhidraProvider.behavior.test.ts | 80 ++++++++++++++++++- src/ghidra/GhidraProviderClient.ts | 12 ++- .../ghidra/ghidraProjectSnapshot.test.ts | 5 +- 4 files changed, 105 insertions(+), 5 deletions(-) diff --git a/src/ghidra/GhidraProjectSnapshot.ts b/src/ghidra/GhidraProjectSnapshot.ts index 75f85566e6..6257dbc0c4 100644 --- a/src/ghidra/GhidraProjectSnapshot.ts +++ b/src/ghidra/GhidraProjectSnapshot.ts @@ -23,11 +23,16 @@ import { RegularFileChangedError, } from "../filesystem/RegularFile.js"; import { ConfigurationError } from "../domain/configurationErrors.js"; +import { AnalysisArtifactChangedError } from "../domain/analysisErrorCore.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; import type { JsonValue } from "../domain/jsonValue.js"; import type { GhidraProjectSelection } from "../config/ghidraExistingProject.js"; import { ghidraProgramNameSchema } from "../config/ghidraExistingProject.js"; +/** Reason for a selected project whose content no longer matches its committed profile. */ +export const GHIDRA_PROJECT_PROFILE_MISMATCH = + "Selected Ghidra project content no longer matches the profile committed by open_binary; when it changed is not established"; + /** Classify file cancellation/cleanup before the generic provider boundary. */ const projectFileOperation = async ( path: string, @@ -296,7 +301,13 @@ export const copyExistingGhidraProject = async ( signal, )) !== expectedSha256 ) - throw new RegularFileChangedError(sourcePath); + // A digest mismatch shows only that the project differs from its + // committed profile, not when it changed relative to this copy. + throw new AnalysisArtifactChangedError( + "open_binary", + sourcePath, + GHIDRA_PROJECT_PROFILE_MISMATCH, + ); } }); }; diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index 369d74d014..5663f10732 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -75,7 +75,12 @@ import { ghidraStartupTimeoutGuidance, type GhidraProviderClientFactory, } from "./GhidraProviderClient.js"; -import type { GhidraHeadlessLauncher } from "./GhidraLauncher.js"; +import { + GhidraLaunchError, + type GhidraHeadlessLauncher, +} from "./GhidraLauncher.js"; +import { AnalysisArtifactChangedError } from "../domain/analysisErrorCore.js"; +import { GHIDRA_PROJECT_PROFILE_MISMATCH } from "./GhidraProjectSnapshot.js"; import type { GhidraInstallationHost } from "./GhidraInstallation.js"; import { GHIDRA_MUTATING_OPERATIONS, @@ -4243,6 +4248,79 @@ describe("Ghidra missing project program", () => { }); }); +describe("Ghidra changed project", () => { + const launchFailure = async (cause: unknown) => { + const root = await mkdtemp(join(tmpdir(), "rea-changed-project-")); + onTestFinished(async () => { + await rm(root, { recursive: true, force: true }); + }); + await mkdir(join(root, "selected.rep")); + await writeFile(join(root, "selected.gpr"), "marker"); + const failure = new GhidraSessionError( + "start", + "Ghidra headless launch failed", + { failure_kind: "start" }, + { + cause: new GhidraLaunchError("Ghidra headless launch failed", { + cause, + }), + }, + ); + const ghidra = provider( + installationHost(), + () => ({ + start: async () => err(failure), + callTool: async () => err(failure), + close: async () => ok(null), + }), + { REA_GHIDRA_PROJECT_PATH: join(root, "selected.gpr") }, + ); + onTestFinished(async () => { + await ghidra.closeOwnedResources(); + }); + const target = executableTarget("elf", "x86_64"); + const profile = await ghidra.resolveAnalysisProfile(target); + if (!profile.ok || profile.value.profile === null) + throw new Error("Expected Ghidra analysis profile"); + const result = await ghidra + .createClient(target, profile.value.profile) + .execute("list_documents", {}); + if (result.ok) throw new Error("Expected a Ghidra start failure"); + return { + error: result.error, + projected: projectAnalysisError(result.error), + }; + }; + + it("projects a source project changed before its verified copy as artifact_changed", async () => { + const path = "/projects/selected"; + const reason = GHIDRA_PROJECT_PROFILE_MISMATCH; + const { error, projected } = await launchFailure( + new AnalysisArtifactChangedError("open_binary", path, reason), + ); + expect(projected).toMatchObject({ + code: "artifact_changed", + details: { + operation: "list_documents", + path, + reason, + boundary: "stable-artifact-read", + }, + }); + expect(projected.remediation.action).toBe( + projectAnalysisError( + new AnalysisArtifactChangedError("list_documents", path, reason), + ).remediation.action, + ); + expect(error.cause).toBeInstanceOf(GhidraSessionError); + }); + + it("keeps the generic start failure for other launch failures", async () => { + const { projected } = await launchFailure(new Error("spawn failed")); + expect(projected.code).toBe("execution_failure"); + }); +}); + describe("Ghidra analysis seeds and configured language", () => { const seedSession = async ( report: (sha256: string) => GhidraSeedReport | undefined, diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index 77bb0f1adf..a01afa9328 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -98,7 +98,7 @@ import { } from "./GhidraRiscvProfile.js"; import { ghidraPePdbLimitations } from "./GhidraPeProfile.js"; import { unverifiedGhidraBuildLimitation } from "./GhidraReleasePolicy.js"; -import { GhidraHeadlessLauncher } from "./GhidraLauncher.js"; +import { GhidraHeadlessLauncher, GhidraLaunchError } from "./GhidraLauncher.js"; import { attestGhidraNativeLoadImage } from "./GhidraLoadImageAttest.js"; import type { NativeAotPeResult } from "../domain/native/nativeAotPe.js"; import { GhidraNativeAotSnapshotOwner } from "./GhidraNativeAotSnapshotOwner.js"; @@ -1197,8 +1197,16 @@ const projectPrimarySessionError = ( ? {} : { partialObservation: failure.partialObservation }), }; + // Launch-time copies, such as the verified existing-project copy, report a + // changed source as the launcher's primary failure; read it through the + // launch wrapper so the refusal keeps its typed projection. + const launchChange = + failure.cause instanceof GhidraLaunchError && + failure.cause.primaryFailure instanceof AnalysisArtifactChangedError + ? failure.cause.primaryFailure + : undefined; const { cause: physicalCause, origin } = artifactFilePrimaryFacts( - failure.cause, + launchChange ?? failure.cause, ); // A client-side ceiling refusal travels as the cause; re-attribute it to this // operation so callers see a typed resource constraint rather than a generic diff --git a/tests/boundary/providers/ghidra/ghidraProjectSnapshot.test.ts b/tests/boundary/providers/ghidra/ghidraProjectSnapshot.test.ts index 1f1e0a0033..effb563d4a 100644 --- a/tests/boundary/providers/ghidra/ghidraProjectSnapshot.test.ts +++ b/tests/boundary/providers/ghidra/ghidraProjectSnapshot.test.ts @@ -15,6 +15,7 @@ import { ArtifactResourceScope } from "../../../../src/artifacts/ArtifactResourc import { copyExistingGhidraProject, fingerprintGhidraProject, + GHIDRA_PROJECT_PROFILE_MISMATCH, resolveGhidraProject, } from "../../../../src/ghidra/GhidraProjectSnapshot.js"; import { parseConfig } from "../../../../src/config/parseConfig.js"; @@ -85,7 +86,9 @@ it("rejects a project whose copied content differs from the committed analysis p }), ).rejects.toMatchObject({ _tag: "AnalysisArtifactChangedError", - cause: { name: "RegularFileChangedError" }, + operation: "open_binary", + path: source, + reason: GHIDRA_PROJECT_PROFILE_MISMATCH, }); }); From a97f82746973613c77b6a93d0576b5383bfbb595 Mon Sep 17 00:00:00 2001 From: RRFZ <247457381+Plastic-time@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:34:31 -0500 Subject: [PATCH 34/94] fix(pe): reject malformed certificate validity times (#8832) --- src/native/pe/PeCertificateAsn1.ts | 41 +++++++++++++++++++++- src/native/pe/PeCertificateParser.test.ts | 38 ++++++++++++++++++++ tests/acceptance/peCertificatesCli.test.ts | 34 ++++++++++++++++++ 3 files changed, 112 insertions(+), 1 deletion(-) diff --git a/src/native/pe/PeCertificateAsn1.ts b/src/native/pe/PeCertificateAsn1.ts index 2f476d4f20..f4f1c20e8b 100644 --- a/src/native/pe/PeCertificateAsn1.ts +++ b/src/native/pe/PeCertificateAsn1.ts @@ -319,12 +319,51 @@ const decodeName = ( return { text, attributes }; }; +// RFC 5280 4.1.2.5 and X.690 11.7-11.8: UTC with seconds, a final "Z", and +// no fractional seconds. UTCTime years 50-99 are 1950-1999, 00-49 are 2000-2049. +const VALIDITY_TIMES = { + 0x17: { + label: "UTCTime", + layout: "YYMMDDHHMMSSZ", + form: /^(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})Z$/u, + }, + 0x18: { + label: "GeneralizedTime", + layout: "YYYYMMDDHHMMSSZ", + form: /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})Z$/u, + }, +} as const; + +const isCalendarInstant = ( + year: number, + [month = 0, day = 0, hour = 24, minute = 60, second = 60]: number[], +): boolean => { + const leap = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0); + const days = [31, leap ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + return ( + day >= 1 && + day <= (days[month - 1] ?? 0) && + hour <= 23 && + minute <= 59 && + second <= 59 + ); +}; + const validityTime = (node: Asn1BerNode, budget: DecodedNameBudget): string => { if (node.tag !== 0x17 && node.tag !== 0x18) return fail("Validity instant is not UTCTime or GeneralizedTime."); // A zero byte becomes "\u0000" (six characters) in the hashed evidence JSON. chargeDecodedName(budget, node.content.length * 6); - return node.content.toString("utf8"); + const { label, layout, form } = VALIDITY_TIMES[node.tag]; + // Latin-1 maps each byte to one character, so no invalid byte can match. + const text = node.content.toString("latin1"); + const [year = 0, ...rest] = form.exec(text)?.slice(1).map(Number) ?? []; + if (rest.length === 0) + return fail(`${label} validity instant is not ${layout}.`); + const fullYear = node.tag === 0x17 ? year + (year < 50 ? 2000 : 1900) : year; + if (!isCalendarInstant(fullYear, rest)) + return fail(`${label} validity instant is not a calendar time.`); + return text; }; export interface ParsedCertificate { diff --git a/src/native/pe/PeCertificateParser.test.ts b/src/native/pe/PeCertificateParser.test.ts index 20bb591eda..3befe7b42b 100644 --- a/src/native/pe/PeCertificateParser.test.ts +++ b/src/native/pe/PeCertificateParser.test.ts @@ -786,6 +786,44 @@ it("rejects a UTF8String name that is not valid UTF-8", () => { ).toBe("commonName=é"); }); +it("accepts only RFC 5280 UTC validity instants", () => { + const name = directoryName(der(0x0c, Buffer.from("Fixture"))); + const certificate = (notBefore: Buffer) => + readAsn1( + derSeq( + derSeq( + derInt(1), + SHA256_ALG, + name, + derSeq(notBefore, der(0x17, Buffer.from("260101000000Z"))), + name, + spki, + ), + SHA256_ALG, + der(0x03, Buffer.from([0])), + ), + ); + for (const [tag, content] of [ + [0x17, Buffer.from([0xff, 0xfe])], + [0x17, Buffer.from("2501010000Z")], + [0x17, Buffer.from("250101000000+0800")], + [0x17, Buffer.from("250229000000Z")], + [0x18, Buffer.from("20250101000000.5Z")], + [0x18, Buffer.from("20250101240000Z")], + ] as const) + expect(() => parseCertificate(certificate(der(tag, content)))).toThrow( + PeCertificateAsn1Failure, + ); + // UTCTime 00 is 2000, a leap year; GeneralizedTime carries 2050 onward. + for (const [tag, time] of [ + [0x17, "000229235959Z"], + [0x18, "20500101000000Z"], + ] as const) + expect( + parseCertificate(certificate(der(tag, Buffer.from(time)))).not_before, + ).toBe(time); +}); + it.each([ ["GeneralString", 0x1b, Buffer.from("name")], ["OCTET STRING", 0x04, Buffer.from("name")], diff --git a/tests/acceptance/peCertificatesCli.test.ts b/tests/acceptance/peCertificatesCli.test.ts index c068c47ecb..1a877b942c 100644 --- a/tests/acceptance/peCertificatesCli.test.ts +++ b/tests/acceptance/peCertificatesCli.test.ts @@ -272,6 +272,40 @@ cliTest( }, ); +cliTest( + "rejects a certificate validity date that is not a calendar day through the public CLI", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-pe-cert-cli-validity-"); + const path = join(root, "validity.exe"); + const payload = pkcs7SignedDataFixture(); + const timeAt = payload.indexOf(Buffer.from("250101000000Z")); + expect(timeAt).toBeGreaterThanOrEqual(0); + // 2025 has no 29 February; the equal-length edit keeps every DER length. + payload.write("250229000000Z", timeAt, "ascii"); + await writeFile(path, peCertificateFixture(false, payload)); + const invalid = await cli.run({ + arguments: ["inspect-pe-certificates", path, "--json"], + }); + expect(invalid.exitCode).toBe(1); + expect(invalid.json).toMatchObject({ + code: "invalid_request", + details: { operation: "inspect_pe_certificates" }, + }); + payload.write("240229000000Z", timeAt, "ascii"); + await writeFile(path, peCertificateFixture(false, payload)); + const valid = await cli.run({ + arguments: ["inspect-pe-certificates", path, "--json"], + }); + expect(valid.exitCode, valid.stderr).toBe(0); + const evidence = toolContract("inspect_pe_certificates").outputSchema.parse( + valid.json, + ); + expect( + evidence.normalized_result.entries[0]?.pkcs7?.certificates[0]?.not_before, + ).toBe("240229000000Z"); + }, +); + cliTest( "inventories repeated certificates within both decoder allowances", async ({ cli }) => { From d5ba7cec60aeadd1bb7a5f4bef343d930651ecf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E4=BD=B3=E6=B5=A9Alben?= <35291372+XiaoBinGan@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:35:16 +0800 Subject: [PATCH 35/94] fix(javascript): link a response body reader only to a settled fetch (#8028) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(javascript): link a response body reader only to a settled fetch `collectResponseConsumers` treated any `.json()`/`.text()`/`.arrayBuffer()` on a binding holding a request result as a response consumer, and gave it the request's endpoint at `resolution: "complete"`. That holds only for a settled `fetch`. Measured on Node 24.22.1: await fetch(u) -> Response json/text/arrayBuffer = function fetch(u) unawaited -> Promise json = undefined; pending.json() THROWS TypeError http.get(u) -> ClientRequest json/text/arrayBuffer = undefined http.request(u) -> ClientRequest json = undefined; request.text() THROWS TypeError new WebSocket(u) -> WebSocket socket.json() THROWS TypeError await http.get(u) -> ClientRequest (await does not help) So four shapes emitted a resolved "this endpoint's body is read here" fact for a call that throws before any body is read: an un-awaited fetch, `http.get`, `http.request`, and a `WebSocket` handshake. Link only requests that settle to a `Response`: a `fetch` call whose value is awaited through expression wrappers. A binding that also holds a request kind without a body reader keeps the consumer a candidate (`partial`) rather than a proven one, and a binding with no settled fetch emits no consumer at all, which matches the `collectReleases` / `forkRejectsStdioArray` precedent — the receiver provably has no such member, so there is no fact to report. * chore: retrigger title and readiness checks after the title was corrected --- ...nticAnalysis.response_body_readers.test.ts | 84 +++++++++++++++++++ .../javascriptSemanticDataEffects.ts | 37 +++++++- 2 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 src/domain/javascript/javascriptSemanticAnalysis.response_body_readers.test.ts diff --git a/src/domain/javascript/javascriptSemanticAnalysis.response_body_readers.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.response_body_readers.test.ts new file mode 100644 index 0000000000..26ef076033 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticAnalysis.response_body_readers.test.ts @@ -0,0 +1,84 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js"; + +/** + * Only `fetch` resolves to a `Response`, and only once its promise settles. + * `http.request`/`http.get` return a `ClientRequest` and `new WebSocket` a + * socket; neither exposes `json`/`text`/`arrayBuffer`, and an un-awaited + * `fetch` returns a `Promise`. In every one of those the call throws at the call + * site instead of reading a body, so it is not a response consumer. + */ +const consumersFor = (source: string) => + analyzeJavaScriptSemantics(source) + .requestOperations.filter(({ kind }) => kind === "response-consumer") + .map(({ method, endpoint, resolution }) => ({ + method, + endpoint, + resolution, + })); + +it.each([ + [ + "an un-awaited fetch", + `async function f(){ const res = fetch("https://api.test/x"); return res.json(); }`, + ], + [ + "an http.get request", + `import http from "node:http"; + function f(){ const req = http.get("https://api.test/x"); return req.json(); }`, + ], + [ + "an http.request request", + `import http from "node:http"; + function f(){ const req = http.request("https://api.test/x"); return req.text(); }`, + ], + [ + "a WebSocket handshake", + `function f(){ const ws = new WebSocket("wss://api.test/x"); return ws.json(); }`, + ], +])("emits no response consumer for %s", (_name, source) => { + expect(consumersFor(source)).toEqual([]); +}); + +it.each([ + ["json", "json"], + ["text", "text"], + ["arrayBuffer", "arrayBuffer"], +] as const)("keeps the settled fetch body reader for %s", (_name, method) => { + expect( + consumersFor( + `async function f(){ const res = await fetch("https://api.test/x"); return res.${method}(); }`, + ), + ).toEqual([ + { + method, + endpoint: "https://api.test/x", + resolution: "complete", + }, + ]); +}); + +it("keeps a settled fetch reader through a wrapper expression", () => { + expect( + consumersFor( + `async function f(){ const res = await fetch("https://api.test/x"); return (res).json(); }`, + ), + ).toHaveLength(1); +}); + +it("keeps the fetch candidate when the binding also holds another request", () => { + // The mixed binding keeps its fetch link, so the consumer stays a candidate + // rather than losing the endpoint entirely. + const consumers = consumersFor(` + import http from "node:http"; + async function f(){ + let res = await fetch("https://api.test/x"); + res = await http.get("https://api.test/y"); + return res.json(); + } + `); + expect(consumers).toEqual([ + { method: "json", endpoint: null, resolution: "partial" }, + ]); +}); diff --git a/src/domain/javascript/javascriptSemanticDataEffects.ts b/src/domain/javascript/javascriptSemanticDataEffects.ts index 7e8e8163b3..4b8385f53a 100644 --- a/src/domain/javascript/javascriptSemanticDataEffects.ts +++ b/src/domain/javascript/javascriptSemanticDataEffects.ts @@ -8,6 +8,7 @@ import type { } from "./javascriptSemanticIr.js"; import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; import { + javaScriptExpressionWrapperTarget, readExactJavaScriptLiteral, semanticStaticPropertyKey, semanticStaticPropertyName, @@ -304,6 +305,28 @@ const soleWriteReachesRead = ( ); }; +/** + * Only `fetch` resolves to a `Response`, and only once its promise settles. + * `http.request`/`http.get` return a `ClientRequest` and `new WebSocket` a + * socket, neither of which has `json`/`text`/`arrayBuffer`; an un-awaited + * `fetch` returns a `Promise`. In all of those the call throws at the call site + * instead of reading a body, so it is not a response consumer at all. + */ +const settlesToResponse = ( + request: RequestCandidate, + context: TraversalContext, +): boolean => { + if (request.operation.method !== "fetch") return false; + let node: t.Node = request.node; + for (;;) { + const parent = context.state.parentsByNode.get(node); + if (parent === undefined) return false; + if (t.isAwaitExpression(parent)) return true; + if (javaScriptExpressionWrapperTarget(parent) !== node) return false; + node = parent; + } +}; + const collectResponseConsumers = ( program: t.Program, context: TraversalContext, @@ -338,11 +361,20 @@ const collectResponseConsumers = ( member.object, member.object.name, ); - const linked = + const candidates = binding === undefined ? [] : (requestsByBinding.get(binding.bindingId) ?? []); + // A body reader only exists on a settled fetch Response. Requests of any + // other kind throw at this call site instead, so they link nothing; if no + // candidate survives, the call is not a response consumer at all. + const linked = candidates.filter((request) => + settlesToResponse(request, context), + ); if (linked.length === 0) return; + // A binding that also receives a request kind without a body reader leaves + // this reader a candidate, not a proven consumer of the fetch response. + const otherKinds = candidates.length !== linked.length; // A consumer of exactly one request through its own result binding // reads that request's response, so it keeps the request's endpoint. const uniqueRequest = @@ -377,7 +409,8 @@ const collectResponseConsumers = ( .sort(compareUnicodeCodePoints), endpoint: soleWrite ? (uniqueRequest?.operation.endpoint ?? null) : null, fields: [], - resolution: uniqueRequest === undefined ? "partial" : "complete", + resolution: + uniqueRequest === undefined || otherKinds ? "partial" : "complete", }); }); return output; From 6aeaa04e54cea0411822ca52632d10978fc341ea Mon Sep 17 00:00:00 2001 From: Wenyu Chiou <162016108+WenyuChiou@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:36:00 -0500 Subject: [PATCH 36/94] fix(javascript): reuse shared AST child traversal (#7967) --- .../electronCallPathBlockPrefixes.test.ts | 121 +++++++++++++++++- .../electronCallPathBlockPrefixes.ts | 2 + 2 files changed, 122 insertions(+), 1 deletion(-) diff --git a/src/domain/javascript/electronCallPathBlockPrefixes.test.ts b/src/domain/javascript/electronCallPathBlockPrefixes.test.ts index 7834d552a4..ce66f1b771 100644 --- a/src/domain/javascript/electronCallPathBlockPrefixes.test.ts +++ b/src/domain/javascript/electronCallPathBlockPrefixes.test.ts @@ -2,7 +2,10 @@ import * as t from "@babel/types"; import { expect, it } from "vitest"; import { ElectronCallPathBlockPrefixes } from "./electronCallPathBlockPrefixes.js"; import { parseJavaScriptSource } from "./javascriptSourceParser.js"; -import { completeSemanticSteps } from "./javascriptSemanticTraversal.js"; +import { + completeSemanticSteps, + traverseJavaScriptAst, +} from "./javascriptSemanticTraversal.js"; it("indexes a real flat block once, retaining contextual guards in source order", () => { const parsed = parseJavaScriptSource(`{ @@ -94,3 +97,119 @@ it("charges nested exit checks to the same budget and does not cache a partial i entries: [{ fact: { kind: "branch", consequentExits: true } }], }); }); + +it("retains labelled-jump and budget semantics when scanning loop children", () => { + const parsed = parseJavaScriptSource(`guard: { + if (outer) { + while (again) { if (leave) break guard; } + throw 0; + } + if (inner) { + while (again) { local: { if (leave) break local; } } + throw 0; + } + send("tail"); + }`); + if (!parsed) throw new Error("Expected parsed program"); + const label = parsed.program.body[0]; + if (!t.isLabeledStatement(label) || !t.isBlockStatement(label.body)) + throw new Error("Expected labelled block"); + const block = label.body; + const tail = block.body.at(-1); + if (!tail) throw new Error("Expected terminal statement"); + const parents = new WeakMap(); + traverseJavaScriptAst(parsed.program, { + enter(node, parent) { + if (parent) parents.set(node, parent); + }, + }); + + let visits = 0; + const complete = new ElectronCallPathBlockPrefixes(() => { + visits += 1; + return true; + }, parents); + const result = completeSemanticSteps(complete.before(block, tail)); + expect(result).toMatchObject({ + before: 2, + entries: [ + { position: 0, fact: { kind: "branch", consequentExits: true } }, + { position: 1, fact: { kind: "branch", consequentExits: true } }, + ], + }); + const noParents = new ElectronCallPathBlockPrefixes(() => true); + const unknownLabels = completeSemanticSteps(noParents.before(block, tail)); + expect(unknownLabels).toMatchObject({ + entries: [ + { fact: { kind: "branch", consequentExits: false } }, + { fact: { kind: "branch", consequentExits: false } }, + ], + }); + // This count is characterized on the unchanged base: changing child + // ordering or budget charging must not silently shift the threshold. + expect(visits).toBe(35); + const requiredVisits = 35; + let remaining = requiredVisits - 1; + const limited = new ElectronCallPathBlockPrefixes( + () => --remaining >= 0, + parents, + ); + expect(completeSemanticSteps(limited.before(block, tail))).toBeUndefined(); + remaining = requiredVisits; + expect(completeSemanticSteps(limited.before(block, tail))).toEqual(result); + expect(remaining).toBeGreaterThanOrEqual(0); +}); + +it.each([ + [126, true], + [127, false], +])( + "retains the conservative loop-child depth boundary at %i wrappers", + (wrappers, exits) => { + let expression: t.Expression = t.identifier("condition"); + for (let index = 0; index < wrappers; index += 1) + expression = t.unaryExpression("!", expression); + const branch = t.ifStatement( + t.identifier("selected"), + t.blockStatement([ + t.whileStatement(expression, t.blockStatement([])), + t.throwStatement(t.numericLiteral(0)), + ]), + ); + const block = t.blockStatement([branch]); + const prefixes = new ElectronCallPathBlockPrefixes(() => true); + expect(prefixes.statement(branch, block)).toMatchObject({ + kind: "branch", + consequentExits: exits, + }); + }, +); + +it("does not charge or enter nested function and class bodies during loop scans", () => { + const inspect = (size: number) => { + const parsed = parseJavaScriptSource(`{ + if (selected) { + while (again) { + function nested() { ${"0;".repeat(size)} } + class Nested { method() { ${"0;".repeat(size)} } } + } + throw 0; + } + }`); + const block = parsed?.program.body[0]; + if (!t.isBlockStatement(block) || !t.isIfStatement(block.body[0])) + throw new Error("Expected parsed branch"); + let visits = 0; + const prefixes = new ElectronCallPathBlockPrefixes(() => { + visits += 1; + return true; + }); + const fact = prefixes.statement(block.body[0], block); + return { visits, fact }; + }; + const small = inspect(1); + const large = inspect(300); + expect(small.fact).toMatchObject({ kind: "branch", consequentExits: true }); + expect(large.fact).toMatchObject({ kind: "branch", consequentExits: true }); + expect(large.visits).toBe(small.visits); +}); diff --git a/src/domain/javascript/electronCallPathBlockPrefixes.ts b/src/domain/javascript/electronCallPathBlockPrefixes.ts index 8df1bc04e4..05887cee92 100644 --- a/src/domain/javascript/electronCallPathBlockPrefixes.ts +++ b/src/domain/javascript/electronCallPathBlockPrefixes.ts @@ -1,6 +1,8 @@ import * as t from "@babel/types"; import { childNodes } from "./javascriptSemanticTraversal.js"; +import { childNodes } from "./javascriptSemanticTraversal.js"; + /** Syntax alone can establish an exit, but never a branch's contextual truth. */ export type ElectronStatementFallthrough = | { readonly kind: "exit" } From 3640f015126a97f1d60fa9821301b1cd8d90f127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E4=BD=B3=E6=B5=A9Alben?= <35291372+XiaoBinGan@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:36:45 +0800 Subject: [PATCH 37/94] fix(javascript): emit no promise for a literal non-callable executor (#8264) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(javascript): emit no promise for a literal non-callable executor Node's `Promise` constructor calls the executor synchronously and throws a `TypeError` when it is not callable, before any promise object exists. The analysis reported a `constructor` promise operation with a hardcoded `sourceResolution: "complete"`, which the graph projects as `creates-promise` `resolved`. Measured on Node 24.22.1: new Promise() -> THREW TypeError new Promise(1) / (-1) -> THREW TypeError new Promise("work") / (`work`) -> THREW TypeError new Promise(true) / (!0) -> THREW TypeError new Promise(null) / (void 1) -> THREW TypeError new Promise(1n) / (/re/) -> THREW TypeError new Promise([]) / ({}) -> THREW TypeError --- accepted --- new Promise(() => {}) -> OK Promise new Promise(async () => {}) -> OK Promise new Promise(function () {}) -> OK Promise new Promise(class {}) -> OK Promise A class passes Node's own `typeof` check, so it keeps its operation. An identifier, a call result or a spread may still supply a function and also keeps its operation. The filter gates the candidate push rather than `promiseCandidateDetails`, which also decides whether an `await` becomes an `awaited-expression`: gating there made `await new Promise(1)` emit a spurious task fact, whereas gating at the push site makes it emit nothing, which is the honest answer. * chore: retrigger title and readiness checks after the title was corrected * fix(javascript): reject a non-finite executor, not only a literal greptile caught that `new Promise(1e309)` still emitted a promise. Babel stores that literal as `Infinity`, and `readExactJavaScriptLiteral` refuses to price a non-finite number, so the guard missed it. Measured on Node 24.22.1 — every one throws before a promise exists: new Promise(1e309) -> THREW TypeError new Promise(1e400) -> THREW TypeError new Promise(Infinity) -> THREW TypeError new Promise(-Infinity) -> THREW TypeError new Promise(NaN) -> THREW TypeError A numeric literal is now read directly rather than through the exact-value reader. `Infinity` and `NaN` are identifiers rather than literals, so they are decided through `isUnmodifiedGlobalReference`; a shadowed binding keeps its promise, which is the same scoping rule the timer fix uses. --- ...Analysis.rejected_promise_executor.test.ts | 90 +++++++++++++++++++ .../javascript/javascriptSemanticPromises.ts | 47 +++++++++- 2 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 src/domain/javascript/javascriptSemanticAnalysis.rejected_promise_executor.test.ts diff --git a/src/domain/javascript/javascriptSemanticAnalysis.rejected_promise_executor.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.rejected_promise_executor.test.ts new file mode 100644 index 0000000000..df055f1261 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticAnalysis.rejected_promise_executor.test.ts @@ -0,0 +1,90 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js"; + +/** + * Node's `Promise` constructor calls the executor synchronously and throws a + * `TypeError` when it is not callable — before any promise object exists. A + * provable literal that cannot be a function, or a missing argument, is + * therefore not a promise construction. + * + * Measured on Node 24.22.1: + * + * new Promise() / (1) / ("work") / (true) / (null) / (void 0) -> THREW TypeError + * new Promise(1n) / (/re/) / ([]) / ({}) -> THREW TypeError + * new Promise(() => {}) / (function () {}) / (class {}) -> OK Promise + */ +const promisesFor = (source: string) => + analyzeJavaScriptSemantics(source).promiseOperations.map( + ({ kind, method, sourceResolution }) => ({ + kind, + method, + sourceResolution, + }), + ); + +it.each([ + ["no argument", `new Promise()`], + ["a numeric executor", `new Promise(1)`], + ["a negative numeric executor", `new Promise(-1)`], + ["a string executor", `new Promise("work")`], + ["a template-literal executor", "new Promise(`work`)"], + ["a boolean executor", `new Promise(true)`], + ["a negated boolean executor", `new Promise(!0)`], + ["a null executor", `new Promise(null)`], + ["a void executor", `new Promise(void 1)`], + ["a BigInt executor", `new Promise(1n)`], + ["a RegExp executor", `new Promise(/re/)`], + // Babel stores an overflowing numeric literal as `Infinity`, which is not + // callable either. + ["an overflowing numeric executor", `new Promise(1e309)`], + ["an Infinity executor", `new Promise(Infinity)`], + ["a NaN executor", `new Promise(NaN)`], + ["an array executor", `new Promise([])`], + ["an object executor", `new Promise({})`], +] as const)("emits no promise for %s", (_name, source) => { + expect(promisesFor(source + ";")).toEqual([]); +}); + +it.each([ + ["an arrow executor", `new Promise(() => {})`], + ["an async arrow executor", `new Promise(async () => {})`], + ["a function expression executor", `new Promise(function () {})`], + ["a class expression executor", `new Promise(class {})`], +] as const)("keeps the promise for %s", (_name, source) => { + expect(promisesFor(source + ";")).toHaveLength(1); +}); + +it("keeps a promise whose executor is an identifier", () => { + expect(promisesFor(`const p = new Promise(run);`)).toEqual([ + { kind: "constructor", method: "new", sourceResolution: "complete" }, + ]); +}); + +it("keeps a promise whose executor is a call result", () => { + expect(promisesFor(`const p = new Promise(makeExecutor());`)).toHaveLength(1); +}); + +it("keeps a promise whose executor is a spread", () => { + expect(promisesFor(`const p = new Promise(...list);`)).toHaveLength(1); +}); + +/** + * Filtering happens at the candidate-push site rather than inside + * `promiseCandidateDetails`, which also decides whether `await` becomes an + * `awaited-expression`. A rejected executor must produce neither fact. + */ +it("emits no awaited-expression task for an awaited rejected executor", () => { + expect(promisesFor(`async function f() { await new Promise(1); }`)).toEqual( + [], + ); +}); + +it("keeps the awaited ownership for a callable executor under await", () => { + const operations = analyzeJavaScriptSemantics( + `async function f() { await new Promise(() => {}); }`, + ).promiseOperations; + expect(operations).toHaveLength(1); + expect(operations[0]?.kind).toBe("constructor"); + expect(operations[0]?.ownership).toBe("awaited"); +}); diff --git a/src/domain/javascript/javascriptSemanticPromises.ts b/src/domain/javascript/javascriptSemanticPromises.ts index 250285e458..08515e4e43 100644 --- a/src/domain/javascript/javascriptSemanticPromises.ts +++ b/src/domain/javascript/javascriptSemanticPromises.ts @@ -8,6 +8,7 @@ import type { import { semanticCallableIdForNode } from "./javascriptSemanticProjection.js"; import { createSemanticCallableIdStack } from "./javascriptSemanticCallableStack.js"; import { + readExactJavaScriptLiteral, semanticStaticPropertyName, unwrapJavaScriptExpression, } from "./javascriptAstValues.js"; @@ -100,6 +101,50 @@ export const collectJavaScriptSemanticPromises = ( }); }; +/** + * Node's `Promise` constructor calls the executor synchronously and throws a + * `TypeError` when it is not callable, before any promise object exists. An + * exactly readable literal that cannot be a function — or a missing argument — + * is therefore not a promise construction, while an identifier, a call result or + * a spread may still supply a function and keeps its operation. + * + * A class expression is callable enough to pass Node's own check, so it is kept. + */ +const rejectsPromiseExecutor = ( + node: t.Node, + state: JavaScriptSemanticAnalysisState, +): boolean => { + if (!t.isNewExpression(node)) return false; + const executor = node.arguments[0]; + if (executor === undefined) return true; + if (t.isSpreadElement(executor)) return false; + const value = unwrapJavaScriptExpression(executor).node; + if (t.isFunctionExpression(value) || t.isArrowFunctionExpression(value)) + return false; + if (t.isClassExpression(value)) return false; + // `NaN` and `Infinity` are unshadowed globals rather than literals, and neither + // is callable. + if ( + t.isIdentifier(value) && + (value.name === "NaN" || value.name === "Infinity") && + isUnmodifiedGlobalReference(value, state, value.name) + ) + return true; + // Every unary operator yields a primitive, and a primitive is never callable. + // A numeric literal is read directly rather than through the exact-value + // reader, which refuses to price a non-finite literal such as `1e309`. + return ( + t.isUnaryExpression(value) || + t.isNumericLiteral(value) || + readExactJavaScriptLiteral(value).found || + t.isTemplateLiteral(value) || + t.isBigIntLiteral(value) || + t.isRegExpLiteral(value) || + t.isArrayExpression(value) || + t.isObjectExpression(value) + ); +}; + const collectCandidates = ( program: t.Program, state: JavaScriptSemanticAnalysisState, @@ -115,7 +160,7 @@ const collectCandidates = ( enter: (node, parent) => { callableStack.enter(node, parent); const details = promiseCandidateDetails(node, state); - if (details !== null) { + if (details !== null && !rejectsPromiseExecutor(node, state)) { output.push({ node, ...details, From 094ec5eb3fbdb0c99e0b1950c6412d037b8f4501 Mon Sep 17 00:00:00 2001 From: STDquantum <60764289+STDquantum@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:37:31 +0800 Subject: [PATCH 38/94] fix(cli): reject missing mcp add option values (#7996) * fix(cli): reject mcp add options missing their values (#7609) * fix(cli): reject mcp add values swallowed by the next flag (#7609) * fix(cli): reject empty mcp add -c value (#7609) * test(cli): verify rejected mcp add preserves settings * chore: keep main formatting out of merge diff --- src/cli.ts | 61 +++++++++++++ .../cli/mcpAddOptionValidation.test.ts | 91 +++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 tests/boundary/cli/mcpAddOptionValidation.test.ts diff --git a/src/cli.ts b/src/cli.ts index 84b7b13522..0203627b9b 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -303,6 +303,67 @@ export const createCli = ( )(1); return Promise.resolve(); } + // Incur's `mcp add` silently skips a valued option with no value and + // swallows the next flag as the value: the registration succeeds without + // the requested agent or command. Reject a trailing --agent/--command/-c, + // a value position occupied by another flag, and an empty `--flag=` form + // up front. + const valued = ["--agent", "--command", "-c"]; + let missingValue: string | undefined; + for (let index = 0; index < command.length; index += 1) { + const next = command[index + 1]; + if ( + valued.includes(command[index]!) && + (next === undefined || next.startsWith("-")) + ) { + missingValue = command[index]; + break; + } + } + const emptyValue = command.find( + (token) => + (token.startsWith("--agent=") || + token.startsWith("--command=") || + token.startsWith("-c=")) && + token.length === token.indexOf("=") + 1, + ); + if (missingValue !== undefined || emptyValue !== undefined) { + const flag = missingValue ?? emptyValue!; + const message = `Missing value for flag: ${flag.replace(/=.*$/, "")}`; + const field = + flag.includes("command") || flag === "-c" || flag.startsWith("-c=") + ? "command" + : "agent"; + const output = parseCliOutputArguments(normalized); + if (!output.validation.ok) + write(renderCliOutputArgumentError(output.validation)); + else + write( + renderCliDocument( + { + code: "VALIDATION_ERROR", + message, + fieldErrors: [ + { + code: "invalid_value", + missing: false, + path: field, + message, + }, + ], + }, + output.format, + process.stdout.isTTY === true, + ), + ); + ( + options?.exit ?? + ((code: number) => { + process.exitCode = code; + }) + )(1); + return Promise.resolve(); + } // Incur's mcp add parser only understands spaced `--agent amp` forms; an // equals form like `--agent=amp` is silently ignored, which would drop // the requested agent and register against the default target. Normalize diff --git a/tests/boundary/cli/mcpAddOptionValidation.test.ts b/tests/boundary/cli/mcpAddOptionValidation.test.ts new file mode 100644 index 0000000000..f9ef555588 --- /dev/null +++ b/tests/boundary/cli/mcpAddOptionValidation.test.ts @@ -0,0 +1,91 @@ +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest("mcp add rejects valued options without a value", async ({ cli }) => { + const root = await createTestTempDirectory("rea-mcp-add-value-"); + const home = join(root, "home"); + const settingsPath = (selectedHome: string) => + join(selectedHome, ".config", "amp", "settings.json"); + const run = (arguments_: readonly string[], selectedHome = home) => + cli.run({ + arguments: [...arguments_, "--json"], + cwd: root, + environment: { + HOME: selectedHome, + USERPROFILE: selectedHome, + XDG_CONFIG_HOME: join(selectedHome, ".config"), + }, + timeoutMs: 20_000, + }); + + for (const [flag, field] of [ + ["--agent", "agent"], + ["--command", "command"], + ["-c", "command"], + ["-c=", "command"], + ["--agent=", "agent"], + ] as const) { + const result = await run(["mcp", "add", flag]); + expect(result.exitCode, result.stderr).toBe(1); + expect(result.json).toMatchObject({ + code: "VALIDATION_ERROR", + fieldErrors: expect.arrayContaining([ + expect.objectContaining({ + code: "invalid_value", + missing: false, + path: field, + }), + ]), + }); + await expect(access(settingsPath(home))).rejects.toMatchObject({ + code: "ENOENT", + }); + } + + // Incur would swallow the following flag as the option value; REA rejects it. + const swallowed = await run(["mcp", "add", "--command", "--agent", "amp"]); + expect(swallowed.exitCode, swallowed.stderr).toBe(1); + expect(swallowed.json).toMatchObject({ + code: "VALIDATION_ERROR", + fieldErrors: expect.arrayContaining([ + expect.objectContaining({ + code: "invalid_value", + missing: false, + path: "command", + }), + ]), + }); + await expect(access(settingsPath(home))).rejects.toMatchObject({ + code: "ENOENT", + }); + + const existingHome = join(root, "existing-home"); + const existingSettingsPath = settingsPath(existingHome); + await mkdir(join(existingHome, ".config", "amp"), { recursive: true }); + const existingSettings = '{"existing":true}\n'; + await writeFile(existingSettingsPath, existingSettings); + const rejectedWithExistingSettings = await run( + ["mcp", "add", "--agent"], + existingHome, + ); + expect( + rejectedWithExistingSettings.exitCode, + rejectedWithExistingSettings.stderr, + ).toBe(1); + expect(rejectedWithExistingSettings.json).toMatchObject({ + code: "VALIDATION_ERROR", + }); + expect(await readFile(existingSettingsPath, "utf8")).toBe(existingSettings); + + // A registration with an explicit agent still writes the Amp registration. + const registered = await run(["mcp", "add", "--agent", "amp"]); + expect(registered.exitCode, registered.stderr).toBe(0); + const settings = JSON.parse(await readFile(settingsPath(home), "utf8")); + expect(settings["amp.mcpServers"].rea).toMatchObject({ + command: expect.any(String), + }); +}); From 2ae7878fc7ce6aa40e6343ba5d1bcb81dbe0c4a0 Mon Sep 17 00:00:00 2001 From: STDquantum <60764289+STDquantum@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:38:15 +0800 Subject: [PATCH 39/94] fix(native): retain all breakpoint locations within budget (#7975) * fix(native): retain all breakpoint locations within budget * fix(native): describe incomplete locations accurately * test(native): cover incomplete location warning * test(native): bind lipo failure to a real fixture * test(native): use valid Mach-O fixture for lipo errors * test(native): drop fixture change already in main * test(native): share breakpoint limitation wording with formatter * test(native): assert limitation delivery via owner constants * style(native): format breakpoint limitation projection --- bridge/native/rea_lldb_tracer.py | 28 +++---- bridge/native/test_rea_lldb_tracer.py | 28 +++++++ src/native/NativeCallObservation.ts | 16 ++-- .../native/nativeCallObservation.test.ts | 83 ++++++++++++++++++- 4 files changed, 132 insertions(+), 23 deletions(-) diff --git a/bridge/native/rea_lldb_tracer.py b/bridge/native/rea_lldb_tracer.py index dccceb56fe..11e365b8cc 100644 --- a/bridge/native/rea_lldb_tracer.py +++ b/bridge/native/rea_lldb_tracer.py @@ -26,7 +26,6 @@ ARGUMENT_REGISTERS = { "x86_64h": ["rdi", "rsi", "rdx", "rcx", "r8", "r9"], } MAX_SELECTOR_BYTES = 1024 -MAX_REPORTED_LOCATIONS = 64 MAX_OTHER_STOPS = 256 MAX_TRACE_BYTES = 8 * 1024 * 1024 MAX_TRACE_FRAMES = 65_536 @@ -505,20 +504,19 @@ def _breakpoint_report(target, created, config): if not _wanted(config["breakpoints"][index], location["symbol"] or ""): continue count += 1 - if len(locations) < MAX_REPORTED_LOCATIONS: - remaining_bytes = ( - MAX_BREAKPOINT_LOCATION_BYTES - retained_location_bytes - ) - if not _fits_trace_budget(location, remaining_bytes): - locations_truncated = True - continue - location_bytes = len(json.dumps(location).encode("utf-8")) - addition = location_bytes + (1 if locations else 0) - if addition > remaining_bytes: - locations_truncated = True - continue - locations.append(location) - retained_location_bytes += addition + remaining_bytes = ( + MAX_BREAKPOINT_LOCATION_BYTES - retained_location_bytes + ) + if not _fits_trace_budget(location, remaining_bytes): + locations_truncated = True + continue + location_bytes = len(json.dumps(location).encode("utf-8")) + addition = location_bytes + (1 if locations else 0) + if addition > remaining_bytes: + locations_truncated = True + continue + locations.append(location) + retained_location_bytes += addition report.append({"index": index, "location_count": count, "locations": locations}) return report, locations_truncated diff --git a/bridge/native/test_rea_lldb_tracer.py b/bridge/native/test_rea_lldb_tracer.py index 9cce55ff28..e1c3e66e97 100644 --- a/bridge/native/test_rea_lldb_tracer.py +++ b/bridge/native/test_rea_lldb_tracer.py @@ -430,6 +430,34 @@ class NativeTraceBudgetTests(unittest.TestCase): self.assertEqual(report[0]["locations"], []) self.assertFalse(locations_truncated) + def test_breakpoint_report_keeps_all_locations_under_aggregate_budget(self): + class Location: + def __init__(self, index): + self.index = index + + def GetAddress(self): + return self.index + + class Breakpoint: + def GetNumLocations(self): + return 65 + + def GetLocationAtIndex(self, index): + return Location(index) + + with patch.object( + tracer, + "_location", + side_effect=lambda address, _target: {"address": address, "symbol": None}, + ), patch.object(tracer, "_wanted", return_value=True): + report, locations_truncated = tracer._breakpoint_report( + object(), [[Breakpoint()]], {"breakpoints": [{}]} + ) + + self.assertEqual(report[0]["location_count"], 65) + self.assertEqual(len(report[0]["locations"]), 65) + self.assertFalse(locations_truncated) + def test_breakpoint_report_bounds_aggregate_location_json(self): tracer.lldb.LLDB_INVALID_ADDRESS = -1 symbol_name = ( diff --git a/src/native/NativeCallObservation.ts b/src/native/NativeCallObservation.ts index 494dab21fc..bbba6c1a6d 100644 --- a/src/native/NativeCallObservation.ts +++ b/src/native/NativeCallObservation.ts @@ -40,6 +40,11 @@ export const NATIVE_CALL_OBSERVATION_LIMITATIONS = [ "The observation window is checked about once a second, and recording pauses the process at every entry, which slows it.", ] as const; +export const NATIVE_CALL_BREAKPOINT_LOCATION_LIST_INCOMPLETE_LIMITATION = + "The breakpoint location list is incomplete; location_count gives the full matching count."; +export const NATIVE_CALL_BREAKPOINT_LOCATION_BUDGET_LIMITATION = + "Resolved breakpoint locations exceeded the aggregate 8388608-byte (8 MiB) metadata budget; omitted locations are counted in location_count. This metadata limit does not change the process outcome. This release has no runtime override."; + /** Launch the active Mach-O under LLDB, record requested call entries, then stop it. */ export const observeNativeCalls = async ( target: BinaryTarget, @@ -216,16 +221,13 @@ const nativeCallFacts = ( `Breakpoints ${unresolved.join(", ")} matched no code in any image loaded while observing; their calls could not be observed.`, ]), ...(run.breakpoints.some( - ({ location_count: count, locations }) => locations.length < count, + ({ location_count: count, locations }) => + locations.length < count && !breakpointLocationLimit, ) - ? [ - "Breakpoints with many matching locations list only the first 64; location_count gives the total.", - ] + ? [NATIVE_CALL_BREAKPOINT_LOCATION_LIST_INCOMPLETE_LIMITATION] : []), ...(breakpointLocationLimit - ? [ - "Resolved breakpoint locations exceeded the aggregate 8388608-byte (8 MiB) metadata budget; omitted locations are counted in location_count. This metadata limit does not change the process outcome. This release has no runtime override.", - ] + ? [NATIVE_CALL_BREAKPOINT_LOCATION_BUDGET_LIMITATION] : []), ...(trace.stdout === null || trace.stderr === null ? ["One or more target output captures were unavailable."] diff --git a/tests/boundary/providers/native/nativeCallObservation.test.ts b/tests/boundary/providers/native/nativeCallObservation.test.ts index beea50d320..cdb2897256 100644 --- a/tests/boundary/providers/native/nativeCallObservation.test.ts +++ b/tests/boundary/providers/native/nativeCallObservation.test.ts @@ -13,6 +13,10 @@ import type { AnalysisError } from "../../../../src/domain/analysisErrorBase.js" import { EvidenceIntegrityError } from "../../../../src/domain/evidenceErrors.js"; import { nativeCallPartialObservationSchema } from "../../../../src/domain/native/nativeCallPartialObservation.js"; import { nativeCallObservationResultSchema } from "../../../../src/domain/native/nativeCallObservation.js"; +import { + NATIVE_CALL_BREAKPOINT_LOCATION_BUDGET_LIMITATION, + NATIVE_CALL_BREAKPOINT_LOCATION_LIST_INCOMPLETE_LIMITATION, +} from "../../../../src/native/NativeCallObservation.js"; import { err, ok, type Result } from "../../../../src/domain/result.js"; import type { NativeCallTrace, @@ -216,6 +220,80 @@ describe("observe_native_calls projection: projects completed observations", () }); }); +describe("observe_native_calls projection: keeps locations within the metadata budget", () => { + it("projects every breakpoint location when the aggregate metadata budget allows it", async () => { + const locations = Array.from({ length: 65 }, (_, index) => ({ + ...LOCATION, + load_address: `0x${(0x100000928 + index * 4).toString(16)}`, + file_address: `0x${(0x100000928 + index * 4).toString(16)}`, + })); + const observed = await observe( + new FixtureTracer( + ok({ + debugger: { + path: "/usr/bin/lldb", + sha256: "a".repeat(64), + version: null, + }, + run: run({ + breakpoints: [ + { index: 0, location_count: locations.length, locations }, + { index: 1, location_count: 1, locations: [LOCATION] }, + ], + }), + stdout: { text: "", bytes: 0, truncated: false, complete: true }, + stderr: { text: "", bytes: 0, truncated: false, complete: true }, + terminated: true, + }), + ), + { breakpoints: BREAKPOINTS }, + ); + if (!observed.ok) throw observed.error; + const result = nativeCallObservationResultSchema.parse( + observed.value.result, + ); + expect(result.breakpoints[0]?.location_count).toBe(65); + expect(result.breakpoints[0]?.locations).toHaveLength(65); + expect(result.coverage).toEqual({ + status: "complete", + event_limit_reached: false, + resource_limit_reached: false, + unresolved_breakpoints: [], + }); + }); + + it("uses a generic warning when locations are missing without a budget signal", async () => { + const observed = await observe( + new FixtureTracer( + ok({ + debugger: { + path: "/usr/bin/lldb", + sha256: "a".repeat(64), + version: null, + }, + run: run({ + breakpoints: [ + { index: 0, location_count: 2, locations: [LOCATION] }, + { index: 1, location_count: 1, locations: [LOCATION] }, + ], + }), + stdout: { text: "", bytes: 0, truncated: false, complete: true }, + stderr: { text: "", bytes: 0, truncated: false, complete: true }, + terminated: true, + }), + ), + { breakpoints: BREAKPOINTS }, + ); + if (!observed.ok) throw observed.error; + const result = nativeCallObservationResultSchema.parse( + observed.value.result, + ); + expect(result.limitations).toContain( + NATIVE_CALL_BREAKPOINT_LOCATION_LIST_INCOMPLETE_LIMITATION, + ); + }); +}); + describe("observe_native_calls projection: reports partial observation coverage", () => { it("retains normal exit and location counts when breakpoint metadata is bounded", async () => { const observed = await observe( @@ -259,7 +337,10 @@ describe("observe_native_calls projection: reports partial observation coverage" unresolved_breakpoints: [], }); expect(result.limitations).toContain( - "Resolved breakpoint locations exceeded the aggregate 8388608-byte (8 MiB) metadata budget; omitted locations are counted in location_count. This metadata limit does not change the process outcome. This release has no runtime override.", + NATIVE_CALL_BREAKPOINT_LOCATION_BUDGET_LIMITATION, + ); + expect(result.limitations).not.toContain( + NATIVE_CALL_BREAKPOINT_LOCATION_LIST_INCOMPLETE_LIMITATION, ); expect(result.limitations.join(" ")).not.toContain("process was killed"); }); From 9cc34178c00cea6fb5f2938b4d7f102d2f83db05 Mon Sep 17 00:00:00 2001 From: Wenyu Chiou <162016108+WenyuChiou@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:38:59 -0500 Subject: [PATCH 40/94] test(native): bind lipo refusal fixture to a real source (#7977) --- src/native/NativeMacOSProviderInput.test.ts | 64 ++++++++++++++++----- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/src/native/NativeMacOSProviderInput.test.ts b/src/native/NativeMacOSProviderInput.test.ts index 34c855ff64..e588f5c12d 100644 --- a/src/native/NativeMacOSProviderInput.test.ts +++ b/src/native/NativeMacOSProviderInput.test.ts @@ -1,7 +1,8 @@ -import { describe, expect, it } from "vitest"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; + import { nativeMacOSCapabilities } from "./NativeMacOSProviderMetadata.js"; import { NativeMacOSProvider } from "./NativeMacOSProvider.js"; import type { JsonValue } from "../domain/jsonValue.js"; @@ -101,22 +102,59 @@ it.each(["inspect_macho", "list_architectures"] as const)( const root = await createTestTempDirectory("rea-macho-lipo-output-"); const path = join(root, "fixture.macho"); await writeFile(path, inspectionImage()); - const client = new NativeMacOSProvider( - {}, - new NativeFixtureRunner({ lipo: "unrecognized slice record\n" }), - "darwin", - ).createClient(await nativeMachoTargetForFile(path)); - const result = await client.execute(operation, {}); - expect(result).toMatchObject({ - ok: false, - error: { - _tag: "AnalysisOutputError", - capturedOutput: { stdout: "unrecognized slice record\n", exit_code: 0 }, - }, + const runner = new NativeFixtureRunner({ + lipo: "unrecognized slice record\n", }); + const run = vi.spyOn(runner, "run"); + const client = new NativeMacOSProvider({}, runner, "darwin").createClient( + await nativeMachoTargetForFile(path), + ); + try { + const result = await client.execute(operation, {}); + expect(run).toHaveBeenCalledWith( + "lipo", + ["-detailed_info", path], + expect.any(Object), + ); + expect(run).toHaveBeenCalledTimes(1); + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisOutputError", + operation, + capturedOutput: { + stdout: "unrecognized slice record\n", + exit_code: 0, + }, + }, + }); + } finally { + expect((await client.close()).ok).toBe(true); + } }, ); +it("refuses a missing Mach-O source before running lipo", async () => { + const root = await createTestTempDirectory("rea-native-lipo-missing-"); + const runner = new NativeFixtureRunner({ + lipo: "unrecognized slice record\n", + }); + const run = vi.spyOn(runner, "run"); + const client = new NativeMacOSProvider({}, runner, "darwin").createClient( + nativeMachoTarget(join(root, "missing")), + ); + try { + const result = await client.execute("inspect_macho", {}); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "ProviderAdapterError" }, + }); + expect(run).not.toHaveBeenCalled(); + } finally { + expect((await client.close()).ok).toBe(true); + } +}); + it("declares the signature snapshot write in provider capabilities", () => { expect( nativeMacOSCapabilities("darwin").find( From f285fd4f8209656192f1511c453f9d70b663d995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E4=BD=B3=E6=B5=A9Alben?= <35291372+XiaoBinGan@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:39:43 +0800 Subject: [PATCH 41/94] fix(javascript): emit no child process for a literal non-object options slot (#8019) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(javascript): emit no child process for a literal non-object options slot Node validates the child_process options object before creating anything, so a literal primitive or a literal array in that slot throws ERR_INVALID_ARG_TYPE synchronously and no child process exists. The analysis still reported a complete spawn for them. Measured on Node 24.22.1: spawn(cmd, "status") -> THROWS ERR_INVALID_ARG_TYPE spawn(cmd, [], "pipe") -> THROWS ERR_INVALID_ARG_TYPE spawn(cmd, [], null) -> THROWS ERR_INVALID_ARG_TYPE spawn(cmd, [], []) -> THROWS ERR_INVALID_ARG_TYPE execFile(cmd, "status") -> THROWS ERR_INVALID_ARG_TYPE fork(worker, "status") -> THROWS ERR_INVALID_ARG_TYPE fork(worker, [], "pipe") -> THROWS ERR_INVALID_ARG_TYPE --- accepted by Node --- spawn(cmd, args, undefined) -> SPAWNED spawn(cmd, args, {}) -> SPAWNED execFile(cmd, [], null) -> SPAWNED fork(worker, [], null) -> SPAWNED exec(cmd, "inherit") -> SPAWNED The asymmetries matter: `spawn` rejects `null` and exempts only `undefined`, while `fork` and `execFile` skip any nullish options, and `exec` spreads its options and validates none of them. Drop a candidate only when the options slot is a provable literal primitive or array, so an unresolved or non-literal value stays a candidate. `spawn(cmd, [], null)` is the sharpest case: it was indistinguishable from the valid `undefined` form, including a resolved connects-stdio relation, for a call that dies before any child exists. * chore: retrigger title and readiness checks after the title was corrected * fix(javascript): reject literal options Node shall reject, and drop a stray file Two review findings on #8019. greptile: `rejectsOptionsShape` still kept `spawn("tool", [], 0n)`, `-0` and `1e400`, which Node also rejects before creating a child (`ERR_INVALID_ARG_TYPE: The "options" argument must be of type object`). Those forms are exactly the ones `readExactJavaScriptLiteral` refuses to price — a BigInt, a non-finite numeric literal and a negated numeric literal — so read the shape directly instead of widening the shared exact-value reader. `NaN` is an identifier rather than a literal, so it still needs scope resolution and is left as a documented gap. Also remove `.rea-hopper-stage-YOryZ6/`, a local Hopper installation stub that an `add -A` swept into the commit. It is unrelated to this change. * fix(javascript): keep a void options slot, reject ! and ~ greptile flagged that `isUnpricedPrimitiveLiteral` dropped `spawn("tool", [], void 0)`, `fork(..., void 0)` and `execFile(..., void 0)`. It checked the unary operand without the operator, but `void ` evaluates to `undefined`, which is the one options value Node accepts in place of an object. Measured on Node 24.22.1: spawn(cmd, [], void 0) -> SPAWNED spawn(cmd, [], void 5) -> SPAWNED spawn(cmd, [], -0) -> THROWS ERR_INVALID_ARG_TYPE spawn(cmd, [], -5) -> THROWS ERR_INVALID_ARG_TYPE spawn(cmd, [], +5) -> THROWS ERR_INVALID_ARG_TYPE spawn(cmd, [], !0) -> THROWS ERR_INVALID_ARG_TYPE spawn(cmd, [], ~0) -> THROWS ERR_INVALID_ARG_TYPE So the operator now decides: `void` keeps the spawn, `!` and `~` reject whatever their operand, and `+`/`-` reject only when the operand is a numeric literal. * fix(javascript): reject a non-finite global in the child-process options slot An unshadowed `NaN` or `Infinity` in the options slot is a number, and Node's options check rejects a number exactly as it rejects any other primitive. The guard read only literal nodes, so the global identifier passed through. Measured on Node 24.22.1: spawn("e", NaN) -> THREW ERR_INVALID_ARG_TYPE spawn("e", Infinity) -> THREW ERR_INVALID_ARG_TYPE execFile("e", NaN) -> THREW ERR_INVALID_ARG_TYPE fork("x.js", NaN) -> THREW ERR_INVALID_ARG_TYPE --- accepted --- exec("e", NaN) -> ok, because exec validates no options spawn("e", undefined) -> ok, the one accepted global spawn("e", null) -> ok spawn("e", []) / ({}) -> ok `isUnshadowedGlobal` keeps a local binding named `NaN` out of the rule, so a shadowed value keeps its fact. Seven regressions cover the rejected forms plus `exec`, `undefined`, `null`, an array, an object and a shadowed `NaN`; all seven fail on the previous commit. --- ...alysis.child_process_options_shape.test.ts | 109 ++++++++++++++++++ .../javascriptSemanticChildProcesses.ts | 56 ++++++++- 2 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 src/domain/javascript/javascriptSemanticAnalysis.child_process_options_shape.test.ts diff --git a/src/domain/javascript/javascriptSemanticAnalysis.child_process_options_shape.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.child_process_options_shape.test.ts new file mode 100644 index 0000000000..9cedf23661 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticAnalysis.child_process_options_shape.test.ts @@ -0,0 +1,109 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js"; + +/** + * Node validates the child_process options object before creating anything, so a + * literal primitive or a literal array in that slot throws + * ERR_INVALID_ARG_TYPE synchronously and no child process exists. `spawn` also + * rejects `null` (`undefined` is its only exemption); `fork` and `execFile` skip + * any nullish options, and `exec` spreads its options and rejects none. + */ +const spawnsFor = (source: string) => + analyzeJavaScriptSemantics( + `import { spawn, exec, execFile, fork } from "node:child_process"; ${source};`, + ).childProcessSpawns; + +it.each([ + ["a string args slot", `spawn("git", "status")`], + ["a numeric args slot", `spawn("git", 1)`], + ["a string options slot", `spawn("git", [], "pipe")`], + ["a null options slot", `spawn("git", [], null)`], + ["an array options slot", `spawn("git", [], [])`], + ["a numeric options slot", `spawn("git", [], 5)`], + ["a string execFile args slot", `execFile("git", "status")`], + ["a string fork args slot", `fork("worker.js", "status")`], + ["a string fork options slot", `fork("worker.js", [], "pipe")`], + ["a BigInt options slot", `spawn("git", [], 0n)`], + ["a negated-zero options slot", `spawn("git", [], -0)`], + ["an overflowing numeric options slot", `spawn("git", [], 1e400)`], + ["a negated numeric options slot", `spawn("git", [], -5)`], + ["a BigInt args slot", `spawn("git", 0n)`], +] as const)("emits no child-process fact for %s", (_name, call) => { + expect(spawnsFor(call)).toEqual([]); +}); + +it.each([ + ["an object options slot", `spawn("tool", [], { stdio: "pipe" })`], + ["an omitted options slot", `spawn("tool", [], undefined)`], + ["options in the second slot", `spawn("tool", { stdio: "pipe" })`], + ["an unresolved options value", `spawn("tool", [], options)`], + // `void ` evaluates to undefined, the one value Node accepts here. + ["a void options slot", `spawn("tool", [], void 0)`], + ["a void expression with an operand", `spawn("tool", [], void 5)`], +] as const)("keeps the spawn for %s", (_name, call) => { + expect(spawnsFor(call)).toHaveLength(1); +}); + +/** fork and execFile skip a nullish options object; exec validates nothing. */ +it.each([ + ["fork with null options", `fork("worker.js", [], null)`], + ["fork with a void options slot", `fork("worker.js", [], void 0)`], + ["execFile with null options", `execFile("tool", [], null)`], + ["execFile with a void options slot", `execFile("tool", [], void 0)`], + ["exec with a string second argument", `exec("tool", "inherit")`], +] as const)("keeps %s", (_name, call) => { + expect(spawnsFor(call)).toHaveLength(1); +}); + +/** `!` and `~` yield a primitive whatever their operand, so Node rejects them. */ +it.each([ + ["a logical-not options slot", `spawn("tool", [], !0)`], + ["a bitwise-not options slot", `spawn("tool", [], ~5)`], + ["a unary-plus options slot", `spawn("tool", [], +5)`], +] as const)("emits no child-process fact for %s", (_name, call) => { + expect(spawnsFor(call)).toEqual([]); +}); + +/** + * An unshadowed `NaN` or `Infinity` in the options slot is a number, and Node + * rejects a number there exactly as it rejects any other primitive: + * + * spawn("e", NaN) -> THREW ERR_INVALID_ARG_TYPE + * spawn("e", Infinity) -> THREW ERR_INVALID_ARG_TYPE + * execFile("e", NaN) -> THREW ERR_INVALID_ARG_TYPE + * fork("x.js", NaN) -> THREW ERR_INVALID_ARG_TYPE + * exec("e", NaN) -> ok, because exec validates no options + * spawn("e", undefined) -> ok, the one accepted global + */ +it.each([ + ["a NaN options slot for spawn", `spawn("tool", NaN)`], + ["an Infinity options slot for spawn", `spawn("tool", Infinity)`], + ["a NaN third-slot options for spawn", `spawn("tool", [], NaN)`], + ["a NaN options slot for execFile", `execFile("tool", NaN)`], + ["an Infinity options slot for execFile", `execFile("tool", Infinity)`], + ["a NaN options slot for fork", `fork("tool.js", NaN)`], + ["an Infinity options slot for fork", `fork("tool.js", Infinity)`], +] as const)("emits no child-process fact for %s", (_name, call) => { + expect(spawnsFor(call)).toEqual([]); +}); + +it.each([ + ["exec not validating its NaN options", `exec("tool", NaN)`, "exec"], + [ + "exec not validating its Infinity options", + `exec("tool", Infinity)`, + "exec", + ], + ["spawn accepting undefined", `spawn("tool", undefined)`, "spawn"], + ["spawn accepting an object first slot", `spawn("tool", {})`, "spawn"], + ["spawn accepting an array", `spawn("tool", [])`, "spawn"], +] as const)("keeps the child-process fact for %s", (_name, call, method) => { + expect(spawnsFor(call).map(({ method: actual }) => actual)).toEqual([method]); +}); + +it("keeps the fact when a local binding shadows NaN", () => { + expect( + spawnsFor(`const NaN = {}; spawn("tool", NaN)`).map(({ method }) => method), + ).toEqual(["spawn"]); +}); diff --git a/src/domain/javascript/javascriptSemanticChildProcesses.ts b/src/domain/javascript/javascriptSemanticChildProcesses.ts index ea7fb69ab9..3b3ad852bd 100644 --- a/src/domain/javascript/javascriptSemanticChildProcesses.ts +++ b/src/domain/javascript/javascriptSemanticChildProcesses.ts @@ -73,7 +73,9 @@ export const collectJavaScriptSemanticChildProcesses = ( ): ChildProcessAnalysis => { const context = createSemanticExpressionShapeResolver(state); const candidates = collectSpawns(program, state, callables).filter( - (candidate) => !forkRejectsStdioArray(candidate, context), + (candidate) => + !forkRejectsStdioArray(candidate, context) && + !rejectsOptionsShape(candidate, context, state), ); const childProcessSpawns = candidates.map((candidate) => immutableSpawn(candidate, context), @@ -485,6 +487,58 @@ const forkRejectsStdioArray = ( return ipcCount !== 1; }; +/** + * Node validates the options object before creating anything, so a literal + * primitive or literal array in that slot throws ERR_INVALID_ARG_TYPE + * synchronously and no child process exists. `spawn` also rejects `null` + * (`undefined` is its only exemption), while `fork` and `execFile` skip any + * nullish options, and `exec` spreads its options and validates none of them. + */ +const rejectsOptionsShape = ( + candidate: SpawnCandidate, + context: ChildSpawnContext, + state: JavaScriptSemanticAnalysisState, +): boolean => { + if (candidate.method === "exec") return false; + const { options } = spawnArguments(candidate, context); + if (options === undefined) return false; + const value = unwrapJavaScriptExpression(options).node; + if (context.isArray(value)) return true; + const literal = readExactJavaScriptLiteral(value); + if (literal.found) + return !(candidate.method !== "spawn" && literal.value === null); + // An unshadowed `NaN` or `Infinity` is a number, and Node's options check + // rejects a number the same way it rejects any other primitive. `undefined` + // stays the one accepted global. + if ( + t.isIdentifier(value) && + (value.name === "NaN" || value.name === "Infinity") && + isUnshadowedGlobal(value, state, value.name) + ) + return true; + return isUnpricedPrimitiveLiteral(value); +}; + +/** + * The exact-value reader deliberately refuses to price a BigInt, a non-finite + * numeric literal such as `1e400`, and a negated numeric literal such as `-0`. + * All are primitive literals, so Node's options check rejects them the same way; + * read the shape directly here rather than widening the shared reader. + * + * `void ` is the exception: it evaluates to `undefined`, which is the one + * options value Node accepts in place of an object, so it must not be dropped. + */ +const isUnpricedPrimitiveLiteral = (value: t.Node): boolean => { + if (t.isNumericLiteral(value) || t.isBigIntLiteral(value)) return true; + if (!t.isUnaryExpression(value)) return false; + if (value.operator === "void") return false; + // `!` and `~` always yield a primitive, whatever their operand. + if (value.operator === "!" || value.operator === "~") return true; + if (value.operator !== "+" && value.operator !== "-") return false; + const argument = unwrapJavaScriptExpression(value.argument).node; + return t.isNumericLiteral(argument) || t.isBigIntLiteral(argument); +}; + /** * Without a string or array `stdio`, Node's `fork` pipes the standard streams * when `silent` is truthy and inherits them otherwise. From 54146627f04964ba26b9becbfd88b40fa5831ea7 Mon Sep 17 00:00:00 2001 From: Beqa <91750923+BeqaKs@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:40:27 +0400 Subject: [PATCH 42/94] fix(cli): emit compact JSON validation errors for demangle-swift (#7622) * fix(cli): emit compact JSON validation errors for demangle-swift Piped and redirected --json argument diagnostics went through Incur's always-indented Formatter, bypassing REA's destination-aware writer. Buffer non-TTY SDK JSON like JSONL and rewrite through writeJsonOutput. * fix(cli): keep token-count for piped SDK JSON documents Compact JSON rewrite for non-TTY SDK output returned early before token controls, so --llms/--schema --format json --token-count printed the document instead of its count. * fix(cli): do not token-count non-document JSON captures Completion hooks and other non-JSON --json pipe output must stay verbatim; token-count on non-JSON SDK captures applies only with sdkDocumentControls. * test(cli): assert demangle JSON validation by code, not wording --- src/cli/cliResultOutput.ts | 44 +++++++- tests/boundary/cli/jsonOutput.test.ts | 102 +++++++++++++++++++ tests/boundary/cli/nativeDemangleCli.test.ts | 23 +++++ 3 files changed, 168 insertions(+), 1 deletion(-) diff --git a/src/cli/cliResultOutput.ts b/src/cli/cliResultOutput.ts index d3bde08fb4..4ea5c5fe87 100644 --- a/src/cli/cliResultOutput.ts +++ b/src/cli/cliResultOutput.ts @@ -303,7 +303,16 @@ export const createCliResultOutput = ( streaming = true; }, captureSdkOutput(text) { - if ((format !== "jsonl" && !sdkDocumentControls) || streaming || handled) + // JSONL always buffers for framing. Piped/redirected JSON also buffers so + // Incur's always-indented Formatter output can be rewritten compactly. + // Terminal JSON stays with Incur, which already matches the indent contract. + // sdkDocumentControls also buffers one-shot SDK documents for token windows. + const captureJson = format === "json" && !isTerminalOutput(destination); + if ( + (format !== "jsonl" && !sdkDocumentControls && !captureJson) || + streaming || + handled + ) return false; // Incur already materializes one-shot documents. Retain references only; // generator events bypass this buffer and remain incremental. @@ -321,6 +330,39 @@ export const createCliResultOutput = ( await writeOutputText(destination, ""); return; } + if (format === "json") { + const text = parts.join(""); + let result: unknown; + try { + result = JSON.parse(text); + } catch { + // Preserve non-JSON SDK output (e.g. completion hooks). Token-count + // only for SDK documents that opted into sdkDocumentControls. + if (exitCode === 0 && options.tokenCount && sdkDocumentControls) { + await writeOutputText( + destination, + `${estimateTokenCount(text)}\n`, + ); + return; + } + await writeOutputText(destination, text); + return; + } + // Compact rewrite must not skip token controls for SDK documents + // (`--llms` / `--schema` with --token-count on a pipe). + if (exitCode === 0 && options.tokenCount) { + const compact = [...jsonOutputParts(result, "json", false)].join( + "", + ); + await writeOutputText( + destination, + `${estimateTokenCount(compact)}\n`, + ); + return; + } + await writeJsonOutput(result, destination, "json"); + return; + } if (exitCode === 0) { if (options.tokenCount) await writeOutputText( diff --git a/tests/boundary/cli/jsonOutput.test.ts b/tests/boundary/cli/jsonOutput.test.ts index 68c145ce17..1ccdf1c24c 100644 --- a/tests/boundary/cli/jsonOutput.test.ts +++ b/tests/boundary/cli/jsonOutput.test.ts @@ -61,6 +61,9 @@ describe("SDK output completion", () => { }); }, }); + // Terminal JSON stays with Incur; only pipes buffer for compact rewrite. + if (format === "json") + Object.defineProperty(destination, "isTTY", { value: true }); const output = createCliResultOutput( parseCliOutputArguments(["--format", format]), destination, @@ -75,6 +78,105 @@ describe("SDK output completion", () => { expect(destination.writableEnded).toBe(false); }, ); + + it("rewrites piped SDK JSON diagnostics compactly", async () => { + const value = { + code: "VALIDATION_ERROR", + message: "REA could not read the command arguments.", + fieldErrors: [{ path: "symbols", missing: true, expected: "array" }], + }; + const indented = `${JSON.stringify(value, null, 2)}\n`; + const chunks: Buffer[] = []; + const destination = new Writable({ + write(chunk: unknown, _encoding, callback) { + if (!Buffer.isBuffer(chunk)) { + callback(new TypeError("Expected a UTF-8 output buffer")); + return; + } + chunks.push(chunk); + callback(); + }, + }); + const output = createCliResultOutput( + parseCliOutputArguments(["--json"]), + destination, + ); + if (output === undefined) throw new Error("Expected valid output controls"); + expect(output.captureSdkOutput(indented)).toBe(true); + expect(Buffer.concat(chunks).toString("utf8")).toBe(""); + await output.finishSdkOutput(1); + const text = Buffer.concat(chunks).toString("utf8"); + expect(text).toBe(`${JSON.stringify(value)}\n`); + expect(text).not.toContain("\n "); + expect(JSON.parse(text)).toEqual(value); + }); + + it("counts tokens for piped SDK JSON documents instead of rewriting them", async () => { + const value = { + version: "incur.v1", + commands: [{ name: "inspect-devicetree" }], + }; + const indented = `${JSON.stringify(value, null, 2)}\n`; + const compact = `${JSON.stringify(value)}\n`; + const chunks: Buffer[] = []; + const destination = new Writable({ + write(chunk: unknown, _encoding, callback) { + if (!Buffer.isBuffer(chunk)) { + callback(new TypeError("Expected a UTF-8 output buffer")); + return; + } + chunks.push(chunk); + callback(); + }, + }); + const output = createCliResultOutput( + parseCliOutputArguments([ + "--llms-full", + "--format", + "json", + "--token-count", + ]), + destination, + ); + if (output === undefined) throw new Error("Expected valid output controls"); + expect( + output.sdkArguments(["--llms-full", "--format", "json", "--token-count"]), + ).toEqual(expect.any(Array)); + expect(output.captureSdkOutput(indented)).toBe(true); + await output.finishSdkOutput(0); + expect(Buffer.concat(chunks).toString("utf8")).toBe( + `${estimateTokenCount(compact)}\n`, + ); + }); + + it("preserves non-JSON piped --json output under --token-count (completion hooks)", async () => { + const hook = "# bash completion hook\ncomplete -F _rea rea\n"; + const chunks: Buffer[] = []; + const destination = new Writable({ + write(chunk: unknown, _encoding, callback) { + if (!Buffer.isBuffer(chunk)) { + callback(new TypeError("Expected a UTF-8 output buffer")); + return; + } + chunks.push(chunk); + callback(); + }, + }); + const output = createCliResultOutput( + parseCliOutputArguments([ + "completions", + "bash", + "--json", + "--token-count", + ]), + destination, + ); + if (output === undefined) throw new Error("Expected valid output controls"); + output.sdkArguments(["completions", "bash", "--json", "--token-count"]); + expect(output.captureSdkOutput(hook)).toBe(true); + await output.finishSdkOutput(0); + expect(Buffer.concat(chunks).toString("utf8")).toBe(hook); + }); }); describe("incremental CLI JSON", () => { diff --git a/tests/boundary/cli/nativeDemangleCli.test.ts b/tests/boundary/cli/nativeDemangleCli.test.ts index 431cf02eff..c18b6f5422 100644 --- a/tests/boundary/cli/nativeDemangleCli.test.ts +++ b/tests/boundary/cli/nativeDemangleCli.test.ts @@ -69,4 +69,27 @@ describe("Swift demangling CLI with line-terminator characters", () => { ]), }); }); + + cliTest( + "emits compact JSON for missing-argument validation on piped stdout", + async ({ cli }) => { + const result = await cli.run({ + arguments: ["demangle-swift", "--json"], + environment, + timeoutMs: 20_000, + }); + expect(result.signal).toBeNull(); + expect(result.exitCode).toBe(1); + expect(result.stderr).toBe(""); + expect(result.json).toEqual( + expect.objectContaining({ + code: "VALIDATION_ERROR", + message: expect.stringMatching(/\S/), + }), + ); + // Piped stdout must stay compact; Incur's Formatter always indents. + expect(result.stdout).not.toContain("\n "); + expect(result.stdout).toBe(`${JSON.stringify(result.json)}\n`); + }, + ); }); From 2f5c0155dcfb27d19bff0fddd27733fa43a5442a Mon Sep 17 00:00:00 2001 From: Beqa <91750923+BeqaKs@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:41:11 +0400 Subject: [PATCH 43/94] fix(web): omit importer integrity for the trusted resolver stub (#8693) * fix(web): omit importer integrity for the trusted resolver stub Import-map integrity that covers the selected importer was applied to REA's replacement resolver script. Strip only those integrity entries from the installed map so native resolution can proceed while preserving dependency integrity and the caller-selected map identity. Fixes #8544 * fix(web): keep __proto__ import-map integrity keys on a null prototype --- src/browser/modules/NativeModuleResolver.ts | 3 +- .../importMapForTrustedResolver.test.ts | 98 +++++++++++++++++++ .../modules/importMapForTrustedResolver.ts | 75 ++++++++++++++ 3 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 src/browser/modules/importMapForTrustedResolver.test.ts create mode 100644 src/browser/modules/importMapForTrustedResolver.ts diff --git a/src/browser/modules/NativeModuleResolver.ts b/src/browser/modules/NativeModuleResolver.ts index 4383a27e0c..3cc45301e4 100644 --- a/src/browser/modules/NativeModuleResolver.ts +++ b/src/browser/modules/NativeModuleResolver.ts @@ -33,6 +33,7 @@ import type { } from "../../domain/webModuleTrace.js"; import { webModuleResolutionSchema } from "../../domain/webModuleTrace.js"; import { ProviderStartupDeadline } from "../../process/ProviderDeadline.js"; +import { importMapForTrustedResolver } from "./importMapForTrustedResolver.js"; const OPERATION = "trace_web_module_imports"; const TIMEOUT_MS = 20000; @@ -367,7 +368,7 @@ const resolveInBrowser = async ( await route.fulfill({ status: 200, contentType: "text/html", - body: `${input.importMap === null ? "" : ``}`, + body: `${input.importMap === null ? "" : ``}`, }); } else if ( !stubSent && diff --git a/src/browser/modules/importMapForTrustedResolver.test.ts b/src/browser/modules/importMapForTrustedResolver.test.ts new file mode 100644 index 0000000000..c1ae0f6200 --- /dev/null +++ b/src/browser/modules/importMapForTrustedResolver.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from "vitest"; +import { importMapForTrustedResolver } from "./importMapForTrustedResolver.js"; + +describe("importMapForTrustedResolver", () => { + const baseUrl = "https://app.test/index.html"; + const importerUrl = "https://app.test/main.js"; + + it("omits integrity that covers the selected importer URL", () => { + const map = { + imports: { dep: "/dep.js" }, + integrity: { + "https://app.test/main.js": "sha256-importer", + "https://app.test/dep.js": "sha256-dep", + }, + }; + expect(importMapForTrustedResolver(map, importerUrl, baseUrl)).toEqual({ + imports: { dep: "/dep.js" }, + integrity: { "https://app.test/dep.js": "sha256-dep" }, + }); + }); + + it("resolves relative integrity keys against the import-map base URL", () => { + const map = { + imports: { dep: "./dep.js" }, + integrity: { + "./main.js": "sha256-importer", + "./dep.js": "sha256-dep", + }, + }; + expect( + importMapForTrustedResolver(map, "https://app.test/main.js", baseUrl), + ).toEqual({ + imports: { dep: "./dep.js" }, + integrity: { "./dep.js": "sha256-dep" }, + }); + }); + + it("matches importer integrity across query and fragment differences", () => { + const map = { + imports: { dep: "/dep.js" }, + integrity: { + "https://app.test/main.js": "sha256-importer", + "https://app.test/dep.js": "sha256-dep", + }, + }; + expect( + importMapForTrustedResolver( + map, + "https://app.test/main.js?build=1#entry", + baseUrl, + ), + ).toEqual({ + imports: { dep: "/dep.js" }, + integrity: { "https://app.test/dep.js": "sha256-dep" }, + }); + }); + + it("drops an empty integrity member after removing the importer entry", () => { + const map = { + imports: { dep: "/dep.js" }, + integrity: { "https://app.test/main.js": "sha256-importer" }, + }; + expect(importMapForTrustedResolver(map, importerUrl, baseUrl)).toEqual({ + imports: { dep: "/dep.js" }, + }); + }); + + it("preserves maps without integrity and dependency-only integrity", () => { + const without = { imports: { dep: "/dep.js" } }; + expect(importMapForTrustedResolver(without, importerUrl, baseUrl)).toBe( + without, + ); + const dependencyOnly = { + imports: { dep: "/dep.js" }, + integrity: { "https://app.test/dep.js": "sha256-dep" }, + }; + expect( + importMapForTrustedResolver(dependencyOnly, importerUrl, baseUrl), + ).toBe(dependencyOnly); + }); + + it("preserves a dependency integrity key named __proto__ without prototype pollution", () => { + const integrity = Object.assign(Object.create(null), { + "https://app.test/main.js": "sha256-importer", + "https://app.test/dep.js": "sha256-dep", + }); + integrity["__proto__"] = "sha256-proto-dep"; + const map = { imports: { dep: "/dep.js" }, integrity }; + const next = importMapForTrustedResolver(map, importerUrl, baseUrl); + const nextIntegrity = next.integrity as { readonly [key: string]: string }; + expect(nextIntegrity["https://app.test/dep.js"]).toBe("sha256-dep"); + expect( + Object.getOwnPropertyDescriptor(nextIntegrity, "__proto__")?.value, + ).toBe("sha256-proto-dep"); + expect(Object.getPrototypeOf(nextIntegrity)).toBe(null); + expect(Object.prototype.hasOwnProperty.call({}, "__proto__")).toBe(false); + }); +}); diff --git a/src/browser/modules/importMapForTrustedResolver.ts b/src/browser/modules/importMapForTrustedResolver.ts new file mode 100644 index 0000000000..f4e254e2e8 --- /dev/null +++ b/src/browser/modules/importMapForTrustedResolver.ts @@ -0,0 +1,75 @@ +import type { JsonValue } from "../../domain/jsonValue.js"; + +/** + * Build the import map installed for REA's trusted resolver stub. + * + * Chromium applies import-map `integrity` to the module script URL. The + * resolver fulfills that URL with REA's replacement bytes, so integrity that + * covers the selected importer must not be installed. Dependency integrity and + * the caller-selected map identity remain unchanged in Evidence. + */ +export const importMapForTrustedResolver = ( + map: { readonly [key: string]: JsonValue }, + importerUrl: string, + baseUrl: string, +): { readonly [key: string]: JsonValue } => { + const integrity = map.integrity; + if ( + integrity === null || + typeof integrity !== "object" || + Array.isArray(integrity) + ) + return map; + const importer = importerModuleIdentity(importerUrl, baseUrl); + const nextIntegrity: { [key: string]: JsonValue } = Object.create(null); + let removed = false; + for (const [key, value] of Object.entries(integrity)) { + if (integrityKeyMatchesImporter(key, baseUrl, importer)) { + removed = true; + continue; + } + nextIntegrity[key] = value; + } + if (!removed) return map; + if (Object.keys(nextIntegrity).length === 0) { + const { integrity: _omit, ...rest } = map; + return rest; + } + return { ...map, integrity: nextIntegrity }; +}; + +const importerModuleIdentity = ( + importerUrl: string, + baseUrl: string, +): { readonly hrefs: ReadonlySet; readonly moduleKey: string } => { + const absolute = new URL(importerUrl, baseUrl); + const withoutHash = new URL(absolute.href); + withoutHash.hash = ""; + return { + hrefs: new Set([absolute.href, withoutHash.href]), + // Chromium's integrity lookup reports the module URL without search/hash. + moduleKey: `${absolute.origin}${absolute.pathname}`, + }; +}; + +const integrityKeyMatchesImporter = ( + key: string, + baseUrl: string, + importer: { + readonly hrefs: ReadonlySet; + readonly moduleKey: string; + }, +): boolean => { + try { + const resolved = new URL(key, baseUrl); + const withoutHash = new URL(resolved.href); + withoutHash.hash = ""; + return ( + importer.hrefs.has(resolved.href) || + importer.hrefs.has(withoutHash.href) || + `${resolved.origin}${resolved.pathname}` === importer.moduleKey + ); + } catch { + return false; + } +}; From 066a23475c56b3144c9fdcc80b77ccaed22305d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A4=80=ED=98=B8?= <101695482+chordpli@users.noreply.github.com> Date: Wed, 4 Nov 2026 20:42:04 +0900 Subject: [PATCH 44/94] fix(artifacts): classify bundle executables by their own name (#7365) * fix(artifacts): classify bundle executables by their own name A directory bundle is selected for its executable, so the executable's canonical name classifies it; the bundle directory spelling is not an artifact alias. Bundle facts now reach artifact, archive and database targets so the admitted-source digest binds to the executable. Fixes #7362 * fix(artifacts): share the artifact identity across every artifact route The trailing-archive fallback also carries the bundle facts now, so a bundle whose executable is a script stub with an appended ZIP keeps its digest bound to the executable. * fix(artifacts): name the classification spelling at the resolver boundary The resolver already knows whether a bundle directory or a file was selected, so it hands the executable or the caller's spelling to the reader instead of inferring it from the Info.plist. Every admitted route now shares one identity, and the .app bundle operations accept a bundle whatever its executable admitted as. * fix(artifacts): route bundle keyed archive reads by bundle facts A bundle whose executable is a plist now carries bundleInfoPlist, so inspect_keyed_archive must read bundle-relative archives instead of the standalone route that compares the selected bundle directory with the executable. * refactor(artifacts): build the selected ESP flash identity from the shared owner The selected ESP flash route from main built path, source path and digest inline. Use the resolver's shared identity so it carries the same facts as every other admitted target. --- src/application/BinaryTargetResolver.ts | 69 +++++++++---------- src/artifacts/ArtifactProvider.ts | 6 +- .../apple/artifactTargetApplicability.ts | 5 +- .../filesystem/artifactSelectedName.test.ts | 57 ++++++++++++++- .../filesystem/keyedArchiveSelection.test.ts | 48 +++++++++++++ 5 files changed, 146 insertions(+), 39 deletions(-) diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.ts index 3ce2e473f3..d69504c5d7 100644 --- a/src/application/BinaryTargetResolver.ts +++ b/src/application/BinaryTargetResolver.ts @@ -189,6 +189,9 @@ export const parseBinaryTarget = async ( const { executable: path, infoPlist } = resolved.value; const bundle = infoPlist === undefined ? {} : { bundleInfoPlist: infoPlist }; + // A bundle directory is selected for its executable, so the executable's + // own name classifies it; a selected file keeps the caller's spelling. + const classifiedAs = infoPlist === undefined ? candidate : path; return withOwnedRegularFile( path, { @@ -199,7 +202,13 @@ export const parseBinaryTarget = async ( try { return await readOpenedBinaryTarget( source, - { path, sourcePath, selectedPath: candidate, ...bundle }, + { + path, + sourcePath, + selectedPath: candidate, + classifiedAs, + ...bundle, + }, options, ); } catch (cause: unknown) { @@ -279,13 +288,23 @@ const readOpenedBinaryTarget = async ( readonly path: string; readonly sourcePath: string; readonly selectedPath: string; + /** Spelling whose name supplies format hints: the selected file, or a bundle's executable. */ + readonly classifiedAs: string; readonly bundleInfoPlist?: string; }, options: BinaryTargetResolverOptions, ): Promise => { const { handle, initial: metadata } = source; - const { path, sourcePath, selectedPath, bundleInfoPlist } = selected; + const { path, sourcePath, selectedPath, classifiedAs, bundleInfoPlist } = + selected; const bundle = bundleInfoPlist === undefined ? {} : { bundleInfoPlist }; + // Every admitted target shares one identity; routes add their own facts. + const identity = async () => ({ + path, + sourcePath, + ...bundle, + sha256: await sha256Handle(handle, path, metadata, signal), + }); const hostArchitecture = options.hostArchitecture ?? process.arch; const { targetKind, formatHint, signal, architecture, entryPoint } = options; throwIfTargetResolutionCancelled(signal); @@ -322,10 +341,7 @@ const readOpenedBinaryTarget = async ( }), ); return ok({ - path, - sourcePath, - ...bundle, - sha256: await sha256Handle(handle, path, metadata, signal), + ...(await identity()), kind: "executable", format: "raw-binary", architecture: admitted.value.architecture, @@ -373,10 +389,7 @@ const readOpenedBinaryTarget = async ( }), ); return ok({ - path, - sourcePath, - ...bundle, - sha256: await sha256Handle(handle, path, metadata, signal), + ...(await identity()), kind: "executable", format: "gamecube-dol", architecture: admitted.value.architecture, @@ -414,10 +427,8 @@ const readOpenedBinaryTarget = async ( ), ); return ok({ - path, - sourcePath, + ...(await identity()), artifactSelectedPath: selectedPath, - sha256: await sha256Handle(handle, path, metadata, signal), kind: "archive", format: "esp-flash-image", espFlash: { @@ -444,10 +455,7 @@ const readOpenedBinaryTarget = async ( }), ); return ok({ - path, - sourcePath, - ...bundle, - sha256: await sha256Handle(handle, path, metadata, signal), + ...(await identity()), kind: "executable", format: "dos-com", architecture: "x86", @@ -456,11 +464,15 @@ const readOpenedBinaryTarget = async ( } // Probe WASM content before suffix-only database routing; an explicit // database selection keeps the caller's chosen interpretation. - const classificationPath = artifactNameForClassification(selectedPath, path); + const classificationPath = artifactNameForClassification(classifiedAs, path); const artifactFormat = targetKind === "database" ? undefined : await detectArtifactFormat(classificationPath, source, signal, path); + const artifactIdentity = async () => ({ + ...(await identity()), + artifactSelectedPath: selectedPath, + }); if ( targetKind === "database" || (targetKind === undefined && @@ -469,19 +481,12 @@ const readOpenedBinaryTarget = async ( path.toLowerCase().endsWith(".hop")) ) return ok({ - path, - sourcePath, - sha256: await sha256Handle(handle, path, metadata, signal), + ...(await identity()), kind: "database", format: "analysis-database", }); if (artifactFormat !== undefined) { - const identity = { - path, - sourcePath, - artifactSelectedPath: selectedPath, - sha256: await sha256Handle(handle, path, metadata, signal), - }; + const identity = await artifactIdentity(); if (artifactFormat === "esp-flash-image") return ok({ ...identity, @@ -512,12 +517,9 @@ const readOpenedBinaryTarget = async ( )) !== undefined ) return ok({ - path, - sourcePath, - sha256: await sha256Handle(handle, path, metadata, signal), + ...(await artifactIdentity()), kind: "archive", format: zipRootPackageFormat(classificationPath), - artifactSelectedPath: selectedPath, }); const reason = typeof detected.error === "string" @@ -535,10 +537,7 @@ const readOpenedBinaryTarget = async ( ); } return ok({ - path, - sourcePath, - ...bundle, - sha256: await sha256Handle(handle, path, metadata, signal), + ...(await identity()), kind: "executable", ...detected.value, }); diff --git a/src/artifacts/ArtifactProvider.ts b/src/artifacts/ArtifactProvider.ts index b4af7af826..a1eddf2eff 100644 --- a/src/artifacts/ArtifactProvider.ts +++ b/src/artifacts/ArtifactProvider.ts @@ -270,8 +270,12 @@ class ArtifactClient implements AnalysisClient { ); } if (operation === "inspect_keyed_archive") { + // A bundle target reads bundle-relative archives whatever its + // executable admitted as; only a plist opened on its own is standalone. const standalone = - this.target.kind === "artifact" && this.target.format === "plist"; + this.target.bundleInfoPlist === undefined && + this.target.kind === "artifact" && + this.target.format === "plist"; const selectedPath = artifactTargetSelectedPath(this.target); const result = standalone ? await inspectStandaloneKeyedArchive({ diff --git a/src/domain/apple/artifactTargetApplicability.ts b/src/domain/apple/artifactTargetApplicability.ts index 38a9412dfd..b6345d0788 100644 --- a/src/domain/apple/artifactTargetApplicability.ts +++ b/src/domain/apple/artifactTargetApplicability.ts @@ -10,8 +10,9 @@ export const artifactTargetRefusal = ( operation: string, target: BinaryTarget, ): ArtifactTargetRefusal | undefined => { - const bundle = - target.kind === "executable" && target.bundleInfoPlist !== undefined; + // Bundle operations read the bundle directory, whatever its executable + // admitted as; the bundle facts alone establish an .app target. + const bundle = target.bundleInfoPlist !== undefined; const refuse = (reason: string): ArtifactTargetRefusal => ({ code: "target_kind_unsupported", reason, diff --git a/tests/boundary/filesystem/artifactSelectedName.test.ts b/tests/boundary/filesystem/artifactSelectedName.test.ts index bcf997b118..60bf87fab4 100644 --- a/tests/boundary/filesystem/artifactSelectedName.test.ts +++ b/tests/boundary/filesystem/artifactSelectedName.test.ts @@ -1,5 +1,5 @@ import { projectCliAnalysisResult } from "../../../src/cli/projectCliAnalysisResult.js"; -import { readFile, symlink, writeFile } from "node:fs/promises"; +import { mkdir, readFile, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; import { expect, it, vi } from "vitest"; @@ -129,3 +129,58 @@ it.skipIf(process.platform === "win32")( expect(admitted.value.kind).toBe("executable"); }, ); + +it.skipIf(process.platform === "win32")( + "classifies a bundle's executable by its own name and keeps the bundle facts", + async () => { + const root = await createTestTempDirectory("rea-artifact-bundle-name-"); + const bundle = join(root, "Fixture.app"); + await mkdir(bundle); + const infoPlist = join(bundle, "Info.plist"); + await writeFile( + infoPlist, + "CFBundleExecutablemain.js", + ); + const executable = join(bundle, "main.js"); + await writeFile(executable, "export const value = 1;\n"); + const admitted = await parseBinaryTarget(bundle); + if (!admitted.ok) throw admitted.error; + expect(admitted.value).toMatchObject({ + path: executable, + kind: "artifact", + format: "javascript", + bundleInfoPlist: infoPlist, + artifactSelectedPath: bundle, + }); + }, +); + +it.skipIf(process.platform === "win32")( + "keeps the bundle facts when the executable is a script stub with a trailing archive", + async () => { + const root = await createTestTempDirectory("rea-artifact-bundle-zip-"); + const bundle = join(root, "Fixture.app"); + await mkdir(bundle); + const infoPlist = join(bundle, "Info.plist"); + await writeFile( + infoPlist, + "CFBundleExecutablemain", + ); + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("payload.txt", new TextReader("appended archive")); + const executable = join(bundle, "main"); + await writeFile( + executable, + Buffer.concat([Buffer.from("#!/bin/sh\nexit 0\n"), await writer.close()]), + ); + const admitted = await parseBinaryTarget(bundle); + if (!admitted.ok) throw admitted.error; + expect(admitted.value).toMatchObject({ + path: executable, + kind: "archive", + format: "zip", + bundleInfoPlist: infoPlist, + artifactSelectedPath: bundle, + }); + }, +); diff --git a/tests/boundary/filesystem/keyedArchiveSelection.test.ts b/tests/boundary/filesystem/keyedArchiveSelection.test.ts index 90518433c9..d6c862d81a 100644 --- a/tests/boundary/filesystem/keyedArchiveSelection.test.ts +++ b/tests/boundary/filesystem/keyedArchiveSelection.test.ts @@ -13,6 +13,8 @@ import { build } from "plist"; import { expect, it } from "vitest"; import { ArtifactProvider } from "../../../src/artifacts/ArtifactProvider.js"; +import { AnalysisProviderRegistry } from "../../../src/application/binary/AnalysisProviderRegistry.js"; +import { composeBinarySession } from "../../../src/application/binary/BinarySessionComposition.js"; import { ArtifactReaderFailure } from "../../../src/artifacts/ArtifactReader.js"; import { inspectBundleKeyedArchive } from "../../../src/artifacts/apple/KeyedArchiveReader.js"; import { AnalysisError } from "../../../src/domain/analysisErrorBase.js"; @@ -402,3 +404,49 @@ const asAnalysisError = (value: unknown): AnalysisError => { if (!(value instanceof AnalysisError)) throw value; return value; }; + +it.skipIf(process.platform === "win32")( + "reads bundle-relative archives from a bundle whose executable is a plist", + async () => { + const root = await createTestTempDirectory("rea-keyed-plist-bundle-"); + const bundlePath = join(root, "Fixture.app"); + await mkdir(join(bundlePath, "Contents", "Resources"), { recursive: true }); + await writeFile( + join(bundlePath, "Info.plist"), + "CFBundleExecutableSettings.plist", + ); + await writeFile( + join(bundlePath, "Settings.plist"), + build({ setting: "value" }), + ); + await writeFile( + join(bundlePath, "Contents", "Resources", "Model.plist"), + Buffer.from(archive), + ); + const session = composeBinarySession( + new AnalysisProviderRegistry([], "auto", () => true), + [new ArtifactProvider(process.env, "darwin")], + ); + try { + const opened = await session.open(bundlePath); + if (!opened.ok) throw opened.error; + expect(session.activeTarget()).toMatchObject({ + kind: "artifact", + format: "plist", + bundleInfoPlist: join(bundlePath, "Info.plist"), + }); + const inspected = await session.execute("inspect_keyed_archive", { + path: "Contents/Resources/Model.plist", + }); + if (!inspected.ok) throw inspected.error; + expect(inspected.value.result).toMatchObject({ + archive_path: "Contents/Resources/Model.plist", + objects: expect.arrayContaining([ + expect.objectContaining({ value: "value" }), + ]), + }); + } finally { + await session.close(); + } + }, +); From 65311d49921b93075a4cbfdf1470de16ce405ad6 Mon Sep 17 00:00:00 2001 From: ShuaixinHuang <1836656514@qq.com> Date: Wed, 4 Nov 2026 19:42:49 +0800 Subject: [PATCH 45/94] fix(process): preserve literal temporary-root absence text (#8859) --- docs/process-capture.md | 10 ++- .../process/processCapture.comparison.test.ts | 47 +++++++++++++ src/domain/process/processScenario.test.ts | 16 +++++ src/domain/process/processScenario.ts | 2 +- .../capture/ProcessCaptureJournal.test.ts | 66 ++++++++++++++++++- src/process/capture/ProcessCaptureJournal.ts | 7 +- .../capture/ProcessNormalization.test.ts | 37 +++++++++++ src/process/capture/ProcessNormalization.ts | 17 ++--- .../processCaptureNormalization.test.ts | 2 +- 9 files changed, 183 insertions(+), 21 deletions(-) diff --git a/docs/process-capture.md b/docs/process-capture.md index 885606daa8..73a1e825cd 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -358,9 +358,13 @@ sibling filename such as `tool-helper`; selected parent directories still normalize within descendant paths, including when the working directory or an observation path is spelled with a trailing `/` or `/.`. A selected filesystem root `/` also aliases descendant paths, without rewriting URL separators or -HTML closing tags. The comparison contract -records these rules as `path-component-boundaries-v3`, so captures made with the -earlier rules cannot be compared as if the text had the same meaning. +HTML closing tags. When no temporary root applies to an observation, no temporary-root +path is selected: literal `` input and sampled command text +remain unchanged. The comparison contract +records these rules as `path-component-boundaries-v4`, so captures made with the +earlier rules cannot be compared as if the text had the same meaning. Historical +captures remain readable; recapture both sides with the same path normalization +version before comparing them. Rendered frame `lines` hold the visible rows after normalization, with trailing U+0020 spaces removed. Blank rows and internal spaces remain. `columns` records diff --git a/src/domain/process/processCapture.comparison.test.ts b/src/domain/process/processCapture.comparison.test.ts index 7e22ea76a7..fac1b74530 100644 --- a/src/domain/process/processCapture.comparison.test.ts +++ b/src/domain/process/processCapture.comparison.test.ts @@ -726,3 +726,50 @@ it("refuses mixed terminal observation versions even with PID and port rules dis }), ); }); + +it("refuses mixed path normalization versions even with PID and port rules disabled", () => { + const base = emptyCapture(); + const scenario = processScenarioSchema.parse({ + executable: "/usr/bin/true", + normalization: { pids: false, ports: false, paths: true }, + }); + const committed = processScenarioCommitment( + scenario, + base.manifest.selected_executable_sha256 ?? undefined, + ); + const contract = processComparisonContract(scenario); + expect(contract.path_normalization_version).toBe( + "path-component-boundaries-v4", + ); + const capture = parseProcessCapture({ + ...base, + normalization: scenario.normalization, + manifest: { + ...base.manifest, + scenario: committed, + full_scenario_sha256: digestProcessCommitment(committed), + normalization_sha256: digestProcessCommitment(scenario.normalization), + comparison_contract: contract, + comparison_contract_sha256: digestProcessCommitment(contract), + }, + }); + const legacyContract = { + ...contract, + path_normalization_version: "path-component-boundaries-v3", + }; + const legacy = parseProcessCapture({ + ...capture, + manifest: { + ...capture.manifest, + comparison_contract: legacyContract, + comparison_contract_sha256: digestProcessCommitment(legacyContract), + }, + }); + expect(() => compareProcessCaptures(capture, legacy)).toThrow( + expect.objectContaining({ + issues: [ + expect.objectContaining({ expected: ["path_normalization_version"] }), + ], + }), + ); +}); diff --git a/src/domain/process/processScenario.test.ts b/src/domain/process/processScenario.test.ts index 2c2e1e5c80..73355d06c1 100644 --- a/src/domain/process/processScenario.test.ts +++ b/src/domain/process/processScenario.test.ts @@ -153,6 +153,22 @@ it("keeps scenario identity without finalization while versioning corrected port ).toBe("b50005d26269bd1db4a49c79e05bca67c3a18d837b45cd1c8ff591c01eb7623a"); const comparison = processComparisonContract(scenario); expect(comparison.port_normalization_version).toBe("contextual-endpoints-v3"); + expect(comparison).not.toHaveProperty("path_normalization_version"); + const withPaths = processComparisonContract( + processScenarioSchema.parse({ + ...scenario, + normalization: { ...scenario.normalization, paths: true }, + }), + ); + expect(withPaths.path_normalization_version).toBe( + "path-component-boundaries-v4", + ); + expect(digestProcessCommitment(withPaths)).not.toBe( + digestProcessCommitment({ + ...withPaths, + path_normalization_version: "path-component-boundaries-v3", + }), + ); const { pid_normalization_version: pidVersion, terminal_observation_version: terminalVersion, diff --git a/src/domain/process/processScenario.ts b/src/domain/process/processScenario.ts index 92a4d1b092..68c0bb17c9 100644 --- a/src/domain/process/processScenario.ts +++ b/src/domain/process/processScenario.ts @@ -162,7 +162,7 @@ export const processComparisonContract = ( ? { pid_normalization_version: "sampled-identifiers-v2" } : {}), ...(scenario.normalization.paths - ? { path_normalization_version: "path-component-boundaries-v3" } + ? { path_normalization_version: "path-component-boundaries-v4" } : {}), ...(scenario.normalization.ports ? { port_normalization_version: "contextual-endpoints-v3" } diff --git a/src/process/capture/ProcessCaptureJournal.test.ts b/src/process/capture/ProcessCaptureJournal.test.ts index 24232c4459..2f8212e611 100644 --- a/src/process/capture/ProcessCaptureJournal.test.ts +++ b/src/process/capture/ProcessCaptureJournal.test.ts @@ -1,6 +1,14 @@ -import { expect, it } from "vitest"; +import { expect, it, vi } from "vitest"; -import { createProcessCaptureJournal } from "./ProcessCaptureJournal.js"; +import { + createProcessCaptureJournal, + scheduleScenarioInteractions, +} from "./ProcessCaptureJournal.js"; + +import { TerminalRenderer } from "./TerminalRenderer.js"; +import { processScenarioSchema } from "../../domain/process/processScenario.js"; +import type { InteractionEvent } from "../../domain/process/processCapture.js"; +import type { ProcessTimer } from "./ProcessTimer.js"; it("records capture events in monotonic observation order", () => { const journal = createProcessCaptureJournal(); @@ -16,3 +24,57 @@ it("records capture events in monotonic observation order", () => { { capture_order: 3, collection: "filesystem_checkpoints", index: 1 }, ]); }); + +it.each([true, false])( + "preserves non-path input markers when paths=%s", + async (paths) => { + vi.useFakeTimers(); + const value = "x y\n"; + const scenario = processScenarioSchema.parse({ + executable: "/usr/bin/node", + working_directory: "/workspace", + events: [{ type: "input", at_ms: 0, data: value }], + normalization: { paths, pids: false, ports: false, patterns: [] }, + }); + const renderer = new TerminalRenderer({ + columns: 80, + rows: 24, + scrollback: 0, + maxBytes: 1024, + normalize: (value) => ({ + state: "normalized", + value, + utf8Bytes: Buffer.byteLength(value), + }), + }); + const timers = new Set(); + const interactions: InteractionEvent[] = []; + const journal = createProcessCaptureJournal(); + try { + scheduleScenarioInteractions({ + scenario, + getTerminal: () => undefined, + timers, + interactions, + renderer, + started: 0, + dispatchedEventIndexes: new Set(), + recordEvent: journal.recordEvent, + }); + await vi.advanceTimersByTimeAsync(1); + expect(interactions).toHaveLength(1); + expect(interactions[0]).toMatchObject({ + type: "input", + data: value, + outcome: "target_exited", + }); + expect(journal.entries).toEqual([ + { capture_order: 0, collection: "interaction_events", index: 0 }, + ]); + } finally { + for (const timer of timers) timer.cancel(); + await renderer.dispose(); + vi.useRealTimers(); + } + }, +); diff --git a/src/process/capture/ProcessCaptureJournal.ts b/src/process/capture/ProcessCaptureJournal.ts index fd143b4f36..b379cc8d10 100644 --- a/src/process/capture/ProcessCaptureJournal.ts +++ b/src/process/capture/ProcessCaptureJournal.ts @@ -96,12 +96,7 @@ export const scheduleScenarioInteractions = ( : { data: event.type === "input" - ? normalizeProcessText( - event.data, - scenario, - "", - -1, - ) + ? normalizeProcessText(event.data, scenario, undefined, -1) : event.type === "resize" ? `${String(event.columns)}x${String(event.rows)}` : event.signal, diff --git a/src/process/capture/ProcessNormalization.test.ts b/src/process/capture/ProcessNormalization.test.ts index 79693c36bc..49827a8460 100644 --- a/src/process/capture/ProcessNormalization.test.ts +++ b/src/process/capture/ProcessNormalization.test.ts @@ -665,3 +665,40 @@ it("shares sampled identifiers before custom patterns in terminal and command no "root=selected child=selected other=201", ); }); + +it("does not alias the temporary-root absence marker in sampled command data", () => { + const command = "/usr/bin/node --marker "; + const selected = [{ ...samples[0]!, command }]; + const normalized = normalizeProcessSamples( + selected, + scenario({ pids: false, ports: false, patterns: [] }), + rootPid, + ); + expect(normalized[0]?.command).toBe( + " --marker ", + ); + expect(selected[0]?.command).toBe(command); +}); + +it("preserves literal absence text while normalizing selected paths with and without a temporary root", () => { + const selected = scenario({ pids: false, ports: false, patterns: [] }); + const text = + "/usr/bin/node /workspace/file /capture/file"; + for (const temporaryRoot of [undefined, "/capture"]) { + const expected = ` /file ${temporaryRoot === undefined ? "/capture/file" : "/file"}`; + expect( + createProcessTextNormalizer(selected, temporaryRoot, rootPid)(text), + ).toBe(expected); + expect( + createBoundedProcessTextNormalizer( + selected, + temporaryRoot, + rootPid, + )(text, 1024), + ).toEqual({ + state: "normalized", + value: expected, + utf8Bytes: Buffer.byteLength(expected), + }); + } +}); diff --git a/src/process/capture/ProcessNormalization.ts b/src/process/capture/ProcessNormalization.ts index bd2fee817f..7247fae277 100644 --- a/src/process/capture/ProcessNormalization.ts +++ b/src/process/capture/ProcessNormalization.ts @@ -231,7 +231,7 @@ export const createProcessIdentifierLookup = ( /** Compile one ordered policy for both bounded retention and unrestricted text. */ const createProcessTextRules = ( scenario: ProcessScenario, - temporaryRoot: string, + temporaryRoot: string | undefined, pid: ProcessIdentifierSource, ): (() => readonly (ProcessTextRule | undefined)[]) => { const rules: (ProcessTextRule | undefined)[] = []; @@ -278,7 +278,7 @@ const createProcessTextRules = ( /** Prepare capture-local matchers while retaining a live terminal PID lookup. */ export const createProcessTextNormalizer = ( scenario: ProcessScenario, - temporaryRoot: string, + temporaryRoot: string | undefined, pid: ProcessIdentifierSource, ): ((value: string) => string) => { const rules = createProcessTextRules(scenario, temporaryRoot, pid); @@ -298,7 +298,7 @@ export const createProcessTextNormalizer = ( /** Normalize terminal text without materializing an intermediate over its byte budget. */ export const createBoundedProcessTextNormalizer = ( scenario: ProcessScenario, - temporaryRoot: string, + temporaryRoot: string | undefined, pid: ProcessIdentifierSource, ): ((value: string, maxBytes: number) => BoundedProcessTextNormalization) => { const rules = createProcessTextRules(scenario, temporaryRoot, pid); @@ -339,7 +339,7 @@ export const createBoundedProcessTextNormalizer = ( export const normalizeProcessText = ( value: string, scenario: ProcessScenario, - temporaryRoot: string, + temporaryRoot: string | undefined, pid: number, ): string => createProcessTextNormalizer(scenario, temporaryRoot, pid)(value); @@ -355,12 +355,12 @@ const directoryIdentity = (path: string): string => { /** Match original paths once, preferring the most specific selected identity. */ const createPathRule = ( scenario: ProcessScenario, - temporaryRoot: string, + temporaryRoot: string | undefined, ): | { readonly matcher: RegExp; readonly replace: RegexReplacement } | undefined => { const aliases = new Map(); - const selected: readonly (readonly [string, string])[] = [ + const selected: readonly (readonly [string | undefined, string])[] = [ [temporaryRoot, ""], [directoryIdentity(scenario.working_directory), ""], [scenario.executable, ""], @@ -372,7 +372,8 @@ const createPathRule = ( ), ]; for (const [path, alias] of selected) - if (path !== "" && !aliases.has(path)) aliases.set(path, alias); + if (path !== undefined && path !== "" && !aliases.has(path)) + aliases.set(path, alias); const paths = [...aliases.keys()].sort( (left, right) => right.length - left.length, ); @@ -429,7 +430,7 @@ export const normalizeProcessSamples = ( ); const normalizeText = createProcessTextNormalizer( scenario, - "", + undefined, identifiers, ); const normalizeCommand = (command: string): string => diff --git a/tests/boundary/process/processCaptureNormalization.test.ts b/tests/boundary/process/processCaptureNormalization.test.ts index 838a550a76..e21bfe8446 100644 --- a/tests/boundary/process/processCaptureNormalization.test.ts +++ b/tests/boundary/process/processCaptureNormalization.test.ts @@ -262,7 +262,7 @@ itWithCaptureCapability( ).toBe(original.replaceAll("\n", "\r\n")); expect(capture.manifest.comparison_contract).toHaveProperty( "path_normalization_version", - "path-component-boundaries-v3", + "path-component-boundaries-v4", ); const legacyContract = { ...capture.manifest.comparison_contract }; delete legacyContract.path_normalization_version; From b668cea2af0fa2608885563195eef6cdea0cb602 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E4=BD=B3=E6=B5=A9Alben?= <35291372+XiaoBinGan@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:43:33 +0800 Subject: [PATCH 46/94] fix(javascript): emit no fs acquisition for a literal non-flag string (#8125) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(javascript): emit no fs acquisition for a literal non-flag string `fs.open` and `fs.openSync` validate a literal flag before touching the filesystem: if (!isFd(flags) && !FLAGS_SET.has(flags)) throw new ERR_INVALID_ARG_VALUE('flags', flags, ...); The throw is synchronous, before a descriptor exists, but the analysis reported an `acquire` operation for those calls. Measured on Node 24.22.1 against a file that does not exist, so each valid flag is exercised without an EEXIST masking it: ACCEPTED as a flags string: a a+ as as+ ax ax+ r r+ rs rs+ sa sa+ sr sr+ w w+ wx wx+ xa xa+ xw xw+ REJECTED with ERR_INVALID_ARG_VALUE: "" A R W rb wb s s+ x r,x A Python-style `"rb"`/`"wb"` is the common case: those are valid in Python and fatal at the Node call site. `FS_OPEN_FLAGS` in `javascriptStaticAnalysisHelpers.ts` already holds exactly those 22 accepted flags. It is now exported and reused so the accepted set has one owner. A numeric flag is a platform bitmask and a computed value may still be valid, so only an exactly readable non-flag string is dropped. `fs.createReadStream` keeps its acquisition: it does not validate the flag synchronously, it returns a stream and reports the failure on the error event. * chore: retrigger title and readiness checks after the title was corrected * fix(javascript): reject every fs flag literal Node rejects The first version only decided string flags. Node's check covers every flag shape, and each one below throws `ERR_INVALID_ARG_VALUE` or `ERR_OUT_OF_RANGE` synchronously, before a descriptor exists: openSync(p, 1.5) -> THREW ERR_OUT_OF_RANGE openSync(p, -1) -> THREW EINVAL openSync(p, -1.5) -> THREW ERR_OUT_OF_RANGE openSync(p, 1e400) -> THREW ERR_OUT_OF_RANGE openSync(p, NaN) -> THREW ERR_OUT_OF_RANGE openSync(p, Infinity) -> THREW ERR_OUT_OF_RANGE openSync(p, 0n) -> THREW ERR_INVALID_ARG_VALUE openSync(p, true) -> THREW ERR_INVALID_ARG_VALUE openSync(p, false) -> THREW ERR_INVALID_ARG_VALUE openSync(p, {}) -> THREW ERR_INVALID_ARG_VALUE openSync(p, []) -> THREW ERR_INVALID_ARG_VALUE --- accepted --- openSync(p, 0) / 1 / 65536 / +5 -> ok openSync(p) / undefined / null -> ok openSync(p, "r") / "wx" / "sr" -> ok `NaN` and `Infinity` are identifiers rather than literals, so they are decided only through `isUnmodifiedGlobalReference`; a shadowed binding keeps the acquisition. * fix(javascript): judge an fs flag by its value, and ignore a shifting spread Two P1 findings on the first version. **A unary operator was judged by its operand.** `openSync(p, void 1.5)` passes `undefined`, so Node applies the default read flag, but the branch saw a fractional operand and dropped a real acquisition. `openSync(p, -0)` is the valid zero bitmask and was also dropped. `staticFlagBits` now evaluates the operator: openSync(p, void 1.5) -> ok (keeps the acquisition) openSync(p, void 0) -> ok openSync(p, -0) -> ok openSync(p, -0.0) -> ok openSync(p, +0) -> ok openSync(p, ~-1) -> ok openSync(p, -1) -> THREW EINVAL openSync(p, -1.5) -> THREW ERR_OUT_OF_RANGE openSync(p, +1.5) -> THREW ERR_OUT_OF_RANGE openSync(p, !0) -> THREW ERR_INVALID_ARG_VALUE openSync(p, ~0) -> THREW EINVAL openSync(p, typeof 1) -> THREW ERR_INVALID_ARG_VALUE **A spread before the flag slot shifts the runtime position.** With `args = [path, "w"]`, `openSync(...args, "0600")` opens with mode `"0600"`, but argument 1 is then the mode rather than the flag. The check now keeps the acquisition whenever argument 0 is a spread, because the flag it inspects is not provably the flag at runtime. * fix(javascript): leave a signed fs bitmask undecided greptile caught that the flag check rejected every negative integer, but Node accepts some signed bitmasks. Measured on Node 24.22.1 against an existing file: openSync(p, -2147483648) -> ok openSync(p, -1073741824) -> ok openSync(p, -1) -> THREW EINVAL openSync(p, -2) -> THREW EINVAL openSync(p, 2147483647) -> THREW EINVAL openSync(p, 2147483648) -> THREW ERR_OUT_OF_RANGE openSync(p, -2147483649) -> THREW ERR_OUT_OF_RANGE So the sign does not decide: some negative values open a descriptor and others do not, and which is which is a platform bitmask question this analysis cannot settle. The check now rejects only a non-integer or a value outside the signed 32-bit range, and leaves an in-range negative undecided — the same choice `greptile` asked for, on the principle that a call which may succeed must keep its acquisition. `!`, `typeof` and `delete` are still rejected, because those yield a boolean or a string regardless of the operand. * fix(javascript): read a no-expression template literal flag Review finding. A no-expression template literal is exactly readable, so it is the same flag string Node receives, but the guard only read `StringLiteral`: fs.openSync(p, ` + "`rb`" + `) -> THREW ERR_INVALID_ARG_VALUE fs.openSync(p, ` + "`r`" + `) -> ok The flag slot now accepts an interpolation-free template; an interpolated template stays undecided. Four regressions cover both directions; two fail without the change. --- ...SemanticAnalysis.open_flag_literal.test.ts | 135 ++++++++++++++++++ .../javascript/javascriptSemanticResources.ts | 91 +++++++++++- .../javascriptStaticAnalysisHelpers.ts | 2 +- 3 files changed, 226 insertions(+), 2 deletions(-) create mode 100644 src/domain/javascript/javascriptSemanticAnalysis.open_flag_literal.test.ts diff --git a/src/domain/javascript/javascriptSemanticAnalysis.open_flag_literal.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.open_flag_literal.test.ts new file mode 100644 index 0000000000..cd1106b3a0 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticAnalysis.open_flag_literal.test.ts @@ -0,0 +1,135 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js"; + +/** + * `fs.open` and `fs.openSync` validate a literal flag string before touching the + * filesystem and throw `ERR_INVALID_ARG_VALUE` synchronously, so no descriptor + * exists and no resource is acquired. Node's accepted flag set is measured, not + * guessed: `a`, `a+`, `as`, `as+`, `ax`, `ax+`, `r`, `r+`, `rs`, `rs+`, `sa`, + * `sa+`, `sr`, `sr+`, `w`, `w+`, `wx`, `wx+`, `xa`, `xa+`, `xw`, `xw+`. + * + * A numeric flag is a platform bitmask and a computed value may still be a valid + * flag, so only an exactly readable non-flag string is dropped. + */ +const acquisitionsFor = (call: string) => + analyzeJavaScriptSemantics( + `import { open, openSync, createReadStream } from "node:fs";\nimport { open as openPromise } from "node:fs/promises";\nfunction g(p, cb) { ${call}; }`, + ).resourceOperations.map(({ kind, method }) => ({ kind, method })); + +it.each([ + ["a Python-style read flag", `openSync(p, "rb")`], + ["a Python-style write flag", `openSync(p, "wb")`], + ["an append-binary flag", `openSync(p, "ab")`], + ["a lowercase x flag", `openSync(p, "x")`], + ["a comma flag", `openSync(p, "r,x")`], + ["an empty flag", `openSync(p, "")`], + ["an uppercase flag", `openSync(p, "R")`], + ["a callback-form read flag", `open(p, "rb", cb)`], + ["a promise-form read flag", `openPromise(p, "rb")`], + // Node also rejects every non-string flag that is not a nullish value or a + // usable bitmask integer. + ["a fractional flag", `openSync(p, 1.5)`], + // A value outside the signed 32-bit range cannot be a bitmask at all. + ["an int32-overflow flag", `openSync(p, 2147483648)`], + ["an int32-underflow flag", `openSync(p, -2147483649)`], + ["a negated fractional flag", `openSync(p, -1.5)`], + ["a non-finite numeric flag", `openSync(p, 1e400)`], + ["a NaN flag", `openSync(p, NaN)`], + ["an Infinity flag", `openSync(p, Infinity)`], + ["a BigInt flag", `openSync(p, 0n)`], + ["a true flag", `openSync(p, true)`], + ["a false flag", `openSync(p, false)`], + ["an object flag", `openSync(p, {})`], + ["an array flag", `openSync(p, [])`], + // A unary operator is judged by its result, not by its operand: `!` yields a + // boolean and `~0` yields `-1`. + ["a logical-not flag", `openSync(p, !0)`], +] as const)("emits no acquisition for %s", (_name, call) => { + expect(acquisitionsFor(call)).toEqual([]); +}); + +it.each([ + ["a read flag", `openSync(p, "r")`], + ["a read-plus flag", `openSync(p, "r+")`], + ["an append-plus flag", `openSync(p, "a+")`], + ["an exclusive write flag", `openSync(p, "wx")`], + ["a symbolic-link flag", `openSync(p, "sr")`], + ["an omitted flag", `openSync(p)`], + ["a computed flag", `openSync(p, flag)`], + ["a numeric bitmask", `openSync(p, 1)`], + ["a zero bitmask", `openSync(p, 0)`], + ["a large bitmask", `openSync(p, 65536)`], + ["a unary-plus bitmask", `openSync(p, +5)`], + ["an explicit undefined flag", `openSync(p, undefined)`], + ["an explicit null flag", `openSync(p, null)`], + // `void ` is `undefined`, which Node accepts as the default read flag, + // so the operand must never decide the outcome. + ["a void flag over a fraction", `openSync(p, void 1.5)`], + ["a void flag over a string", `openSync(p, void "r")`], + // `-0` is the valid zero bitmask, and `~-1` is `0`. + // The sign does not prove rejection: Node accepts some signed bitmasks such as + // `-2147483648` and rejects others such as `-1`, so an in-range negative is + // deliberately left undecided and keeps its acquisition. + ["a minimum int32 bitmask", `openSync(p, -2147483648)`], + ["a negative power-of-two bitmask", `openSync(p, -1073741824)`], + ["an in-range negative bitmask", `openSync(p, -1)`], + // `~0` is `-1` and `~1` is `-2`. Node happens to reject both, but that is the + // same undecidable signed-bitmask question, so they stay undecided too. + ["a bitwise-not of zero", `openSync(p, ~0)`], + ["a bitwise-not of one", `openSync(p, ~1)`], + ["a maximum int32 bitmask", `openSync(p, 2147483647)`], + ["a negative-zero flag", `openSync(p, -0)`], + ["a negative-zero expression flag", `openSync(p, -0.0)`], + ["a unary-plus zero flag", `openSync(p, +0)`], + ["a bitwise-not of negative one", `openSync(p, ~-1)`], + // A spread before the flag slot shifts the runtime position, so argument 1 is + // not provably the flag: `openSync(...args, "0600")` opens with a mode. + ["a spread before the flag", `openSync(...args, "0600")`], + ["a spread supplying every argument", `openSync(...args)`], + ["a spread in the flag slot", `openSync(p, ...flags)`], +] as const)("keeps the acquisition for %s", (_name, call) => { + expect(acquisitionsFor(call)).toEqual([ + { kind: "acquire", method: "openSync" }, + ]); +}); + +it("keeps the callback-form acquisition with an accepted flag", () => { + expect(acquisitionsFor(`open(p, "r", cb)`)).toEqual([ + { kind: "acquire", method: "open" }, + ]); +}); + +/** + * `fs.createReadStream` does not validate the flag synchronously: it returns a + * stream and surfaces the failure on the stream's error event, so the stream + * object is still acquired. + */ +it("keeps a read stream whose flags option is invalid", () => { + expect(acquisitionsFor(`createReadStream(p, { flags: "rb" })`)).toEqual([ + { kind: "acquire", method: "createReadStream" }, + ]); +}); + +/** + * Review finding: a no-expression template literal is exactly readable, so it is the + * same flag string Node receives. + * + * fs.openSync(p, `rb`) -> THREW ERR_INVALID_ARG_VALUE + * fs.openSync(p, `r`) -> ok + */ +it.each([ + ["a rejected template flag", 'openSync("x", `rb`)'], + ["another rejected template flag", 'openSync("x", `bad`)'], +] as const)("emits no fs acquisition for %s", (_name, call) => { + expect(acquisitionsFor(call)).toEqual([]); +}); + +it.each([ + ["an accepted template flag", 'openSync("x", `r`)'], + ["an interpolated template flag", 'openSync("x", `${flag}`)'], +] as const)("keeps the fs acquisition for %s", (_name, call) => { + expect(acquisitionsFor(call)).toEqual([ + { kind: "acquire", method: "openSync" }, + ]); +}); diff --git a/src/domain/javascript/javascriptSemanticResources.ts b/src/domain/javascript/javascriptSemanticResources.ts index d441c70d1c..1e5534d40a 100644 --- a/src/domain/javascript/javascriptSemanticResources.ts +++ b/src/domain/javascript/javascriptSemanticResources.ts @@ -26,7 +26,7 @@ import { traverseDataEffects, type DataEffectTraversalContext, } from "./javascriptSemanticDataEffectHelpers.js"; -import { range } from "./javascriptStaticAnalysisHelpers.js"; +import { FS_OPEN_FLAGS, range } from "./javascriptStaticAnalysisHelpers.js"; const RELEASE_METHODS = ["close", "destroy", "end"] as const; @@ -75,6 +75,90 @@ export const collectJavaScriptSemanticResources = ( ]; }; +/** + * `fs.open`/`fs.openSync` validate the flag before touching the filesystem, so + * a flag Node rejects at the call site throws synchronously and no descriptor + * exists. Node accepts only a known flag string, a nullish value (which defaults + * to `"r"`), or a bitmask integer; everything else throws. A computed value may + * still be valid, so only an exactly readable literal is decided. + */ +const rejectsOpenFlagLiteral = ( + node: t.CallExpression | t.OptionalCallExpression, + context: DataEffectTraversalContext, +): boolean => { + // A spread before the flag slot shifts the runtime position, so argument 1 is + // not provably the flag. `openSync(...args, "0600")` with + // `args = [path, "w"]` opens with mode `"0600"`, not with an invalid flag. + if (t.isSpreadElement(node.arguments[0])) return false; + const flag = node.arguments[1]; + if (flag === undefined || t.isSpreadElement(flag)) return false; + const value = unwrapJavaScriptExpression(flag).node; + // A no-expression template literal is exactly readable, so it is the same flag + // string Node receives. + const flagText = t.isStringLiteral(value) + ? value.value + : t.isTemplateLiteral(value) && value.expressions.length === 0 + ? value.quasis[0]?.value.cooked + : undefined; + if (flagText !== undefined && flagText !== null) + return !FS_OPEN_FLAGS.has(flagText); + // A non-integer or out-of-int32 value is not a usable bitmask. The sign alone + // does not decide: Node accepts some signed bitmasks such as `-2147483648` and + // rejects others such as `-1`, so an in-range negative is left undecided. + const bits = staticFlagBits(value); + if (bits !== undefined) + return ( + bits !== null && + (!Number.isInteger(bits) || bits > 2_147_483_647 || bits < -2_147_483_648) + ); + // `NaN` and `Infinity` are unshadowed globals, and neither is a bitmask. + if ( + t.isIdentifier(value) && + (value.name === "NaN" || value.name === "Infinity") && + isUnmodifiedGlobalReference(value, context.state, value.name) + ) + return true; + // BigInt, boolean, object and array literals all fail Node's flag check. + return ( + t.isBigIntLiteral(value) || + t.isBooleanLiteral(value) || + t.isObjectExpression(value) || + t.isArrayExpression(value) + ); +}; + +/** + * The number a literal flag argument evaluates to, or `null` when it evaluates to + * `undefined`. Returns `undefined` when the value is not exactly readable. + * + * `void ` is `undefined`, which Node accepts as the default read flag, so + * it must never be judged by its operand. `!`, `typeof` and `delete` yield a + * boolean or a string rather than a bitmask. + */ +const staticFlagBits = (value: t.Node): number | null | undefined => { + if (t.isNumericLiteral(value)) return value.value; + if (!t.isUnaryExpression(value)) return undefined; + if (value.operator === "void") return null; + if ( + value.operator === "!" || + value.operator === "typeof" || + value.operator === "delete" + ) + return Number.NaN; + const operand = unwrapJavaScriptExpression(value.argument).node; + if (!t.isNumericLiteral(operand)) return undefined; + if (value.operator === "-") return -operand.value; + if (value.operator === "+") return operand.value; + if (value.operator === "~") return ~operand.value; + return undefined; +}; + +/** `fs.open`, `fs.openSync` and `fs/promises.open` take the flag in slot 1. */ +const openFlagInFirstArgument = (reference: AcquisitionReference): boolean => + reference.method === "open" || reference.method === "openSync" + ? reference.specifier === "fs" || reference.specifier === "fs/promises" + : false; + const collectAcquisitions = ( program: t.Program, context: DataEffectTraversalContext, @@ -84,6 +168,11 @@ const collectAcquisitions = ( if (!t.isCallExpression(node) && !t.isOptionalCallExpression(node)) return; const reference = acquisitionMethod(node.callee, context.state); if (reference === null) return; + if ( + openFlagInFirstArgument(reference) && + rejectsOpenFlagLiteral(node, context) + ) + return; output.push({ node, specifier: reference.specifier, diff --git a/src/domain/javascript/javascriptStaticAnalysisHelpers.ts b/src/domain/javascript/javascriptStaticAnalysisHelpers.ts index 40a2f4166e..209b3eacdd 100644 --- a/src/domain/javascript/javascriptStaticAnalysisHelpers.ts +++ b/src/domain/javascript/javascriptStaticAnalysisHelpers.ts @@ -530,7 +530,7 @@ const WINDOW_OPEN_CALL = /(?:^|\.)(?:window|self|globalThis|top|parent|opener|frames|document)\.open$/u; /** Node `fs.open` flag strings, which are never request URLs. */ -const FS_OPEN_FLAGS = new Set([ +export const FS_OPEN_FLAGS: ReadonlySet = new Set([ "a", "a+", "as", From 848f16541746d11cf0ebd507f992b1c570128afb Mon Sep 17 00:00:00 2001 From: Ace <1160684883@qq.com> Date: Wed, 4 Nov 2026 19:44:18 +0800 Subject: [PATCH 47/94] feat(harmony): disclose unclassified inventory files (#4968) * feat(harmony): disclose unclassified inventory files * test(harmony): let createEvidence own JSON admission in the partial-inventory case * test(harmony): extract the projection assertion from the archive journey * docs(harmony): qualify the unclassified disclosure rules and pin exact-case manifests * test(harmony): replace limitation prose with structured coverage facts The partial-inventory control froze the sentence "Source inventory coverage is partial; absence is unknown." verbatim. docs/testing.md asserts structured limit facts instead of freezing explanatory sentences and puts documentation wording in review, so the sentence is no longer substring-matched. `coverage.status` is a disjunction of independent causes in this projection: incomplete inventory scope, unavailable content, and an exceeded bridge budget. Asserting only `status: "partial"` left that attribution unstated, so the two other causes are now pinned as well. `unavailable_components` being empty and `bridge_candidate_coverage` being complete with zero candidates make the partial result attributable to the missing occurrence alone. Also drops an `artifact_id` comparison that could not fail: the fixture's four files carry the same bytes, the preceding assertion already pins that shared digest, and `artifact_id` is derived from the digest. Distinct paths sharing content stay covered by the path array and the independent digest expectations. Narrows a comment that claimed exact-case matching for every name in the control; only the manifest names are case-sensitive, because the App Pack child suffix match is case-insensitive. * docs(harmony): name the disclosure location for a nested child package The unclassified-disclosure paragraph used "there" to refer back to a clause that had just named `app_pack_children`, so a reader could take a nested `.hap` or `.hsp` member of a HAP or HSP root to be reported as an App Pack child. It is reported in `unclassified_components`: the child list is populated only for an App Pack root. Names the location explicitly. --- docs/harmony-analysis.md | 20 +++ src/contracts/applicationToolContracts.ts | 2 +- src/domain/harmony/harmonyApplication.ts | 8 + .../mobileDirectoryComponents.test.ts | 1 + .../acceptance/harmonyArtifactJourney.test.ts | 56 +++++-- .../filesystem/harmony/application.test.ts | 145 ++++++++++++++++++ 6 files changed, 215 insertions(+), 17 deletions(-) diff --git a/docs/harmony-analysis.md b/docs/harmony-analysis.md index a671a2ef7d..72fbeec5c1 100644 --- a/docs/harmony-analysis.md +++ b/docs/harmony-analysis.md @@ -36,6 +36,8 @@ The MCP tool `project_harmony_application_graph` takes the same - exact component paths, artifact IDs, and hashes (manifests, Ark bytecode, resources, native libraries, JavaScript, signing) +- `unclassified_components` with the same paths and content identities for + authenticated files outside those categories and the App Pack child list - a packaging-model hint: Stage is claimed only from an exactly cased `module.json` entry outside a classified resource path, FA only from `config.json` outside a classified resource path, otherwise `unknown` @@ -55,6 +57,24 @@ being classified as code files. Ark and JavaScript runtime hints summarize the classified bytecode and JavaScript lists; supported source extensions share the same classification rule as their component rows. +`unclassified_components` preserves files such as `pkgContextInfo.json`, +`.pages.info`, a HarmonyOS `.har` shared library, and a non-certificate +`META-INF` entry without interpreting their contents. The `.har` suffix also +names HTTP Archive JSON, so identifying one needs verified ZIP bytes rather than +a path rule. Different paths sharing the same bytes remain separate entries. App +Pack children of an App Pack root are already reported in `app_pack_children`, +so they are not repeated as unclassified files. A nested `.hap` or `.hsp` member +of a HAP or HSP root is disclosed in `unclassified_components` instead, because +child-package inventory is not recursive. Unreadable or encrypted content remains +in `unavailable_components` with unknown identity instead. + +`coverage.status: "complete-within-inventory"` describes the authenticated +source inventory and bounded bridge projection, not semantic understanding of +every file. Unclassified files do not make a complete inventory partial; missing +inventory pages, source coverage limits, unavailable content and bridge limits +still qualify coverage independently. An empty unclassified list on a partial +inventory does not establish that the whole package has no unclassified files. + Every claim is path-presence only. Manifest semantics (module identity, abilities, entry points, permissions, dependencies), Ark bytecode contents and versions, resource resolution, signing verification, and observed native diff --git a/src/contracts/applicationToolContracts.ts b/src/contracts/applicationToolContracts.ts index e8bb3cfb27..fe47b7f5d7 100644 --- a/src/contracts/applicationToolContracts.ts +++ b/src/contracts/applicationToolContracts.ts @@ -372,7 +372,7 @@ export const APPLICATION_TOOL_CONTRACTS = [ name: "project_harmony_application_graph", ...toolContractMetadata("project_harmony_application_graph"), description: - "Project authenticated HarmonyOS HAP, HSP, or App Pack inventory Evidence into an execution-free application inventory. Reports packaging-model hints (Stage module.json versus FA config.json), exact component paths and hashes (manifests, Ark bytecode, resources, native libraries, JavaScript, signing), App Pack child packages, runtime-family hints, and path-based N-API bridge hypotheses without decoding Ark bytecode, parsing manifest semantics, or claiming observed runtime calls.", + "Project authenticated HarmonyOS HAP, HSP, or App Pack inventory Evidence into an execution-free application inventory. Reports packaging-model hints (Stage module.json versus FA config.json), exact component paths and hashes (manifests, Ark bytecode, resources, native libraries, JavaScript, signing), authenticated files outside those categories in unclassified_components, App Pack child packages, runtime-family hints, and path-based N-API bridge hypotheses without decoding Ark bytecode, parsing manifest semantics, or claiming observed runtime calls.", kind: "application", inputSchema: applicationInventoryProjectionInputSchema, outputSchema: evidenceResultOf(harmonyApplicationProjectionResultSchema), diff --git a/src/domain/harmony/harmonyApplication.ts b/src/domain/harmony/harmonyApplication.ts index 9bc297e78b..1e52571bb1 100644 --- a/src/domain/harmony/harmonyApplication.ts +++ b/src/domain/harmony/harmonyApplication.ts @@ -43,6 +43,7 @@ export const harmonyApplicationProjectionResultSchema = z.strictObject({ javascript: z.array(projectedComponentSchema), signing: z.array(projectedComponentSchema), }), + unclassified_components: z.array(projectedComponentSchema), unavailable_components: z.array(unavailableComponentSchema), app_pack_children: z.array( z.union([projectedComponentSchema, unavailableComponentSchema]), @@ -97,6 +98,10 @@ export const projectHarmonyInventory = ( } const manifests = classifyManifests(fileEntries); const classified = classify(all, manifests.components); + const classifiedFiles = new Set([ + ...Object.values(classified).flat(), + ...children, + ]); const bridgeProjection = bridgeCandidates( classified.bytecode, classified.native_libraries, @@ -138,6 +143,9 @@ export const projectHarmonyInventory = ( .sort(compareUnicodeCodePoints), packaging_model: manifests.packagingModel, components: classified, + unclassified_components: all.filter( + (component) => !classifiedFiles.has(component), + ), unavailable_components: unavailable, app_pack_children: children, runtime_families: runtimeFamilies(all, classified), diff --git a/tests/acceptance/applications/mobileDirectoryComponents.test.ts b/tests/acceptance/applications/mobileDirectoryComponents.test.ts index 57bbb233f0..ed39edbfe1 100644 --- a/tests/acceptance/applications/mobileDirectoryComponents.test.ts +++ b/tests/acceptance/applications/mobileDirectoryComponents.test.ts @@ -260,6 +260,7 @@ cliTest.for(fixtures)( inventory_complete: true, }); if ("packaging_model" in projection) { + expect(projection.unclassified_components, mode).toEqual([]); expect(projection.packaging_model, mode).toBe( mode === "files" ? "stage" : "unknown", ); diff --git a/tests/acceptance/harmonyArtifactJourney.test.ts b/tests/acceptance/harmonyArtifactJourney.test.ts index c0b78a2c6d..2b383e292c 100644 --- a/tests/acceptance/harmonyArtifactJourney.test.ts +++ b/tests/acceptance/harmonyArtifactJourney.test.ts @@ -59,6 +59,8 @@ const harmonyFixture = async () => { await writer.add("ets/current", new TextReader("modules.abc"), { unixMode: 0o120777, }); + await writer.add("pkgContextInfo.json", new TextReader("{}")); + await writer.add(".pages.info", new TextReader("page metadata fixture")); const javascriptPaths = ["cts", "jsx", "mts", "ts", "tsx"].map( (extension) => `assets/js/index.${extension}`, ); @@ -67,6 +69,43 @@ const harmonyFixture = async () => { return { bytes: await writer.close(), javascriptPaths }; }; +/** Assert the projected Harmony graph for one archive format over the MCP adapter. */ +const expectHarmonyProjection = ( + graph: HarmonyApplicationProjectionResult, + format: string, + sha256: string, + javascriptPaths: readonly string[], +): void => { + expect(graph).toMatchObject({ + root_format: format, + root_sha256: sha256, + coverage: { status: "complete-within-inventory" }, + unavailable_components: [], + unclassified_components: [ + expect.objectContaining({ + path: ".pages.info", + sha256: createHash("sha256") + .update("page metadata fixture") + .digest("hex"), + }), + expect.objectContaining({ + path: "pkgContextInfo.json", + sha256: createHash("sha256").update("{}").digest("hex"), + }), + ], + }); + expect(graph.components.manifests.map(({ path: member }) => member)).toEqual([ + "module.json", + ]); + expect(graph.components.bytecode.map(({ path: member }) => member)).toEqual([ + "ets/modules.abc", + ]); + expect(graph.components.javascript.map(({ path: member }) => member)).toEqual( + javascriptPaths, + ); + expect(graph.runtime_families).toEqual(["ark", "javascript"]); +}; + test("preserves Harmony archive formats through the public CLI/MCP lifecycle", async ({ cli, workspace, @@ -157,22 +196,7 @@ test("preserves Harmony archive formats through the public CLI/MCP lifecycle", a const graph = harmonyApplicationProjectionResultSchema.parse( parseEvidence(projected.structuredContent).normalized_result, ); - expect(graph).toMatchObject({ - root_format: format, - root_sha256: sha256, - coverage: { status: "complete-within-inventory" }, - unavailable_components: [], - }); - expect( - graph.components.manifests.map(({ path: member }) => member), - ).toEqual(["module.json"]); - expect( - graph.components.bytecode.map(({ path: member }) => member), - ).toEqual(["ets/modules.abc"]); - expect( - graph.components.javascript.map(({ path: member }) => member), - ).toEqual(javascriptPaths); - expect(graph.runtime_families).toEqual(["ark", "javascript"]); + expectHarmonyProjection(graph, format, sha256, javascriptPaths); const projectionInput = workspace.path("projection-input.json"); await writeFile( projectionInput, diff --git a/tests/boundary/filesystem/harmony/application.test.ts b/tests/boundary/filesystem/harmony/application.test.ts index 1e3718734a..1406bcb14f 100644 --- a/tests/boundary/filesystem/harmony/application.test.ts +++ b/tests/boundary/filesystem/harmony/application.test.ts @@ -6,6 +6,7 @@ import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; import { describe, expect, it } from "vitest"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; +import { fragmentInventoryEvidence } from "../../../fixtures/artifactEvidence.js"; import { createNonApplicationZipInventory, inventoryApplicationPackage, @@ -18,6 +19,7 @@ import { projectHarmonyApplicationEvidence } from "../../../../src/application/h import { createDirectAnalysis } from "../../../../src/composition/directAnalysis.js"; import { harmonyApplicationProjectionResultSchema } from "../../../../src/domain/harmony/harmonyApplication.js"; import { artifactInventoryResultSchema } from "../../../../src/domain/artifactGraph.js"; +import { createEvidence } from "../../../../src/domain/evidence.js"; import { parseBinaryTarget } from "../../../../src/application/BinaryTargetResolver.js"; import { ZIP_PACKAGE_FORMATS, @@ -37,6 +39,62 @@ const stageHapEntries = [ { path: "META-INF/FIXTURE.p7b", content: "opaque signing" }, ] as const; +describe("HarmonyOS unclassified inventory", () => { + it("retains distinct unclassified paths sharing content with a classified file", async () => { + const path = await writeApplicationZip( + "rea-harmony-unclassified-", + "Fixture.hap", + [ + { path: "module.json", content: "{}" }, + { path: "pkgContextInfo.json", content: "{}" }, + { path: ".pages.info", content: "{}" }, + { path: "metadata-copy.json", content: "{}" }, + { path: "resources/base/bridge.js", content: "bridge();" }, + ], + ); + const inventory = await inventoryApplicationPackage(path); + const original = JSON.stringify(inventory); + const projection = requireDeterministicProjection( + projectHarmonyApplicationEvidence, + inventory, + harmonyApplicationProjectionResultSchema, + ); + expect(projection.unclassified_components).toEqual([ + expect.objectContaining({ + path: ".pages.info", + sha256: + "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + }), + expect.objectContaining({ path: "metadata-copy.json" }), + expect.objectContaining({ path: "pkgContextInfo.json" }), + ]); + expect(projection.components.resources.map(({ path }) => path)).toEqual([ + "resources/base/bridge.js", + ]); + expect(projection.components.javascript.map(({ path }) => path)).toEqual([ + "resources/base/bridge.js", + ]); + expect(projection.coverage).toEqual({ + status: "complete-within-inventory", + inventory_complete: true, + }); + const pages = fragmentInventoryEvidence(inventory, 3); + const assembled = requireSuccessfulProjection( + projectHarmonyApplicationEvidence({ + inventory_evidence: [...pages].reverse(), + }), + ); + const fromPages = harmonyApplicationProjectionResultSchema.parse( + assembled.normalized_result, + ); + expect(fromPages.unclassified_components).toEqual( + projection.unclassified_components, + ); + expect(fromPages.coverage).toEqual(projection.coverage); + expect(JSON.stringify(inventory)).toBe(original); + }); +}); + it("retains inventoried certificate-block suffixes without claiming verified signatures", async () => { const names = ["CERT.RSA", "CERT.DSA", "CERT.EC", "FIXTURE.p7b"]; const path = await writeApplicationZip( @@ -147,6 +205,7 @@ describe("HarmonyOS application projection", () => { }), ]); expect(left.app_pack_children).toEqual([]); + expect(left.unclassified_components).toEqual([]); expect(JSON.stringify(left)).not.toContain("opaque signing"); expect(left.limitations).toContain( "Manifest, module, resource, and signing semantics require a dedicated HarmonyOS provider; this projection reports exact inventory paths and hashes only.", @@ -189,6 +248,7 @@ describe("HarmonyOS application projection", () => { expect(projection.app_pack_children.map(({ path }) => path).sort()).toEqual( ["entry.hap", "feature.hsp"], ); + expect(projection.unclassified_components).toEqual([]); expect(projection.packaging_model).toBe("unknown"); expect(projection.limitations).toContain( "App Pack child packages are listed by inventory path only; their contents are not recursively inventoried by this projection.", @@ -312,6 +372,76 @@ describe("HarmonyOS packaging boundaries", () => { }); }); +describe("HarmonyOS partial inventory classification", () => { + it("does not claim complete classification from a missing inventory occurrence", async () => { + const path = await writeApplicationZip( + "rea-harmony-partial-", + "Fixture.hap", + [ + { path: "module.json", content: "{}" }, + { path: ".pages.info", content: "page metadata" }, + ], + ); + const inventory = await inventoryApplicationPackage(path); + if (inventory.subject === null) throw new Error("Missing fixture subject"); + const result = artifactInventoryResultSchema.parse( + inventory.normalized_result, + ); + const partial = createEvidence( + { + path: inventory.subject.local_path, + sha256: inventory.subject.digest.sha256, + format: inventory.subject.format, + }, + inventory.provider, + { + operation: inventory.operation, + parameters: inventory.parameters, + result: { + ...result, + occurrences: result.occurrences.filter( + ({ logical_path }) => logical_path !== ".pages.info", + ), + }, + confidence: "derived", + authority: inventory.authority, + }, + ); + const projection = requireDeterministicProjection( + projectHarmonyApplicationEvidence, + partial, + harmonyApplicationProjectionResultSchema, + ); + expect(projection.unclassified_components).toEqual([]); + // Coverage is the structured fact a partial inventory reports, and the two + // independent causes of a partial projection stay empty, so `partial` is + // attributable to the missing occurrence alone. The prose limitation that + // accompanies it is explanatory wording, so it stays in review rather than + // in a substring assertion. + expect(projection.coverage).toEqual({ + status: "partial", + inventory_complete: false, + }); + expect(projection.unavailable_components).toEqual([]); + expect(projection.bridge_candidate_coverage).toEqual({ + status: "complete", + total_candidates: 0, + emitted_candidates: 0, + omitted_candidates: 0, + }); + const tampered = { + ...partial, + normalized_result: inventory.normalized_result, + }; + expect( + projectHarmonyApplicationEvidence({ inventory_evidence: [tampered] }), + ).toMatchObject({ + ok: false, + error: { _tag: "EvidenceIntegrityError" }, + }); + }); +}); + describe("HarmonyOS inventory coverage", () => { it("retains complete traversal and case-collision limits in the application projection", async () => { const path = await writeApplicationZip( @@ -375,6 +505,9 @@ describe("HarmonyOS inventory coverage", () => { await writer.add("ets/modules.abc", new TextReader("opaque bytecode"), { password: "fixture-only", }); + await writer.add("pkgContextInfo.json", new TextReader("{}"), { + password: "fixture-only", + }); await writeFile(path, await writer.close()); const evidence = await inventoryApplicationPackage(path); const projection = requireDeterministicProjection( @@ -383,6 +516,7 @@ describe("HarmonyOS inventory coverage", () => { harmonyApplicationProjectionResultSchema, ); expect(projection.components.bytecode).toEqual([]); + expect(projection.unclassified_components).toEqual([]); expect(projection.unavailable_components).toEqual([ expect.objectContaining({ path: "ets/modules.abc", @@ -390,6 +524,12 @@ describe("HarmonyOS inventory coverage", () => { sha256: null, encrypted: true, }), + expect.objectContaining({ + path: "pkgContextInfo.json", + artifact_id: null, + sha256: null, + encrypted: true, + }), ]); expect(projection.coverage).toEqual({ status: "partial", @@ -421,6 +561,11 @@ describe("HarmonyOS manifest path rules", () => { ); expect(projection.packaging_model).toBe("unknown"); expect(projection.components.manifests).toEqual([]); + // Exact-case manifest names only, so these members belong to no manifest + // category and are disclosed as unclassified rather than silently dropped. + expect( + projection.unclassified_components.map(({ path }) => path).sort(), + ).toEqual(["CONFIG.JSON", "MODULE.JSON", "PACK.INFO"]); }); it.each([ From 54d819f3822cb67dcff121753840f8f014a4bc10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E4=BD=B3=E6=B5=A9Alben?= <35291372+XiaoBinGan@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:45:02 +0800 Subject: [PATCH 48/94] fix(javascript): emit no timer for a literal non-function callback (#8044) * fix(javascript): emit no timer for a literal non-function callback Node validates the timer callback before creating a handle: if (typeof callback !== 'function') throw new ERR_INVALID_ARG_TYPE('callback', 'function', callback); The throw is synchronous, before any timer exists, but the analysis reported a `schedule` operation for such calls. Measured on Node 24.22.1: setTimeout("code", 0) -> THROWS ERR_INVALID_ARG_TYPE setTimeout(`code`, 0) -> THROWS ERR_INVALID_ARG_TYPE setTimeout(1, 0) -> THROWS ERR_INVALID_ARG_TYPE setTimeout(1n, 0) -> THROWS ERR_INVALID_ARG_TYPE setTimeout(true, 0) -> THROWS ERR_INVALID_ARG_TYPE setTimeout(null, 0) -> THROWS ERR_INVALID_ARG_TYPE setTimeout(void 0, 0) -> THROWS ERR_INVALID_ARG_TYPE setTimeout([], 0) -> THROWS ERR_INVALID_ARG_TYPE setTimeout({}, 0) -> THROWS ERR_INVALID_ARG_TYPE setTimeout() -> THROWS ERR_INVALID_ARG_TYPE setImmediate("code") -> THROWS ERR_INVALID_ARG_TYPE setInterval("code", 0) -> THROWS ERR_INVALID_ARG_TYPE --- accepted by Node --- setTimeout(function(){}, 0) -> OK setTimeout(() => {}, 0) -> OK setTimeout(fn, 0) -> OK setTimeout(class {}, 0) -> OK (fails only when the class is invoked) So a schedule is dropped only for an exactly readable literal that is not a function, or for a missing argument. An identifier, a call result or a spread may still be a function at runtime and keeps its schedule. * chore: retrigger title and readiness checks after the title was corrected * fix(javascript): scope the timer callback rule to Node's timers module greptile flagged two problems with the first version. P1 - browser timers disappeared. The rule was applied to bare globals, but browser and Electron renderer sources reach this analysis without a Node/browser distinction, and there `setTimeout("refresh()", 0)` is a real timer: the host evaluates the string. The rule now applies only when the timer is imported from `node:timers`, where that form cannot exist. A bare global schedule keeps its fact whatever argument 0 holds. P2 - unused import. `readExactJavaScriptLiteral` was imported but never used; the helper reads shapes through `unwrapJavaScriptExpression`. Removed. Measured on Node 24.22.1 (the throw is what makes a node:timers schedule impossible; the bare-global case is intentionally left alone): import { setTimeout } from "node:timers"; setTimeout("code", 0) -> THROWS ERR_INVALID_ARG_TYPE setTimeout(null, 0) -> THROWS ERR_INVALID_ARG_TYPE setTimeout(fn, 0) -> OK --- bare global, host may be a browser --- setTimeout("code", 0) -> kept as a schedule 24 cases cover both sides plus the cancellation link. * fix(javascript): reject every non-function global and unary timer callback Reviewing the timer callback guard against Node 24.22.1 found two gaps. Both are in the same "a call that always throws must not emit a resolved fact" family. **An unmodified global that is not a function was accepted.** The guard read literal nodes only, so a bare global identifier passed through even though Node rejects it before creating a handle: setTimeout(undefined) -> THREW ERR_INVALID_ARG_TYPE setTimeout(NaN) -> THREW ERR_INVALID_ARG_TYPE setTimeout(Infinity) -> THREW ERR_INVALID_ARG_TYPE `isUnmodifiedGlobalReference` keeps a shadowed binding out of the rule, so a local `const NaN = () => {}` keeps its schedule. **Only `void` was treated as a non-function unary operator.** Every unary operator yields a primitive, and Node rejects a primitive callback: setTimeout(!0) -> THREW ERR_INVALID_ARG_TYPE setTimeout(~0) -> THREW ERR_INVALID_ARG_TYPE setTimeout(-1) -> THREW ERR_INVALID_ARG_TYPE setTimeout(+"1") -> THREW ERR_INVALID_ARG_TYPE setTimeout(typeof 1) -> THREW ERR_INVALID_ARG_TYPE A class expression is still kept, because it passes Node's own `typeof` check and only fails when invoked. Fifteen regressions cover the rejected forms plus arrow, identifier, class, spread and shadowed-global cases; ten fail on the previous commit. * fix(javascript): let sloppy eval shadow the non-function global callback Review finding. The new global check read `undefined`, `NaN` and `Infinity` as unmodified globals, but sloppy `eval` can introduce a local function under one of those names, which then shadows the global. Measured on Node 24.22.1 in script scope: function g() { eval("var undefined = () => {}"); setTimeout(undefined, 0); } -> schedules a timer function g() { eval("var NaN = () => {}"); setTimeout(NaN, 0); } -> schedules a timer `semanticOpaqueNameMayHaveChanged` now has to agree that no visible activation can introduce such a name before the global reading is conclusive. In a module the same `eval` is strict and introduces no outer binding, so the global reading still holds and the timer is still dropped. Three regressions cover the sloppy forms, and a fourth pins the no-eval cases as still dropped. The three sloppy cases fail without the guard. --- ...ptSemanticAnalysis.timer_callbacks.test.ts | 162 ++++++++++++++++++ .../javascriptSemanticAsyncEffects.ts | 70 ++++++++ 2 files changed, 232 insertions(+) create mode 100644 src/domain/javascript/javascriptSemanticAnalysis.timer_callbacks.test.ts diff --git a/src/domain/javascript/javascriptSemanticAnalysis.timer_callbacks.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.timer_callbacks.test.ts new file mode 100644 index 0000000000..3587a2c7d0 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticAnalysis.timer_callbacks.test.ts @@ -0,0 +1,162 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js"; + +/** + * A schedule imported from Node's `timers` module requires a function in + * argument 0. Node validates it before creating a handle, so a provable literal + * that is not a function throws `ERR_INVALID_ARG_TYPE: The "callback" argument + * must be of type function` synchronously and no timer exists. + * + * Only exactly readable literals are decided: an identifier or a call may still + * be a function at runtime. + */ +const nodeTimersCalls = (call: string) => + analyzeJavaScriptSemantics( + `import { setTimeout, setInterval, setImmediate } from "node:timers"; function g(fn) { ${call}; }`, + ).timerOperations.map(({ kind, method }) => ({ kind, method })); + +/** A bare global timer may belong to a browser, where a string is evaluated. */ +const globalTimerCalls = (call: string) => + analyzeJavaScriptSemantics(`function g(fn) { ${call}; }`).timerOperations.map( + ({ kind, method }) => ({ kind, method }), + ); + +it.each([ + ["a string callback", `setTimeout("code", 0)`], + ["a template-literal callback", "setTimeout(`code`, 0)"], + ["a numeric callback", `setTimeout(1, 0)`], + ["a BigInt callback", `setTimeout(1n, 0)`], + ["a boolean callback", `setTimeout(true, 0)`], + ["a null callback", `setTimeout(null, 0)`], + ["a void callback", `setTimeout(void 0, 0)`], + ["an array callback", `setTimeout([], 0)`], + ["an object callback", `setTimeout({}, 0)`], + ["no arguments at all", `setTimeout()`], + ["a setImmediate string callback", `setImmediate("code")`], + ["a setInterval string callback", `setInterval("code", 0)`], +] as const)("emits no node timers schedule for %s", (_name, call) => { + expect(nodeTimersCalls(call)).toEqual([]); +}); + +it.each([ + ["a function expression", `setTimeout(function () {}, 0)`, "setTimeout"], + ["an arrow function", `setTimeout(() => {}, 0)`, "setTimeout"], + ["an identifier", `setTimeout(fn, 0)`, "setTimeout"], + ["an identifier without a delay", `setTimeout(fn)`, "setTimeout"], + ["a call result", `setTimeout(select(), 0)`, "setTimeout"], + ["a class expression", `setTimeout(class {}, 0)`, "setTimeout"], + ["a resolved setImmediate", `setImmediate(fn)`, "setImmediate"], + ["a resolved setInterval", `setInterval(fn, 10)`, "setInterval"], +] as const)("keeps the node timers schedule for %s", (_name, call, method) => { + expect(nodeTimersCalls(call)).toEqual([{ kind: "schedule", method }]); +}); + +/** + * Browser and Electron renderer sources reach this analysis without a + * Node/browser distinction, and there a string callback is evaluated by the + * host. A bare global timer therefore keeps its schedule whatever the argument. + */ +it.each([ + ["a string callback", `setTimeout("code", 0)`], + ["a string setInterval callback", `setInterval("code", 0)`], + ["a null callback", `setTimeout(null, 0)`], +] as const)("keeps the bare global timer for %s", (_name, call) => { + expect(globalTimerCalls(call)).toHaveLength(1); +}); + +it("keeps a cancellation whose handle is a real timer", () => { + const operations = analyzeJavaScriptSemantics(` + import { setTimeout } from "node:timers"; + const handle = setTimeout(() => {}, 10); + clearTimeout(handle); + `).timerOperations; + expect(operations.map(({ kind }) => kind)).toEqual(["schedule", "cancel"]); + expect(operations[1]?.resolution).toBe("complete"); +}); + +/** + * `undefined`, `NaN` and `Infinity` are unmodified globals and none is a + * function, and every unary operator yields a primitive rather than a function. + * Node rejects each before creating a handle: + * + * setTimeout(undefined) -> THREW ERR_INVALID_ARG_TYPE + * setTimeout(NaN) -> THREW ERR_INVALID_ARG_TYPE + * setTimeout(Infinity) -> THREW ERR_INVALID_ARG_TYPE + * setTimeout(!0) -> THREW ERR_INVALID_ARG_TYPE + * setTimeout(-1) -> THREW ERR_INVALID_ARG_TYPE + * setTimeout(typeof 1) -> THREW ERR_INVALID_ARG_TYPE + * --- still scheduled --- + * setTimeout(() => {}) -> ok + * setTimeout(class {}) -> ok, a class passes Node's typeof check + */ +it.each([ + ["an undefined callback", `setTimeout(undefined)`], + ["a NaN callback", `setTimeout(NaN)`], + ["an Infinity callback", `setTimeout(Infinity)`], + ["a void callback", `setTimeout(void 0)`], + ["a logical-not callback", `setTimeout(!0)`], + ["a bitwise-not callback", `setTimeout(~0)`], + ["a negated callback", `setTimeout(-1)`], + ["a unary-plus callback", `setTimeout(+"1")`], + ["a typeof callback", `setTimeout(typeof 1)`], + ["a NaN immediate callback", `setImmediate(NaN)`], + ["an Infinity interval callback", `setInterval(Infinity)`], +] as const)("emits no timer for %s", (_name, call) => { + expect(nodeTimersCalls(call)).toEqual([]); +}); + +it.each([ + ["an arrow callback", `setTimeout(() => {})`, "setTimeout"], + ["an identifier callback", `setTimeout(handler)`, "setTimeout"], + ["a class callback", `setTimeout(class {})`, "setTimeout"], + ["an immediate arrow callback", `setImmediate(() => {})`, "setImmediate"], +] as const)("keeps the timer for %s", (_name, call, method) => { + expect(nodeTimersCalls(call)).toEqual([{ kind: "schedule", method }]); +}); + +it("keeps the timer when a local binding shadows the global", () => { + expect(nodeTimersCalls(`const NaN = () => {}; setTimeout(NaN)`)).toEqual([ + { kind: "schedule", method: "setTimeout" }, + ]); + expect( + nodeTimersCalls(`const undefined = () => {}; setTimeout(undefined)`), + ).toEqual([{ kind: "schedule", method: "setTimeout" }]); +}); + +/** + * Sloppy `eval` can introduce a local binding named `undefined`, `NaN` or + * `Infinity`, which then shadows the global the guard would otherwise read. The + * opaque-effects owner has to agree before the global reading is conclusive: + * + * function g() { eval("var undefined = () => {}"); setTimeout(undefined, 0); } + * -> schedules a timer in Node (sloppy script scope) + * + * In a module the same `eval` is strict and introduces no outer binding, so the + * global reading still holds. + */ +const sloppyTimersCalls = (call: string) => + analyzeJavaScriptSemantics( + `const { setTimeout } = require("node:timers"); function g() { ${call}; }`, + ).timerOperations.map(({ kind, method }) => ({ kind, method })); + +it.each([ + [ + "an eval-introduced undefined", + `eval('var undefined = () => {}'); setTimeout(undefined, 0)`, + ], + ["an eval-introduced NaN", `eval('var NaN = () => {}'); setTimeout(NaN, 0)`], + [ + "an eval-introduced Infinity", + `eval('var Infinity = () => {}'); setTimeout(Infinity, 0)`, + ], +] as const)("keeps the timer after %s", (_name, call) => { + expect(sloppyTimersCalls(call)).toEqual([ + { kind: "schedule", method: "setTimeout" }, + ]); +}); + +it("still drops a global callback when no eval can shadow it", () => { + expect(sloppyTimersCalls(`setTimeout(undefined, 0)`)).toEqual([]); + expect(sloppyTimersCalls(`setTimeout(NaN, 0)`)).toEqual([]); +}); diff --git a/src/domain/javascript/javascriptSemanticAsyncEffects.ts b/src/domain/javascript/javascriptSemanticAsyncEffects.ts index ea9e990bd1..4c5d56fa4b 100644 --- a/src/domain/javascript/javascriptSemanticAsyncEffects.ts +++ b/src/domain/javascript/javascriptSemanticAsyncEffects.ts @@ -21,6 +21,7 @@ import { import { semanticBindingAtRead } from "./javascriptSemanticMutationInitializers.js"; import { traverseJavaScriptValueAst } from "./javascriptSemanticTraversal.js"; import { isUnmodifiedGlobalReference } from "./javascriptSemanticAmbientWrites.js"; +import { semanticOpaqueNameMayHaveChanged } from "./javascriptSemanticOpaqueEffects.js"; import { range } from "./javascriptStaticAnalysisHelpers.js"; import { builtinDataEffectMethod, @@ -286,10 +287,79 @@ const emitterIdentity = ( }; }; +/** + * A timer proven to come from the Node `timers` module rather than a bare + * global. Browser and Electron renderer sources reach this analysis without a + * Node/browser distinction, and there `setTimeout("refresh()", 0)` is a real + * timer: the string is evaluated by the host. Node's callback rule therefore + * applies only when the timer is imported from `node:timers`, where that form + * cannot exist. + */ +const nodeBuiltinTimerMethod = ( + callee: t.Node, + state: JavaScriptSemanticAnalysisState, +): JavaScriptSemanticTimerOperation["method"] | null => + admittedTimerMethod( + builtinDataEffectMethod(callee, state, ["timers", "node:timers"]) ?? "", + ); + +/** + * A schedule imported from Node's `timers` module requires a function in + * argument 0. Node validates it before creating a handle, so a provable literal + * that is not a function throws ERR_INVALID_ARG_TYPE synchronously and no timer + * exists. Only literals the analyzer can read exactly are decided; an identifier + * or call may still be a function at runtime. + */ +const schedulesWithNonFunctionCallback = ( + node: t.Node, + state: JavaScriptSemanticAnalysisState, +): boolean => { + if (!t.isCallExpression(node) && !t.isOptionalCallExpression(node)) + return false; + const argument = node.arguments[0]; + // No argument at all cannot be a function. A spread may still expand to one. + if (argument === undefined) return true; + if (t.isSpreadElement(argument)) return false; + const value = unwrapJavaScriptExpression(argument).node; + if (t.isFunctionExpression(value) || t.isArrowFunctionExpression(value)) + return false; + if ( + t.isStringLiteral(value) || + t.isNumericLiteral(value) || + t.isBigIntLiteral(value) || + t.isBooleanLiteral(value) || + t.isNullLiteral(value) || + t.isTemplateLiteral(value) || + t.isArrayExpression(value) || + t.isObjectExpression(value) + ) + return true; + // `undefined`, `NaN` and `Infinity` are unmodified globals, and none is a + // function. A shadowed binding may still hold one, so only the global counts. + if ( + t.isIdentifier(value) && + ["undefined", "NaN", "Infinity"].includes(value.name) && + isUnmodifiedGlobalReference(value, state, value.name) && + !semanticOpaqueNameMayHaveChanged(value, state) + ) + return true; + // A class passes Node's own typeof check; invoking it is a separate failure, + // so the schedule itself still happens. Every other unary operator (including + // `void`, `!`, `~`, `typeof` and `delete`) yields a primitive, never a + // function. + return t.isUnaryExpression(value); +}; + const collectTimer = (input: TimerCollectionInput): void => { const { node, ownerCallableId, state, output, ancestors } = input; const method = timerMethod(node.callee, state); if (method === null) return; + if ( + TIMER_SCHEDULE_METHODS.some((candidate) => candidate === method) && + nodeBuiltinTimerMethod(node.callee, state) !== null && + schedulesWithNonFunctionCallback(node, input.state) + ) + return; const kind = TIMER_SCHEDULE_METHODS.some((candidate) => candidate === method) ? "schedule" : "cancel"; From b9138a12613e35ad8df27015c495a6c112d0e537 Mon Sep 17 00:00:00 2001 From: Lewis Wang Date: Wed, 4 Nov 2026 19:45:46 +0800 Subject: [PATCH 49/94] fix(native): select versioned arm64e slices for codesign by family name (#8713) * fix(native): select versioned arm64e slices for codesign by family name Current clang marks arm64e slices with pointer-authentication ABI version 1 (subtype 0x81000002, lipo name arm64e.v1). The codesign selector table accepted only ABI version 0, so inspect-signature failed with capability_unavailable whenever such a slice was selected, for example on any universal arm64e + x86_64 binary built today. codesign has no versioned name: -a arm64e.v1 is "unknown architecture name", and -a arm64e resolves to the first FAT slice with the arm64e base subtype, whatever its ABI version. A selector is therefore exact only for that first slice. Accept any versioned pointer-authentication ABI for arm64e, and return a selector only when the slice is the first in table order to carry that name; a later slice sharing it stays unknown instead of being inspected through the earlier one. This also removes the separate duplicate-name check from signature coverage. Co-Authored-By: Claude Opus 5.5 * fix(native): keep arm64e selectors unknown behind unmapped variants codesign resolves a name to the first FAT slice with the same CPU type and base subtype, so an earlier capability variant whose own name is unknown may shadow a later arm64e slice. Compare identities rather than successful name mappings. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- src/native/MachoArchitectureSelection.ts | 50 +++++--- src/native/NativeMacOSProvider.ts | 30 ++--- tests/boundary/cli/nativeSignatureCli.test.ts | 51 +++++++- .../native/nativeInventory.golden.test.ts | 112 +++++++++++++++++- 4 files changed, 209 insertions(+), 34 deletions(-) diff --git a/src/native/MachoArchitectureSelection.ts b/src/native/MachoArchitectureSelection.ts index 4c1e035e3f..8463faa0ec 100644 --- a/src/native/MachoArchitectureSelection.ts +++ b/src/native/MachoArchitectureSelection.ts @@ -46,33 +46,49 @@ export const machoSubtypeRank = ( return rank < 0 ? policy.subtypes.length : rank; }; -/** Return a tool selector only for an explicitly supported CPU subtype. */ -export const machoKnownCodeSignSelector = ( +/** codesign's architecture name for one FAT CPU identity, when known. */ +const codeSignArchitectureName = ( cpuType: number, cpuSubtype: number, ): string | undefined => { const capabilities = (cpuSubtype >>> 24) & 0xff; - // arm64e pointer authentication and x86_64 LIB64 have known tool selectors. - // Other capability variants may have distinct names; do not collapse them. + const base = machoCpuSubtypeBase(cpuSubtype); + // codesign names arm64e by its base subtype whatever its versioned + // pointer-authentication ABI (CPU_SUBTYPE_PTRAUTH_ABI plus a version), and + // x86_64 LIB64 as x86_64. Other capability variants may have distinct + // names; do not collapse them. if ( capabilities !== 0 && - !( - cpuType === 0x0100000c && - machoCpuSubtypeBase(cpuSubtype) === 2 && - capabilities === 0x80 - ) && - !( - cpuType === 0x01000007 && - machoCpuSubtypeBase(cpuSubtype) === 3 && - capabilities === 0x80 - ) + !(cpuType === 0x0100000c && base === 2 && (capabilities & 0xf0) === 0x80) && + !(cpuType === 0x01000007 && base === 3 && capabilities === 0x80) ) return undefined; return Object.values(MACHO_ARCHITECTURE_POLICIES) .find((policy) => policy.cpuType === cpuType) - ?.subtypes.find( - ({ subtype }) => subtype === machoCpuSubtypeBase(cpuSubtype), - )?.name; + ?.subtypes.find(({ subtype }) => subtype === base)?.name; +}; + +/** + * Return the codesign selector that names exactly the FAT slice at `index`. + * codesign resolves a name to the first slice in table order with that CPU + * type and base subtype, whatever its capability bits. A later slice sharing + * them has no exact selector, even behind an earlier variant whose own name + * is unknown. + */ +export const machoKnownCodeSignSelector = ( + slices: readonly { readonly cpuType: number; readonly cpuSubtype: number }[], + index: number, +): string | undefined => { + const slice = slices[index]; + if (slice === undefined) return undefined; + const base = machoCpuSubtypeBase(slice.cpuSubtype); + return slices.findIndex( + (other) => + other.cpuType === slice.cpuType && + machoCpuSubtypeBase(other.cpuSubtype) === base, + ) === index + ? codeSignArchitectureName(slice.cpuType, slice.cpuSubtype) + : undefined; }; /** Keep the selected lipo record, including its container coordinates. */ diff --git a/src/native/NativeMacOSProvider.ts b/src/native/NativeMacOSProvider.ts index 9269614512..104b3c853a 100644 --- a/src/native/NativeMacOSProvider.ts +++ b/src/native/NativeMacOSProvider.ts @@ -792,13 +792,15 @@ class NativeMacOSClient implements AnalysisClient { ), ); const target = this.target; - const selectedSlice = + const slices = + target.format === "mach-o-universal" ? target.universalSlices : undefined; + const selectedIndex = target.format === "mach-o-universal" - ? target.universalSlices?.find( + ? (slices?.findIndex( (slice) => slice.offset === target.universalSlice.offset, - ) - : undefined; - if (target.format === "mach-o-universal" && selectedSlice === undefined) + ) ?? -1) + : -1; + if (target.format === "mach-o-universal" && selectedIndex < 0) return err( new AnalysisCapabilityUnavailableError( IDENTITY.id, @@ -807,14 +809,11 @@ class NativeMacOSClient implements AnalysisClient { ), ); const selectedArchitecture = - selectedSlice === undefined + slices === undefined || selectedIndex < 0 ? architecture === "x86" ? "i386" : architecture - : machoKnownCodeSignSelector( - selectedSlice.cpuType, - selectedSlice.cpuSubtype, - ); + : machoKnownCodeSignSelector(slices, selectedIndex); if (selectedArchitecture === undefined) return err( new AnalysisCapabilityUnavailableError( @@ -1931,14 +1930,15 @@ const codeSignArchitectures = ( if (target.kind === "executable" && target.format === "mach-o-universal") { if (target.universalSlices === undefined) return { architectures: [], unknownSlices: true }; - const names = target.universalSlices.map(({ cpuType, cpuSubtype }) => - machoKnownCodeSignSelector(cpuType, cpuSubtype), + const slices = target.universalSlices; + // Each known selector names exactly one slice, so known names are unique. + const names = slices.map((_, index) => + machoKnownCodeSignSelector(slices, index), ); const known = names.filter((name): name is string => name !== undefined); return { - architectures: [...new Set(known)], - unknownSlices: - known.length !== names.length || new Set(known).size !== names.length, + architectures: known, + unknownSlices: known.length !== names.length, }; } const universal = /^Mach-O universal \(([^)\r\n]+)\)$/u.exec( diff --git a/tests/boundary/cli/nativeSignatureCli.test.ts b/tests/boundary/cli/nativeSignatureCli.test.ts index cae0929f86..f69de01daa 100644 --- a/tests/boundary/cli/nativeSignatureCli.test.ts +++ b/tests/boundary/cli/nativeSignatureCli.test.ts @@ -1,5 +1,5 @@ import { execFile } from "node:child_process"; -import { copyFile, mkdir } from "node:fs/promises"; +import { copyFile, mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; @@ -91,3 +91,52 @@ describe.skipIf(process.platform !== "darwin")( ); }, ); + +describe.skipIf(process.platform !== "darwin" || process.arch !== "arm64")( + "native signature CLI for a universal arm64e binary", + () => { + cliTest( + "inspects the selected arm64e slice whatever its pointer-authentication ABI version", + async ({ cli }) => { + const directory = await createTestTempDirectory( + "rea-signature-arm64e-", + ); + const source = join(directory, "main.c"); + await writeFile(source, "int main(void) { return 0; }\n"); + // Current clang marks arm64e slices with ABI version 1 (arm64e.v1). + const program = join(directory, "universal"); + await run("/usr/bin/xcrun", [ + "clang", + "-arch", + "arm64e", + "-arch", + "x86_64", + source, + "-o", + program, + ]); + const { stderr } = await run("/usr/bin/codesign", [ + "-d", + "-a", + "arm64e", + "--verbose=4", + program, + ]); + const cdhash = /^CDHash=([0-9a-f]+)$/mu.exec(stderr)?.[1]; + expect(cdhash).toBeDefined(); + const result = await cli.run({ + arguments: ["inspect-signature", program, "--json"], + environment: { + REA_LOG_LEVEL: "silent", + REA_ANALYSIS_PROVIDER: "auto", + }, + }); + expect(result.exitCode, result.stdout).toBe(0); + expect(result.json).toMatchObject({ + normalized_result: { signed: true, cdhashes: [cdhash] }, + }); + }, + 60_000, + ); + }, +); diff --git a/tests/boundary/providers/native/nativeInventory.golden.test.ts b/tests/boundary/providers/native/nativeInventory.golden.test.ts index 7d42727f2e..eeb8efc235 100644 --- a/tests/boundary/providers/native/nativeInventory.golden.test.ts +++ b/tests/boundary/providers/native/nativeInventory.golden.test.ts @@ -269,7 +269,9 @@ it("retains distinct same-family FAT slices for signature coverage", async () => }); }); -it.each([42, 0x81000002])( +// 0xc1000002 carries an arm64e capability flag outside the versioned +// pointer-authentication ABI, so codesign may name it differently. +it.each([42, 0xc1000002])( "reports unknown signature coverage for unmapped FAT subtype %i", async (subtype) => { const path = join( @@ -312,6 +314,114 @@ it.each([42, 0x81000002])( }, ); +const ARM64E_V0 = { type: CPU.arm64.type, subtype: 0x80000002 } as const; +const ARM64E_V1 = { type: CPU.arm64.type, subtype: 0x81000002 } as const; + +it.each([ + ["arm64e.v1 beside x86_64", [ARM64E_V1, CPU.x86_64], false], + ["arm64e.v1 before arm64e", [ARM64E_V1, ARM64E_V0], true], + ["arm64e before arm64e.v1", [ARM64E_V0, ARM64E_V1], true], +] as const)( + "inspects the selected %s slice through codesign's arm64e selector", + async (_label, cpus, laterSliceShadowed) => { + const path = join( + await createTestTempDirectory("rea-native-arm64e-abi-"), + "fixture", + ); + await writeFile( + path, + fatImage(cpus.map((cpu) => ({ cpu, bytes: machoImage({ cpu }) }))), + ); + const header = parseExecutableHeader(await readFile(path), "arm64"); + if (!header.ok || header.value.format !== "mach-o-universal") + throw new Error("expected universal fixture"); + // Admission selects the first slice; codesign resolves -a arm64e to the + // first arm64e slice in table order, whatever its ABI version. + expect(header.value.universalSlice.offset).toBe( + header.value.universalSlices[0]?.offset, + ); + const replay = signatureRunner({ + signed: await fixture("codesign.txt"), + unsigned: await fixture("dyld-inventory/codesign-unsigned.txt"), + entitlements: await fixture("entitlements.xml"), + unsignedHeader: true, + }); + const calls: (readonly string[])[] = []; + const runner: NativeCommandRunner = { + run(tool, args, options) { + calls.push(args); + return replay.run(tool, args, options); + }, + }; + const client = new NativeMacOSProvider({}, runner, "darwin").createClient({ + ...(await nativeMachoTargetForFile(path)), + ...header.value, + }); + const result = await client.execute("inspect_signature", {}); + if (!result.ok) throw result.error; + expect(calls[0]).toEqual(expect.arrayContaining(["-a", "arm64e"])); + // A later slice with the same selector name has no exact selector. + const unknown = expect.stringContaining("signature states are unknown"); + expect( + inspectSignatureSchema.parse(result.value.result).limitations, + ).toEqual( + laterSliceShadowed + ? expect.arrayContaining([unknown]) + : expect.not.arrayContaining([unknown]), + ); + }, +); + +it("keeps an arm64e.v1 slice unknown behind an unmapped arm64e variant", async () => { + const path = join( + await createTestTempDirectory("rea-native-arm64e-shadow-"), + "fixture", + ); + const variant = { type: CPU.arm64.type, subtype: 0xc1000002 }; + await writeFile( + path, + fatImage( + [variant, ARM64E_V1, CPU.x86_64].map((cpu) => ({ + cpu, + bytes: machoImage({ cpu }), + })), + ), + ); + const header = parseExecutableHeader(await readFile(path), "x64"); + if (!header.ok || header.value.format !== "mach-o-universal") + throw new Error("expected universal fixture"); + expect(header.value.universalSlice.offset).toBe( + header.value.universalSlices[2]?.offset, + ); + const replay = signatureRunner({ + signed: await fixture("codesign.txt"), + unsigned: await fixture("dyld-inventory/codesign-unsigned.txt"), + entitlements: await fixture("entitlements.xml"), + unsignedHeader: true, + }); + const calls: (readonly string[])[] = []; + const runner: NativeCommandRunner = { + run(tool, args, options) { + calls.push(args); + return replay.run(tool, args, options); + }, + }; + const client = new NativeMacOSProvider({}, runner, "darwin").createClient({ + ...(await nativeMachoTargetForFile(path)), + ...header.value, + }); + const result = await client.execute("inspect_signature", {}); + if (!result.ok) throw result.error; + // codesign -a arm64e may resolve to the earlier variant, so neither arm64e + // slice has an exact selector. + expect(calls.filter((args) => args.includes("arm64e"))).toEqual([]); + expect(inspectSignatureSchema.parse(result.value.result).limitations).toEqual( + expect.arrayContaining([ + expect.stringContaining("signature states are unknown"), + ]), + ); +}); + it.each(["x64", "arm64"] as const)( "inspects an x86_64 LIB64 slice when %s is selected", async (host) => { From 7570aabd9d484c41738461253f66d793c4fc766e Mon Sep 17 00:00:00 2001 From: Beqa <91750923+BeqaKs@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:47:28 +0400 Subject: [PATCH 50/94] docs(skill): route SPA passive capture through allowed_origins (#7252) Default page-origin scope excludes cross-origin APIs. Point agents at JavaScript analysis for API-origin discovery, require page origin plus API origins in allowed_origins (list replaces default), and bump the packaged skill to version 60. --- .agents/skills/reverse-engineer-anything/SKILL.md | 11 ++++++++++- src/generatedPackageMetadata.ts | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index 2bafebfe53..a974b8c698 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -2,7 +2,7 @@ name: reverse-engineer-anything description: Reverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. metadata: - version: "59" + version: "60" --- # REA @@ -121,6 +121,15 @@ explicit path or endpoint and do not need it. cleanup effects, and do not require a preliminary static session unless their own contract says so. - User-owned browser page already open: `list_browser_targets`. +- Passive traffic capture of a SPA: the default scope is the page's own origin, + and data APIs usually live on another origin. Discover API origins from + absolute URLs in loaded bundles or script text via JavaScript analysis tools + (another passive capture without those origins still scopes to the page + origin, and `completeness.excluded` counts lack URLs). Pass both the page + origin and those API origins in `allowed_origins` — the list replaces the + default scope — and read `completeness.excluded` before reporting that an + API is absent. For long captures with body shapes, use + `export_evidence_bundle` instead of expecting an inline result. - Browser action the agent must perform (click, type, navigate): `capture_browser_scenario` with declared actions. See [runtime observation](references/runtime-observation.md) for connect/launch diff --git a/src/generatedPackageMetadata.ts b/src/generatedPackageMetadata.ts index 2d67071390..ce2709114c 100644 --- a/src/generatedPackageMetadata.ts +++ b/src/generatedPackageMetadata.ts @@ -6,5 +6,5 @@ export const PACKAGE_METADATA = { serverSdkVersion: "2.3.1", clientSdkVersion: "2.3.1", coreSdkVersion: "2.3.1", - skillVersion: "59", + skillVersion: "60", } as const; From 1dc9e8774d15716d7fa523e8945f4508af7b4895 Mon Sep 17 00:00:00 2001 From: Abel Yagubyan Date: Wed, 4 Nov 2026 03:48:12 -0800 Subject: [PATCH 51/94] fix(cli): keep marker-only Markdown paragraph facts literal (#7813) Fixes #7634 Co-authored-by: Abelo9996 --- src/cli/cliOutputFormatter.ts | 2 +- tests/boundary/cli/markdownLiteralCells.test.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/cli/cliOutputFormatter.ts b/src/cli/cliOutputFormatter.ts index ed4cd5910b..5fe9ecda17 100644 --- a/src/cli/cliOutputFormatter.ts +++ b/src/cli/cliOutputFormatter.ts @@ -350,7 +350,7 @@ const encodeMarkdownWhitespace = (whitespace: string): string => const escapeMarkdownText = (value: string): string => escapeTableCell(value) .replace(/^(\s*)([-+=])/, "$1\\$2") - .replace(/^(\s*\d{1,9})([.)])(?=\s)/, "$1\\$2") + .replace(/^(\s*\d{1,9})([.)])(?=\s|$)/, "$1\\$2") .replace(/^[ \t]+/, encodeMarkdownWhitespace) .replace(/[ \t]+$/u, encodeMarkdownWhitespace); diff --git a/tests/boundary/cli/markdownLiteralCells.test.ts b/tests/boundary/cli/markdownLiteralCells.test.ts index 36639b6875..9c3a8f2758 100644 --- a/tests/boundary/cli/markdownLiteralCells.test.ts +++ b/tests/boundary/cli/markdownLiteralCells.test.ts @@ -325,6 +325,8 @@ cliTest( list: [literal], bullet: "- literal", numbered: "1. literal", + numberedBare: "1.", + parenBare: "1)", rule: "---", indented: " literal\n- next\n# heading", nested: { value: 1 }, @@ -365,6 +367,8 @@ cliTest( expect(text(parseFragment(html))).toContain(literal); expect(html).toContain("**key**
# injected"); expect(html).toContain("literal
- next
# heading"); + expect(html).toContain("

1.

"); + expect(html).toContain("

1)

"); expect(html).not.toMatch(/<(?:strong|em|ul|ol|li|T|pre|hr)(?:\s|>)/u); expect(html).not.toContain('href="wrong"'); expect(html).not.toContain(" Date: Wed, 4 Nov 2026 11:48:56 +0000 Subject: [PATCH 52/94] test(javascript): cover Request endpoint identity and provenance (#6365) fix(javascript): resolve inline Request URLs in fetch calls --- ...iptStaticAnalysis.partialEndpoints.test.ts | 62 +++++++++++++++++++ .../javascriptArtifactGraphEndpoints.test.ts | 18 +++++- 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/src/domain/javascript/javascriptStaticAnalysis.partialEndpoints.test.ts b/src/domain/javascript/javascriptStaticAnalysis.partialEndpoints.test.ts index 37be9a1df8..b0b184efa5 100644 --- a/src/domain/javascript/javascriptStaticAnalysis.partialEndpoints.test.ts +++ b/src/domain/javascript/javascriptStaticAnalysis.partialEndpoints.test.ts @@ -10,6 +10,68 @@ const endpoints = (source: string) => resolution: endpoint.resolution, })); +it.each(["window.Request", "globalThis.Request", "self.Request"])( + "resolves an inline literal URL from the built-in %s constructor", + (constructor) => { + expect( + endpoints(`fetch(new ${constructor}("/api/chat", { method: "POST" }));`), + ).toEqual([ + { + kind: "network", + value: "/api/chat", + mechanism: "call:fetch", + resolution: undefined, + }, + ]); + }, +); + +it.each(["globalThis.fetch", "self.fetch", "fetch?."])( + "resolves an inline Request URL passed to %s", + (callee) => { + expect(endpoints(`${callee}(new Request(\`/api/chat\`));`)).toEqual([ + { + kind: "network", + value: "/api/chat", + mechanism: `call:${callee.replace("?.", "")}`, + resolution: undefined, + }, + ]); + }, +); + +it.each([ + [ + "const Request = custom;", + 'fetch(new Request("/api/chat"));', + 'new Request("/api/chat")', + ], + [ + "Request = custom;", + 'fetch(new Request("/api/chat"));', + 'new Request("/api/chat")', + ], + [ + "globalThis.Request = custom;", + 'fetch(new Request("/api/chat"));', + 'new Request("/api/chat")', + ], + [ + "const window = custom;", + 'fetch(new window.Request("/api/chat"));', + 'new window.Request("/api/chat")', + ], + ["", "fetch(new Request(target));", "new Request(target)"], + ["", "fetch(new Request(...urls));", "new Request(...urls)"], +])( + "retains an unproven Request target: %s %s", + (prefix, source, expression) => { + expect(endpoints(prefix + source)).toEqual([ + expect.objectContaining({ value: expression, resolution: "partial" }), + ]); + }, +); + it("resolves unchanged lexical strings and object properties", () => { expect( endpoints(` diff --git a/tests/boundary/filesystem/javascript-applications/javascriptArtifactGraphEndpoints.test.ts b/tests/boundary/filesystem/javascript-applications/javascriptArtifactGraphEndpoints.test.ts index 374dcb034b..86ffd5efdf 100644 --- a/tests/boundary/filesystem/javascript-applications/javascriptArtifactGraphEndpoints.test.ts +++ b/tests/boundary/filesystem/javascript-applications/javascriptArtifactGraphEndpoints.test.ts @@ -115,7 +115,7 @@ it("preserves partial and exact endpoint identity through graph projection", asy ); await writeFile( join(root, "app.js"), - 'fetch(target); fetch("target"); function other(target) { fetch(target); }', + 'fetch(target); fetch("target"); fetch(new Request("/api/chat", { method: "POST" })); function other(target) { fetch(target); }', ); const snapshot = await scanCanonicalArtifactInventory(root, {}); const reader = createJavaScriptArtifactReader(root, "directory"); @@ -154,6 +154,22 @@ it("preserves partial and exact endpoint identity through graph projection", asy limitations: [expect.stringContaining("not an exact literal")], }, }); + const request = endpoints.find(({ observations }) => + observations.some(({ properties }) => properties.value === "/api/chat"), + ); + expect(request?.observations[0]).toMatchObject({ + properties: { + endpoint_kind: "network", + value: "/api/chat", + mechanism: "call:fetch", + }, + evidence: { + authority: "ast-static-analysis", + state: "observed", + confidence: "exact", + }, + }); + expect(request?.observations[0]?.properties.resolution).toBeUndefined(); } finally { await reader.close(); await rm(root, { recursive: true, force: true }); From 4d2aff2ec6eadb0e8dd475748a410ca0f3a37687 Mon Sep 17 00:00:00 2001 From: Akshad Jogi <107343579+akshad-exe@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:19:40 +0530 Subject: [PATCH 53/94] fix(mcp): report session tool target requirements accurately (#7340) * fix(mcp): report session tool target requirements accurately - Declare requiresActiveTarget on inspect_address_context in sessionToolContracts.ts so closed-session discovery stops advertising an address query that fails with target_unavailable. - Route session-kind tools through targetAvailability in CapabilityInventory.ts before the available fallback, and report target_unsupported with mode-level remediation for get_navigation_context and inspect_address_context on artifact and archive targets. - Keep each navigation mode's missing_operations and remediation aligned with the target restriction so recommendations name the target instead of a provider. * test(mcp): cover closed-session and artifact target requirements - Add CapabilityInventory controls for closed-session address and explicit-path APK availability, artifact and archive restrictions with per-mode navigation flags, and structured reasons instead of pinned remediation sentences. - Add a production stdio MCP regression in sessionTargetRequirements.test.ts that observes both binary_session entries before and after the calls, the target_unavailable address error, the successful unsigned-ZIP APK call, and the sibling navigation prerequisite. --- src/application/CapabilityInventory.test.ts | 73 +++++++++ src/application/CapabilityInventory.ts | 38 +++-- src/contracts/sessionToolContracts.ts | 13 +- .../mcp/sessionTargetRequirements.test.ts | 150 ++++++++++++++++++ 4 files changed, 260 insertions(+), 14 deletions(-) create mode 100644 tests/boundary/mcp/sessionTargetRequirements.test.ts diff --git a/src/application/CapabilityInventory.test.ts b/src/application/CapabilityInventory.test.ts index 8425bd1ab9..074dd92d23 100644 --- a/src/application/CapabilityInventory.test.ts +++ b/src/application/CapabilityInventory.test.ts @@ -196,6 +196,54 @@ it("reports the active resource session's dispatch policy without limiting indep ); }); +it("derives closed-session address and explicit-path APK availability from their contracts", () => { + expect(entry("inspect_address_context", status())).toMatchObject({ + available: false, + reason: "target_required", + remediation: expect.any(String), + }); + expect(entry("inspect_apk_signature", status())).toMatchObject({ + available: true, + reason: "available", + remediation: null, + }); + expect( + entry( + "inspect_address_context", + status({ + open: true, + kind: "executable", + capabilities: [ + { + operation: "inspect_address_context", + available: true, + reason: null, + }, + ], + }), + ), + ).toMatchObject({ available: true, reason: "available" }); + expect(entry("inspect_apk_signature", status({ open: true }))).toMatchObject({ + available: true, + reason: "available", + }); + for (const kind of ["artifact", "archive"] as const) { + expect( + entry("inspect_address_context", status({ open: true, kind })), + ).toMatchObject({ + available: false, + reason: "target_unsupported", + remediation: expect.stringContaining("explicit document"), + }); + expect( + entry("inspect_apk_signature", status({ open: true, kind })), + ).toMatchObject({ + available: true, + reason: "available", + }); + } +}); + describe("ADB and Apktool host availability", () => { it("uses only their selected owner probes", () => { const ready = { @@ -513,6 +561,31 @@ describe("compact tool availability", () => { }); }); +it("reports target restrictions for navigation modes even when providers are declared", () => { + const capabilities = [ + "current_document", + "current_address", + "resolve_containing_procedure", + ].map((operation) => ({ operation, available: true, reason: null })); + for (const kind of ["artifact", "archive"] as const) { + const navigation = entry( + "get_navigation_context", + status({ open: true, kind, capabilities }), + ); + expect(navigation).toMatchObject({ + available: false, + reason: "target_unsupported", + default_mode_available: false, + modes: [ + { name: "current_selection", available: false }, + { name: "explicit_document", available: false }, + ], + }); + for (const mode of navigation.modes ?? []) + expect(mode.remediation).toBe(navigation.remediation); + } +}); + describe("partial navigation availability", () => { it("requires a target before diagnosing navigation provider modes", () => { expect(entry("get_navigation_context", status())).toMatchObject({ diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index b1316b5d6f..2d7d023645 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -416,12 +416,25 @@ const availabilityFor = (context: AvailabilityContext): Availability => { ) return { reason: "available", remediation: null }; if (context.name === "get_navigation_context") { + const target = targetAvailability(context); + if (target !== null) { + return { + ...target, + defaultModeAvailable: false, + modes: NAVIGATION_CONTEXT_MODES.map((mode) => ({ + name: mode.name, + available: false, + missing_operations: [...mode.requirements], + remediation: target.remediation, + })), + }; + } const modes = availabilityForModes( NAVIGATION_CONTEXT_MODES, "current_selection", context.descriptors, ); - return { ...modes, ...targetAvailability(context) }; + return modes; } const javascriptApplication = javascriptApplicationAvailability(context); if (javascriptApplication !== null) return javascriptApplication; @@ -467,11 +480,10 @@ const javascriptApplicationAvailability = ({ : null; }; -const workflowAvailabilityFor = ({ - name, - kind, - policy, -}: AvailabilityContext): Availability | null => { +const workflowAvailabilityFor = ( + context: AvailabilityContext, +): Availability | null => { + const { name, kind, policy } = context; if (name === "start_gdb_session" || name === "start_rizin_debug_session") return hostRestrictedAvailability( policy, @@ -681,7 +693,10 @@ const workflowAvailabilityFor = ({ remediation: "Start REA with its JavaScript runtime provider available.", }; - if (kind === "session") return { reason: "available", remediation: null }; + if (kind === "session") + return ( + targetAvailability(context) ?? { reason: "available", remediation: null } + ); return null; }; @@ -822,12 +837,17 @@ const targetAvailability = ({ targetKind !== undefined && targetKind !== "executable" && targetKind !== "database" && - (kind === "official-proxy" || kind === "enhanced") + (kind === "official-proxy" || + kind === "enhanced" || + name === "get_navigation_context" || + name === "inspect_address_context") ) return { reason: "target_unsupported", remediation: - "Inventory or extract a native executable, then call open_binary on that executable.", + name === "inspect_address_context" + ? "Inventory or extract a native executable, then call open_binary on that executable. An explicit document can still be queried, but its facets may be unavailable." + : "Inventory or extract a native executable, then call open_binary on that executable.", }; if ( targetOpen && diff --git a/src/contracts/sessionToolContracts.ts b/src/contracts/sessionToolContracts.ts index 44d5655bf6..d74c15b768 100644 --- a/src/contracts/sessionToolContracts.ts +++ b/src/contracts/sessionToolContracts.ts @@ -184,9 +184,12 @@ export const SESSION_TOOL_CONTRACTS = [ ), requiresActiveTarget: true, }, - session( - "inspect_address_context", - "Inspect one explicit reproducible address for its analyzed name, containing procedure, regular and inline comments, and matching bookmarks. Each unsupported facet returns a typed unavailable outcome. A facet rejected for invalid input retains its canonical error and issues alongside any successful facets. Use xrefs, assembly, or pseudocode for those deeper views.", - addressContextInputSchema, - ), + { + ...session( + "inspect_address_context", + "Inspect one explicit reproducible address for its analyzed name, containing procedure, regular and inline comments, and matching bookmarks. Each unsupported facet returns a typed unavailable outcome. A facet rejected for invalid input retains its canonical error and issues alongside any successful facets. Use xrefs, assembly, or pseudocode for those deeper views.", + addressContextInputSchema, + ), + requiresActiveTarget: true, + }, ] as const satisfies readonly ToolContract[]; diff --git a/tests/boundary/mcp/sessionTargetRequirements.test.ts b/tests/boundary/mcp/sessionTargetRequirements.test.ts new file mode 100644 index 0000000000..5dc9879b28 --- /dev/null +++ b/tests/boundary/mcp/sessionTargetRequirements.test.ts @@ -0,0 +1,150 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; + +import { buildCapabilityInventory } from "../../../src/application/CapabilityInventory.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { expandToolAvailability } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { zipWith } from "../../fixtures/apkSignature/v1SignatureBlock.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it("advertises closed-session address and explicit-path APK tools according to their actual prerequisites", async () => { + const root = await createTestTempDirectory( + "rea-session-target-requirements-", + ); + const path = join(root, "unsigned.zip"); + await writeFile(path, await zipWith([{ name: "sample.txt", data: "hello" }])); + const { call } = await connectLocalToolsMcp(); + const status = async () => { + const reply = await call("binary_session", {}); + expect(reply.isError).not.toBe(true); + const result = toolContract("binary_session").outputSchema.parse( + reply.structuredContent, + ).result; + expect(result).toMatchObject({ open: false, cleanup_pending: false }); + return expandToolAvailability(result.tool_availability); + }; + const availability = ( + tools: Awaited>, + name: string, + ) => { + const entry = tools.find((tool) => tool.name === name); + expect(entry, `missing ${name} in binary_session`).toBeDefined(); + return entry; + }; + + const before = await status(); + expect(availability(before, "get_navigation_context")).toMatchObject({ + available: false, + reason: "target_required", + }); + expect.soft(availability(before, "inspect_address_context")).toMatchObject({ + available: false, + reason: "target_required", + }); + const owningAvailability = buildCapabilityInventory( + { open: false, capabilities: [] }, + { processCaptureEnabled: false }, + ); + expect(availability(before, "inspect_address_context")?.remediation).toBe( + owningAvailability.find((tool) => tool.name === "inspect_address_context") + ?.remediation, + ); + expect.soft(availability(before, "inspect_apk_signature")).toMatchObject({ + available: true, + reason: "available", + remediation: null, + }); + + const address = await call("inspect_address_context", { address: "0x1000" }); + expect(address.isError).toBe(true); + expect(parseMcpToolError(address)).toMatchObject({ + error: { code: "target_unavailable" }, + }); + const apk = await call("inspect_apk_signature", { path }); + expect(apk.isError, JSON.stringify(apk)).not.toBe(true); + expect(apk.structuredContent).toMatchObject({ + operation: "inspect_analysis_view", + }); + + const after = await status(); + expect(availability(after, "inspect_address_context")).toMatchObject({ + available: false, + reason: "target_required", + }); + expect(availability(after, "inspect_apk_signature")).toMatchObject({ + available: true, + reason: "available", + }); +}); + +it("reports default address resolution unavailable for an artifact while retaining explicit-document results", async () => { + const root = await createTestTempDirectory("rea-address-artifact-status-"); + const path = join(root, "sample.plist"); + await writeFile( + path, + 'CFBundleNameExample', + ); + const { call } = await connectLocalToolsMcp(); + const opened = await call("open_binary", { path }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + for (const detail of ["summary", "complete"] as const) { + const reply = await call("binary_session", { detail }); + expect(reply.isError).not.toBe(true); + const result = toolContract("binary_session").outputSchema.parse( + reply.structuredContent, + ).result; + const entry = expandToolAvailability(result.tool_availability).find( + (tool) => tool.name === "inspect_address_context", + ); + expect(entry).toMatchObject({ + available: false, + reason: "target_unsupported", + remediation: expect.stringContaining("explicit document"), + }); + const navigation = expandToolAvailability(result.tool_availability).find( + (tool) => tool.name === "get_navigation_context", + ); + expect(navigation).toMatchObject({ + available: false, + reason: "target_unsupported", + default_mode_available: false, + modes: [ + { name: "current_selection", available: false }, + { name: "explicit_document", available: false }, + ], + }); + } + for (const input of [{}, { document: "sample.plist" }]) { + const navigation = await call("get_navigation_context", input); + expect(parseMcpToolError(navigation)).toMatchObject({ + error: { + code: "capability_unavailable", + details: { selection_reason: "target_unsupported" }, + }, + }); + } + const implicit = await call("inspect_address_context", { address: "0x1000" }); + expect(parseMcpToolError(implicit)).toMatchObject({ + error: { + code: "capability_unavailable", + details: { selection_reason: "target_unsupported" }, + }, + }); + const explicit = await call("inspect_address_context", { + address: "0x1000", + document: "sample.plist", + }); + expect(explicit.isError, JSON.stringify(explicit)).not.toBe(true); + expect(explicit.structuredContent).toMatchObject({ + result: { + name: { state: "unavailable" }, + procedure: { state: "unavailable" }, + comment: { state: "unavailable" }, + inline_comment: { state: "unavailable" }, + bookmarks: { state: "unavailable" }, + }, + }); +}); From ebea0f009ffc6353f0ae61254b1d2b5dcc4f08d2 Mon Sep 17 00:00:00 2001 From: hhang <17309584214@163.com> Date: Wed, 4 Nov 2026 19:50:39 +0800 Subject: [PATCH 54/94] fix(snapshot): retain one-shot Ghidra pseudocode results (#8089) * fix(snapshot): retain one-shot Ghidra pseudocode results * test(snapshot): cover evidence-only pseudocode migration --------- Co-authored-by: hhang <169027046+xu91102@users.noreply.github.com> --- docs/cli.md | 5 + src/application/AnalysisProvider.ts | 2 + src/application/DirectAnalysis.ts | 9 +- .../binary/AnalysisSnapshotCache.test.ts | 31 ++++ .../binary/AnalysisSnapshotCache.ts | 21 ++- src/application/binary/BinarySession.ts | 8 +- .../binary/BinarySessionExecution.ts | 9 +- .../directAnalysisSnapshotReplay.test.ts | 164 ++++++++++++++++++ 8 files changed, 243 insertions(+), 6 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index bcb344daaf..28af2b71e4 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -637,6 +637,11 @@ rea analyze /absolute/path/to/program --provider ghidra --snapshot /absolute/pat ``` An exact CLI cache hit is normally read before starting a provider process. +`decompile --provider ghidra --snapshot ...` also retains its single selected +Program's pseudocode result, even without an explicit document selector. Old +snapshots that contain only the Evidence need one new query to add that replay +entry. This does not retain every body from `decompile-all` or make a persistent +session's implicit document replayable. On Windows, Ghidra first admits the selected source through its native snapshot owner, so replay can start that provider. In MCP, `open_binary` accepts `snapshot_path` to import a snapshot atomically diff --git a/src/application/AnalysisProvider.ts b/src/application/AnalysisProvider.ts index 6940e3c421..63a498d810 100644 --- a/src/application/AnalysisProvider.ts +++ b/src/application/AnalysisProvider.ts @@ -30,6 +30,8 @@ import { export interface ExecutionOptions { /** Ephemeral state shared only by one composed workflow. */ readonly workScope?: AnalysisWorkScope; + /** A single CLI answer bound to its isolated target, not a persistent default document. */ + readonly snapshotContext?: "one-shot-target"; readonly signal?: AbortSignal; readonly progress?: ProgressReporter; /** diff --git a/src/application/DirectAnalysis.ts b/src/application/DirectAnalysis.ts index 70935557c5..bb4f104ba2 100644 --- a/src/application/DirectAnalysis.ts +++ b/src/application/DirectAnalysis.ts @@ -484,6 +484,7 @@ const runAnalysis = async ( evidenceParameters, signal, evidenceProfile, + saveSnapshot: snapshotPath !== undefined, }); if (!executed.ok) return executed; const { cachePolicy } = executed.value; @@ -594,6 +595,7 @@ const executeAnalysisTool = async (input: { readonly evidenceParameters: Readonly>; readonly signal: AbortSignal; readonly evidenceProfile: AnalysisProfileCommitment | undefined; + readonly saveSnapshot: boolean; }): Promise< Result< { @@ -658,7 +660,12 @@ const executeAnalysisTool = async (input: { : { cachePolicy: result.value.cachePolicy }), }); } - let result = await session.execute(tool, input.arguments, { signal }); + let result = await session.execute(tool, input.arguments, { + signal, + ...(input.saveSnapshot && tool === "procedure_pseudo_code" + ? { snapshotContext: "one-shot-target" as const } + : {}), + }); if ( tool === "xrefs" && typeof input.arguments.address === "string" && diff --git a/src/application/binary/AnalysisSnapshotCache.test.ts b/src/application/binary/AnalysisSnapshotCache.test.ts index d3396d4a6a..bf048e6f3e 100644 --- a/src/application/binary/AnalysisSnapshotCache.test.ts +++ b/src/application/binary/AnalysisSnapshotCache.test.ts @@ -777,7 +777,9 @@ describe("analysis snapshot cache policy", () => { ); }, ); +}); +describe("document-scoped snapshot admission", () => { it("retains caller observations without a document while refusing to replay them", () => { const descriptor: CapabilityDescriptor = { operation: "procedure_callers", @@ -829,6 +831,35 @@ describe("analysis snapshot cache policy", () => { document: "fixture", }), ).toBe(true); + const ghidra = { + ...pseudocode, + provider: { ...pseudocode.provider, id: "ghidra" }, + }; + expect( + isSnapshotCacheable("procedure_pseudo_code", ghidra, parameters), + ).toBe(false); + expect( + isSnapshotCacheable( + "procedure_pseudo_code", + ghidra, + parameters, + "one-shot-target", + ), + ).toBe(true); + for (const refused of [ + pseudocode, + { ...ghidra, cachePolicy: "live" as const }, + { ...ghidra, effects: { ...ghidra.effects, mutatesArtifact: true } }, + { ...ghidra, effects: { ...ghidra.effects, changesPermissions: true } }, + ]) + expect( + isSnapshotCacheable( + "procedure_pseudo_code", + refused, + parameters, + "one-shot-target", + ), + ).toBe(false); }); }); diff --git a/src/application/binary/AnalysisSnapshotCache.ts b/src/application/binary/AnalysisSnapshotCache.ts index 1289c2cb43..3232b79c60 100644 --- a/src/application/binary/AnalysisSnapshotCache.ts +++ b/src/application/binary/AnalysisSnapshotCache.ts @@ -36,6 +36,7 @@ import type { AnalysisExecution, AnalysisOperation, CapabilityDescriptor, + ExecutionOptions, } from "../AnalysisProvider.js"; import { OFFICIAL_TOOL_CONTRACTS } from "../../contracts/officialToolContracts.js"; import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; @@ -94,6 +95,16 @@ const replayDocumentSelected = ( !DOCUMENT_SCOPED_OPERATIONS.has(operation) || typeof parameters.document === "string"; +/** Ghidra's private Program belongs to this one-shot target; GUI defaults do not. */ +const isOneShotPseudocode = ( + operation: AnalysisOperation, + descriptor: CapabilityDescriptor | undefined, + context: ExecutionOptions["snapshotContext"], +): boolean => + context === "one-shot-target" && + operation === "procedure_pseudo_code" && + descriptor?.provider.id === "ghidra"; + /** * Caller and callee walks omit `document`. Their observations are still * Evidence. Other document-scoped reads, including `procedure_pseudo_code` @@ -112,12 +123,14 @@ export const isSourceEvidenceRetainable = ( operation: AnalysisOperation, descriptor: CapabilityDescriptor | undefined, parameters: Readonly>, + context?: ExecutionOptions["snapshotContext"], ): descriptor is CapabilityDescriptor => !STATEFUL_OPERATIONS.has(operation) && !RESOURCE_SCOPED_OPERATIONS.has(operation) && (!DOCUMENT_SCOPED_OPERATIONS.has(operation) || typeof parameters.document === "string" || - DOCUMENTLESS_SOURCE_EVIDENCE_OPERATIONS.has(operation)) && + DOCUMENTLESS_SOURCE_EVIDENCE_OPERATIONS.has(operation) || + isOneShotPseudocode(operation, descriptor, context)) && (!CURSOR_DEFAULT_OPERATIONS.has(operation) || typeof parameters.address === "string") && descriptor?.effects.mutatesArtifact === false && @@ -130,9 +143,11 @@ export const isSnapshotCacheable = ( operation: AnalysisOperation, descriptor: CapabilityDescriptor | undefined, parameters: Readonly>, + context?: ExecutionOptions["snapshotContext"], ): descriptor is CapabilityDescriptor => - isSourceEvidenceRetainable(operation, descriptor, parameters) && - replayDocumentSelected(operation, parameters) && + isSourceEvidenceRetainable(operation, descriptor, parameters, context) && + (replayDocumentSelected(operation, parameters) || + isOneShotPseudocode(operation, descriptor, context)) && descriptor.cachePolicy !== "live"; /** Windows Ghidra must admit the selected source through its native snapshot owner. */ diff --git a/src/application/binary/BinarySession.ts b/src/application/binary/BinarySession.ts index 84369c5253..87129021d3 100644 --- a/src/application/binary/BinarySession.ts +++ b/src/application/binary/BinarySession.ts @@ -1418,6 +1418,7 @@ export class BinarySession active: this.#active, operation: name, parameters: arguments_, + snapshotContext: options?.snapshotContext, nativeSourceAdmitted: this.#active?.nativeSourceAdmitted === true, unboundOperationError: (operation, route) => this.#providerRouter.unboundOperationError(operation, route), @@ -1479,7 +1480,12 @@ export class BinarySession } if ( execution.ok && - isSourceEvidenceRetainable(name, capability, arguments_) && + isSourceEvidenceRetainable( + name, + capability, + arguments_, + options?.snapshotContext, + ) && profile !== undefined ) { const evidence = createAnalysisExecutionEvidence( diff --git a/src/application/binary/BinarySessionExecution.ts b/src/application/binary/BinarySessionExecution.ts index 20d4dd063d..fa0703f93b 100644 --- a/src/application/binary/BinarySessionExecution.ts +++ b/src/application/binary/BinarySessionExecution.ts @@ -15,6 +15,7 @@ import type { AnalysisExecution, AnalysisOperation, CapabilityDescriptor, + ExecutionOptions, } from "../AnalysisProvider.js"; import { isSnapshotCacheable, @@ -41,6 +42,7 @@ interface PrepareSessionExecutionInput { readonly active: ActiveExecutionBinding | undefined; readonly operation: AnalysisOperation; readonly parameters: Readonly>; + readonly snapshotContext?: ExecutionOptions["snapshotContext"]; readonly nativeSourceAdmitted: boolean; readonly unboundOperationError: ( operation: AnalysisOperation, @@ -94,7 +96,12 @@ export const prepareSessionExecution = ( : undefined; const cacheable = profile !== undefined && - isSnapshotCacheable(operation, capability, parameters); + isSnapshotCacheable( + operation, + capability, + parameters, + input.snapshotContext, + ); const cached = cacheable && (!requiresNativeSourceAdmissionBeforeReplay(capability.provider.id) || diff --git a/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts b/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts index d26a2e7381..4180ca2d03 100644 --- a/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts +++ b/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts @@ -289,6 +289,170 @@ const createSnapshotReplayFixture = async ( }; }; +describe("one-shot Ghidra pseudocode snapshots", () => { + it("keeps persistent documentless reads out of the provider cache", async () => { + const fixture = await createSnapshotReplayFixture("rea-session-code-"); + const session = createTestBinarySession( + fixture.createProvider({ + profile: createAnalysisProfile( + { ...GHIDRA_PROVIDER_IDENTITY, version: "12.1.4" }, + { fixture: true }, + ), + supportedOperations: ["procedure_pseudo_code"], + }), + ); + try { + expect((await session.open(fixture.path)).ok).toBe(true); + for (let index = 0; index < 2; index++) + expect( + ( + await session.execute("procedure_pseudo_code", { + procedure: "fixture", + }) + ).ok, + ).toBe(true); + const snapshot = session.exportAnalysisSnapshot(); + if (!snapshot.ok) throw snapshot.error; + expect(snapshot.value.entries).toEqual([]); + expect(snapshot.value.evidence_bundle.records).toEqual([]); + expect( + fixture.calls.filter((name) => name === "procedure_pseudo_code"), + ).toHaveLength(2); + } finally { + expect((await session.close()).ok).toBe(true); + } + }); + + it.skipIf(process.platform === "win32")( + "replays the exact selected-target answer without opening another provider", + async () => { + const fixture = await createSnapshotReplayFixture("rea-one-shot-code-"); + const profile = createAnalysisProfile( + { ...GHIDRA_PROVIDER_IDENTITY, version: "12.1.4" }, + { fixture: true }, + ); + const dependencies = fixture.dependencies( + fixture.createProvider({ + profile, + supportedOperations: ["procedure_pseudo_code"], + }), + ); + const options = { snapshotPath: fixture.snapshotPath }; + const parameters = { procedure: "fixture" }; + const first = await runDirectAnalysis( + dependencies, + fixture.path, + "procedure_pseudo_code", + parameters, + options, + ); + expect(first.ok).toBe(true); + const second = await runDirectAnalysis( + dependencies, + fixture.path, + "procedure_pseudo_code", + parameters, + options, + ); + expect(second).toEqual(first); + expect(fixture.starts).toHaveLength(1); + const saved = await readAnalysisSnapshot(fixture.snapshotPath); + if (!saved.ok) throw saved.error; + expect(saved.value.entries).toMatchObject([ + { operation: "procedure_pseudo_code", parameters }, + ]); + }, + ); + + it("does not make another provider's implicit document replayable", async () => { + const fixture = await createSnapshotReplayFixture("rea-implicit-document-"); + const dependencies = fixture.dependencies( + fixture.createProvider({ + supportedOperations: ["procedure_pseudo_code"], + }), + ); + for (let index = 0; index < 2; index++) { + const result = await runDirectAnalysis( + dependencies, + fixture.path, + "procedure_pseudo_code", + { procedure: "fixture" }, + { snapshotPath: fixture.snapshotPath }, + ); + expect(result.ok).toBe(true); + } + expect(fixture.starts).toHaveLength(2); + const saved = await readAnalysisSnapshot(fixture.snapshotPath); + if (!saved.ok) throw saved.error; + expect(saved.value.entries).toEqual([]); + }); +}); + +describe("legacy one-shot Ghidra pseudocode snapshots", () => { + it.skipIf(process.platform === "win32")( + "restores a missing replay entry without replacing the original Evidence bundle", + async () => { + const fixture = await createSnapshotReplayFixture("rea-legacy-code-"); + const dependencies = fixture.dependencies( + fixture.createProvider({ + profile: createAnalysisProfile( + { ...GHIDRA_PROVIDER_IDENTITY, version: "12.1.4" }, + { fixture: true }, + ), + supportedOperations: ["procedure_pseudo_code"], + }), + ); + const parameters = { procedure: "fixture" }; + const query = () => + runDirectAnalysis( + dependencies, + fixture.path, + "procedure_pseudo_code", + parameters, + { snapshotPath: fixture.snapshotPath }, + ); + const first = await query(); + if (!first.ok) throw new Error(JSON.stringify(first.error)); + const saved = await readAnalysisSnapshot(fixture.snapshotPath); + if (!saved.ok) throw saved.error; + const written = await withArtifactResourceScope((resources) => + writeAnalysisSnapshot( + { ...saved.value, entries: [] }, + fixture.snapshotPath, + true, + resources, + ), + ); + if (!written.ok) throw written.error; + const legacy = await readAnalysisSnapshot(fixture.snapshotPath); + if (!legacy.ok) throw legacy.error; + expect(legacy.value.entries).toEqual([]); + expect(legacy.value.evidence_bundle).toEqual(saved.value.evidence_bundle); + + const upgraded = await query(); + expect(upgraded).toEqual(first); + expect(fixture.starts).toHaveLength(2); + const restored = await readAnalysisSnapshot(fixture.snapshotPath); + if (!restored.ok) throw restored.error; + expect(restored.value.entries).toMatchObject([ + { operation: "procedure_pseudo_code", parameters }, + ]); + expect(restored.value.evidence_bundle).toEqual( + saved.value.evidence_bundle, + ); + + const replay = await query(); + expect(replay).toEqual(first); + if (!replay.ok) throw new Error(JSON.stringify(replay.error)); + expect(replay.value.evidence_id).toBe(first.value.evidence_id); + expect(fixture.starts).toHaveLength(2); + expect( + fixture.calls.filter((name) => name === "procedure_pseudo_code"), + ).toHaveLength(2); + }, + ); +}); + const withAlternateWorkflowProfile = ( snapshot: AnalysisSnapshot, path: string, From 103684bce1971f25414f7e8dd879069720f6c315 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:51:27 +0800 Subject: [PATCH 55/94] fix(ghidra): guide recovery from unresolved procedure selectors (#8470) * fix(ghidra): guide recovery from missing procedures * fix(ghidra): preserve mapped offset lookup recovery * fix(ghidra): distinguish unmapped procedure offsets * fix(ghidra): guide ambiguous selector recovery * fix(ghidra): clarify fresh import excludes project edits --- bridge/ghidra/ReaGhidraBridge.java | 14 +- docs/installation.md | 7 + .../verify-real-ghidra-address-context.mjs | 32 ++++ scripts/verify-real-ghidra-inventory.mjs | 36 ++++- src/ghidra/GhidraProvider.behavior.test.ts | 151 ++++++++++++++++-- src/ghidra/GhidraProviderClient.ts | 48 ++++++ src/ghidra/GhidraSessionError.ts | 8 +- .../ghidra/ReaFileOffsetSourceProbe.java | 6 +- 8 files changed, 279 insertions(+), 23 deletions(-) diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index eb26f63d84..efed6af767 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -1010,7 +1010,7 @@ public final class ReaGhidraBridge extends HeadlessScript { } catch (RequestFailure failure) { // Existing names/addresses win. Only an unresolved numeric input // can fall back to source coordinates; ambiguity never falls back. - if (!failure.code.equals("not_found") || !selector.matches("0x[0-9a-fA-F]+")) + if (!failure.code.equals("procedure_not_found") || !selector.matches("0x[0-9a-fA-F]+")) throw failure; } } @@ -1037,17 +1037,17 @@ public final class ReaGhidraBridge extends HeadlessScript { } List
candidates = new ArrayList<>(mappedAddresses); if (candidates.size() > 1) - throw new RequestFailure("ambiguous", "File offset " + coordinate + + throw new RequestFailure("file_offset_ambiguous", "File offset " + coordinate + " has multiple loaded mappings: " + candidates.stream().map(ReaGhidraBridge::canonicalAddress).toList() + "; select an explicit analysis address"); if (candidates.isEmpty()) - throw new RequestFailure("not_found", "File offset " + coordinate + + throw new RequestFailure("file_offset_unmapped", "File offset " + coordinate + " has no loaded source mapping for the admitted target file; Ghidra image base: " + canonicalAddress(currentProgram.getImageBase()) + ". Inspect mappings with rea inspect-native-load-image --provider ghidra --json or MCP inspect_native_load_image."); Address mapped = candidates.get(0); Function function = functionAtOrContaining(mapped); if (function == null) - throw new RequestFailure("not_found", "File offset " + coordinate + + throw new RequestFailure("procedure_not_found", "File offset " + coordinate + " maps to analysis address " + canonicalAddress(mapped) + ", but no procedure starts at or contains that address in the current Ghidra analysis. " + "Choose an observed procedure with CLI rea search . --kind procedures --mode regex --provider ghidra --json or MCP list_procedures."); @@ -2458,7 +2458,7 @@ public final class ReaGhidraBridge extends HeadlessScript { if (matchProcedureNames(entry, selector).matched()) matches.add(entry.function); } if (matches.size() > 1) - throw new RequestFailure("ambiguous", "Ghidra procedure name is ambiguous: " + value + + throw new RequestFailure("procedure_ambiguous", "Ghidra procedure name is ambiguous: " + value + "; select an exact entry address from " + matches.stream() .map(function -> canonicalAddress(function.getEntryPoint())).toList()); if (matches.size() == 1) return matches.get(0); @@ -2466,11 +2466,11 @@ public final class ReaGhidraBridge extends HeadlessScript { // Retain legacy bare-hex address input only when no symbol matches. Address address = explicitAddress ? parseReaAddress(value) : tryParseAddress(value); if (address == null) - throw new RequestFailure("not_found", "Unknown Ghidra procedure name or address: " + value); + throw new RequestFailure("procedure_not_found", "Unknown Ghidra procedure name or address: " + value); Function function = functionAtOrContaining(address); if (function == null) // Other procedure operations retain their analysis-address semantics. - throw new RequestFailure("not_found", "No procedure exists at the requested address: " + value + + throw new RequestFailure("procedure_not_found", "No procedure exists at the requested address: " + value + "; Ghidra image base: " + canonicalAddress(currentProgram.getImageBase()) + ". Choose an observed analysis address. CLI: rea search . --kind procedures --mode regex --provider ghidra --json; " + "MCP: list_procedures. Inspect the loaded base and source mappings with CLI rea analyze --provider ghidra --json " + diff --git a/docs/installation.md b/docs/installation.md index 46b282f8bf..c733c40a6e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1459,6 +1459,13 @@ they are not interpreted as overrides applied by this session. Project content participates in snapshot identity, so different project states cannot reuse the same profile's cached observations. +If a function selector is missing, use `list_procedures` or `search_procedures` +to inspect the selected project's observed inventory. A missing entry does not +prove the function is absent from the executable. To explicitly compare with a +fresh import, start another REA process without `REA_GHIDRA_PROJECT_PATH` and +`REA_GHIDRA_PROGRAM_NAME`; the fresh analysis will not include the selected +project's annotations. + To prepare a new project explicitly, use your existing Ghidra installation in a new directory you own (this writes that project; REA does not save it for you): diff --git a/scripts/verify-real-ghidra-address-context.mjs b/scripts/verify-real-ghidra-address-context.mjs index 3d9cac7d4b..55c6c1f301 100644 --- a/scripts/verify-real-ghidra-address-context.mjs +++ b/scripts/verify-real-ghidra-address-context.mjs @@ -161,6 +161,38 @@ try { fromNumericOffset.normalized_result, fromOffset.normalized_result, ); + const missingProcedure = "rea_missing_procedure_fixture"; + const missingName = await client.callTool( + { + name: "procedure_pseudo_code", + arguments: { procedure: missingProcedure }, + }, + { timeout: 240000 }, + ); + assert.equal(missingName.isError, true); + const missingNameError = JSON.parse( + missingName.content.find((item) => item.type === "text").text, + ).error; + assert.equal(missingNameError.code, "invalid_request"); + assert.match(missingNameError.message, /rea_missing_procedure_fixture/u); + assert.match(missingNameError.remediation.action, /list_procedures/u); + assert.match(missingNameError.remediation.action, /does not prove/u); + assert.equal(missingNameError.recovery, undefined); + const unmappedOffset = await client.callTool( + { + name: "procedure_pseudo_code", + arguments: { procedure: "file-offset:0x1fffffffffffff" }, + }, + { timeout: 240000 }, + ); + assert.equal(unmappedOffset.isError, true); + const unmappedError = JSON.parse( + unmappedOffset.content.find((item) => item.type === "text").text, + ).error; + assert.equal(unmappedError.code, "invalid_request"); + assert.match(unmappedError.message, /no loaded source mapping/u); + assert.match(unmappedError.remediation.action, /inspect_native_load_image/u); + assert.equal(unmappedError.recovery, undefined); const missingAddress = "ram:0xffffffff"; const rejected = await client.callTool( { name: "procedure_pseudo_code", arguments: { procedure: missingAddress } }, diff --git a/scripts/verify-real-ghidra-inventory.mjs b/scripts/verify-real-ghidra-inventory.mjs index d89f726562..61ae20d1af 100644 --- a/scripts/verify-real-ghidra-inventory.mjs +++ b/scripts/verify-real-ghidra-inventory.mjs @@ -46,8 +46,40 @@ async function expectRemoteFailure(client, operation, parameters, code) { return { operation, code }; } -async function verifyBridgeRejections(client) { +async function verifyBridgeRejections(client, names) { + const global = names.find((item) => + matchesSymbol(item.value, "rea_ghidra_inventory_global"), + ); + if (global === undefined) + throw new Error("Ghidra initialized global probe is unavailable"); + const mapping = await call(client, "address_to_file_offset", { + document: null, + address: global.address, + }); + if (!Number.isSafeInteger(mapping.file_offset)) + throw new Error("Ghidra initialized global has no exact file offset"); return { + unknownProcedure: await expectRemoteFailure( + client, + "procedure_address", + { document: null, procedure: "rea_missing_procedure_fixture" }, + "procedure_not_found", + ), + mappedDataOffset: await expectRemoteFailure( + client, + "procedure_pseudo_code", + { + document: null, + procedure: `file-offset:0x${mapping.file_offset.toString(16)}`, + }, + "procedure_not_found", + ), + unmappedFileOffset: await expectRemoteFailure( + client, + "procedure_pseudo_code", + { document: null, procedure: "file-offset:0x1fffffffffffff" }, + "file_offset_unmapped", + ), unknownDocument: await expectRemoteFailure( client, "list_segments", @@ -86,7 +118,7 @@ export async function verifyInventoryOperations({ }); const rejections = - variant === "debug" ? await verifyBridgeRejections(client) : null; + variant === "debug" ? await verifyBridgeRejections(client, names) : null; const stringSearch = await call(client, "search_strings", { pattern: String.raw`^REA_GHIDRA_(?:INVENTORY_ENTRY|LEAF_VALUE)$`, mode: "regex", diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index 5663f10732..b5c94aaf9a 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -1375,24 +1375,153 @@ describe("Ghidra result projection", () => { }); }); -describe("Ghidra file-offset outcome projection", () => { - const remoteFailure = (remoteCode: string, message: string) => - provider(installationHost(), () => ({ - start: () => Promise.resolve(ok(sessionInfo())), - callTool: () => - Promise.resolve( +const remoteFailure = (remoteCode: string, message: string) => + provider(installationHost(), () => ({ + start: () => Promise.resolve(ok(sessionInfo())), + callTool: () => + Promise.resolve( + err( + new GhidraSessionError( + "remote", + message, + { remote_code: remoteCode }, + { remoteCode }, + ), + ), + ), + close: () => Promise.resolve(ok(null)), + })); + +describe("Ghidra procedure lookup projection", () => { + it.each([ + { + code: "procedure_not_found", + selector: "Unresolved_Selected_Name", + message: + "Unknown Ghidra procedure name or address: Unresolved_Selected_Name", + remedy: "list_procedures", + qualification: "does not prove", + }, + { + code: "file_offset_unmapped", + selector: "file-offset:0xdeadbeef", + message: "File offset 0xdeadbeef has no loaded source mapping", + remedy: "inspect_native_load_image", + qualification: "does not prove", + }, + { + code: "procedure_ambiguous", + selector: "overloaded", + message: + "Ghidra procedure name is ambiguous: overloaded; select an exact entry address from [0x1000, 0x2000]", + remedy: "entry address", + qualification: "reported candidates", + }, + { + code: "file_offset_ambiguous", + selector: "file-offset:0x100", + message: + "File offset 0x100 has multiple loaded mappings: [0x1000, 0x2000]", + remedy: "analysis address", + qualification: "does not select one automatically", + }, + ])( + "reports $code without schema repair", + async ({ code, selector, message, remedy, qualification }) => { + const client = await createElfClient(remoteFailure(code, message)); + try { + const result = await client.execute("procedure_pseudo_code", { + procedure: selector, + }); + if (result.ok) throw new Error("Expected a missing procedure"); + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + code: "invalid_request", + message, + details: { + issues: [ + { + path: ["procedure"], + observed: { procedure: selector }, + }, + ], + diagnostics: { remote_code: code }, + }, + }); + expect(projected).not.toHaveProperty("recovery"); + expect(projected.remediation.action).toContain(remedy); + expect(projected.remediation.action).toContain(qualification); + expect(projected.remediation.action).not.toContain( + "REA_GHIDRA_PROJECT_PATH", + ); + } finally { + await client.close(); + } + }, + ); + + it("offers an explicit fresh import when the missing procedure came from a selected project", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-missing-procedure-")); + await mkdir(join(root, "selected.rep")); + await writeFile(join(root, "selected.gpr"), "marker"); + const message = "Unknown Ghidra procedure name or address: missing"; + const ghidra = provider( + installationHost(), + () => ({ + start: async () => + ok({ + ...sessionInfo(), + read_only: true, + existing_project: true as const, + analysis_complete: null, + capabilities: GHIDRA_SESSION_CAPABILITIES.filter( + (name) => !GHIDRA_MUTATING_OPERATIONS.has(name), + ), + }), + callTool: async () => err( new GhidraSessionError( "remote", message, - { remote_code: remoteCode }, - { remoteCode }, + {}, + { + remoteCode: "procedure_not_found", + }, ), ), - ), - close: () => Promise.resolve(ok(null)), - })); + close: async () => ok(null), + }), + { REA_GHIDRA_PROJECT_PATH: join(root, "selected.gpr") }, + ); + try { + const client = await createElfClient(ghidra); + try { + const result = await client.execute("procedure_address", { + procedure: "missing", + }); + if (result.ok) throw new Error("Expected a missing procedure"); + const projected = projectAnalysisError(result.error); + expect(projected).not.toHaveProperty("recovery"); + expect(projected.remediation.action).toContain( + "REA_GHIDRA_PROJECT_PATH", + ); + expect(projected.remediation.action).toContain( + "completeness is unknown", + ); + expect(projected.remediation.action).toContain( + "will not include the selected project's annotations or other edits", + ); + } finally { + await client.close(); + } + } finally { + await ghidra.closeOwnedResources(); + await rm(root, { recursive: true, force: true }); + } + }); +}); +describe("Ghidra file-offset outcome projection", () => { it.each([ [ "address_unmapped", diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index a01afa9328..0468cd4a36 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -1261,6 +1261,54 @@ const projectPrimarySessionError = ( return new AnalysisInputError(operation, context, [ { path: ["name"], reason: "invalid_value", message: failure.message }, ]); + if ( + failure.kind === "remote" && + (failure.remoteCode === "procedure_not_found" || + failure.remoteCode === "file_offset_unmapped" || + failure.remoteCode === "procedure_ambiguous" || + failure.remoteCode === "file_offset_ambiguous") + ) { + let recovery: string; + switch (failure.remoteCode) { + case "file_offset_unmapped": + recovery = + "Use inspect_native_load_image (CLI: `rea inspect-native-load-image --provider ghidra --json`) to inspect the admitted file's loaded mappings, then select an offset with an observed mapping. A missing mapping does not prove the file contains no relevant data."; + break; + case "file_offset_ambiguous": + recovery = + "Choose an explicit analysis address from the reported mappings. Use inspect_native_load_image to inspect the loaded mappings; REA does not select one automatically."; + break; + case "procedure_ambiguous": + recovery = + "Choose an explicit procedure entry address from the reported candidates. Use list_procedures or search_procedures to inspect the current Ghidra inventory."; + break; + default: + recovery = + "Use list_procedures or search_procedures (CLI: `rea search --kind procedures --provider ghidra --json`) to inspect the current Ghidra inventory and choose an observed name or address. An absent entry does not prove the function is absent from the artifact." + + (limits.ghidraExistingProject !== undefined + ? " The selected project's previous analysis completeness is unknown. To explicitly request a separate fresh import, start a new REA process without REA_GHIDRA_PROJECT_PATH and REA_GHIDRA_PROGRAM_NAME; that analysis will not include the selected project's annotations or other edits." + : ""); + } + return new AnalysisInputError( + operation, + { + ...context, + userMessage: failure.message, + conformsToInputContract: true, + remediationAction: recovery, + }, + [ + { + path: ["procedure"], + reason: "invalid_value", + message: failure.message, + ...(typeof request?.procedure === "string" + ? { observed: { procedure: request.procedure } } + : {}), + }, + ], + ); + } if ( failure.kind === "cancelled" || (failure.kind === "remote" && failure.remoteCode === "decompile_cancelled") diff --git a/src/ghidra/GhidraSessionError.ts b/src/ghidra/GhidraSessionError.ts index 1fe87cb1cf..8201e12415 100644 --- a/src/ghidra/GhidraSessionError.ts +++ b/src/ghidra/GhidraSessionError.ts @@ -189,7 +189,13 @@ const createGhidraSessionError = (failure: { /** Bridge input failures share classification with their domain error projection. */ export const isGhidraInputFailureCode = (code: string | undefined): boolean => - code === "invalid_request" || code === "not_found" || code === "ambiguous"; + code === "invalid_request" || + code === "not_found" || + code === "procedure_not_found" || + code === "file_offset_unmapped" || + code === "procedure_ambiguous" || + code === "file_offset_ambiguous" || + code === "ambiguous"; /** Bind session-failure construction to its causal diagnostic scope. */ export const bindGhidraSessionFailure = diff --git a/tests/conformance/ghidra/ReaFileOffsetSourceProbe.java b/tests/conformance/ghidra/ReaFileOffsetSourceProbe.java index 9c3bf4e1cf..e184f989e6 100644 --- a/tests/conformance/ghidra/ReaFileOffsetSourceProbe.java +++ b/tests/conformance/ghidra/ReaFileOffsetSourceProbe.java @@ -49,7 +49,8 @@ public final class ReaFileOffsetSourceProbe extends HeadlessScript { var entry = address.add(0x910); currentProgram.getFunctionManager().createFunction("foreign_only", entry, new AddressSet(entry), SourceType.USER_DEFINED); - rejected(resolve, bridge, "no loaded source mapping", "file-offset:0x910", new JsonObject()); + rejectedCode(resolve, bridge, "file_offset_unmapped", "no loaded source mapping", + "file-offset:0x910", new JsonObject()); rejectedCode(reverse, bridge, "file_mapping_absent", "has no authoritative mapping to the admitted target file", reverseInput("0x140100910")); @@ -100,7 +101,8 @@ public final class ReaFileOffsetSourceProbe extends HeadlessScript { var duplicate = memory.createFileBytes("duplicate.bin", 0, bytes.length, new ByteArrayInputStream(bytes), monitor); memory.createInitializedBlock("duplicate", toAddr(0x140201000L), duplicate, 0x200, 0x200, false); - rejected(resolve, bridge, "multiple loaded mappings", "file-offset:0x210", new JsonObject()); + rejectedCode(resolve, bridge, "file_offset_ambiguous", "multiple loaded mappings", + "file-offset:0x210", new JsonObject()); println("REA_FILE_OFFSET_SOURCE_OK foreign/partial/uninitialized/unmapped rejected; original round trip; duplicate ambiguity; injected backend failure context retained"); } From 398eb0d1f16b410b64a96615c6463fbf2ab2f38d Mon Sep 17 00:00:00 2001 From: Christopher Pruijsen Date: Wed, 4 Nov 2026 11:52:16 +0000 Subject: [PATCH 56/94] feat(setup): register LM Studio during setup (#8611) * feat(setup): register LM Studio during setup Register rea in mcp.json in the LM Studio home through a new lm_studio client. The home follows LM Studio's own lookup, read-only: the path in ~/.lmstudio-home-pointer, else an existing ~/.cache/lm-studio, else ~/.lmstudio. An unreadable or relative pointer reports a path error instead of a guessed home. LM Studio parses mcp.json with strict JSON.parse, so comments, trailing commas and a BOM are rejected before any backup or write. The entry carries command, args and env with no type or timeout. Local models have small context windows, so the registration selects REA_MCP_INPUT_SCHEMA_PROFILE=compact unless a profile is set explicitly, and doctor reports an entry without a profile as stale. One predicate now owns that default for Kimi Code and LM Studio. * fix(doctor): resolve provider budgets for LM Studio as setup does --- docs/client-configuration.md | 16 ++ docs/installation.md | 3 + .../ClientConfigurationDocument.ts | 10 +- src/application/ClientRegistrationStatus.ts | 16 +- src/application/SetupClientConfiguration.ts | 3 +- src/application/SupportedClients.ts | 43 +++- .../filesystem/clientLifecycle.test.ts | 1 + .../clientRegistrationStatus.test.ts | 18 +- .../lmStudioClientConfiguration.test.ts | 201 ++++++++++++++++++ 9 files changed, 295 insertions(+), 16 deletions(-) create mode 100644 tests/boundary/filesystem/lmStudioClientConfiguration.test.ts diff --git a/docs/client-configuration.md b/docs/client-configuration.md index 54b5e27493..5f04937b10 100644 --- a/docs/client-configuration.md +++ b/docs/client-configuration.md @@ -310,6 +310,22 @@ Kimi Code also discovers personal skills in `~/.agents/skills` and project-local and verifies the matching skill files through the normal shared-skill flow. Start a new Kimi Code session to pick up MCP and skill changes. +## LM Studio + +For [LM Studio](https://lmstudio.ai/docs/app/mcp), setup writes `mcpServers.rea` +into `mcp.json` in the LM Studio home directory. Setup finds that directory as +LM Studio does: the path stored in `~/.lmstudio-home-pointer`, otherwise +`~/.cache/lm-studio` when it exists, otherwise `~/.lmstudio`. Setup only reads +the pointer file. LM Studio parses `mcp.json` as strict JSON, so setup rejects +a file with comments, trailing commas, or a byte order mark without a backup +or a write. The entry carries `command`, `args`, and `env`, and setup keeps +every other server. Local models often have small context windows, so setup +selects `REA_MCP_INPUT_SCHEMA_PROFILE=compact` unless the stored entry or the +setup environment selects a profile explicitly. Doctor reports an entry with no +explicit profile as stale, as for Kimi Code. LM Studio 0.4.26 on macOS loaded +this registration, and a local model called an REA tool through it. Linux and +Windows remain unverified. + ## Pi For [Pi](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/docs/mcp.md), diff --git a/docs/installation.md b/docs/installation.md index c733c40a6e..322a0a55aa 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -654,6 +654,7 @@ hosted connector; see [its configuration](client-configuration.md#other-client-s | Factory Droid | `droid` | | Trae | `trae` | | WorkBuddy | `workbuddy` | +| LM Studio | `lm_studio` | | ZCode | `zcode` | | DeepSeek Harness | `dsh` | | FreeBuff | `freebuff` | @@ -663,6 +664,8 @@ hosted connector; see [its configuration](client-configuration.md#other-client-s `freebuff` requires `rea-agents@6.8.0` or later. Older releases reject `rea setup --client freebuff` as an unsupported client. +`lm_studio` requires a release after 6.8.0. + See [client configuration and troubleshooting](client-configuration.md) for configuration paths, profiles, trust policies, and schema-limit workarounds. diff --git a/src/application/ClientConfigurationDocument.ts b/src/application/ClientConfigurationDocument.ts index ce2e56daeb..fccc63be21 100644 --- a/src/application/ClientConfigurationDocument.ts +++ b/src/application/ClientConfigurationDocument.ts @@ -152,9 +152,10 @@ const parseDocument = ( // Pi loads `mcp.json` with a strict `JSON.parse` and no BOM tolerance, so // comments, trailing commas, a BOM, or an empty existing file must be // rejected instead of registering into a document the client cannot load. - // Kimi also reads mcp.json with JSON.parse and an empty-file fallback. - if (format === "pi" || format === "kimi") { - if (format === "kimi" && text.trim().length === 0) return {}; + // Kimi and LM Studio also read mcp.json with JSON.parse; an empty file holds + // nothing to preserve. + if (format === "pi" || format === "kimi" || format === "lm_studio") { + if (format !== "pi" && text.trim().length === 0) return {}; return requireObject(JSON.parse(text), "The configuration root"); } // An empty file, which some clients create before any server is added, @@ -1748,7 +1749,8 @@ export const clientRegistrationEntry = ( : { environment: { ...environment } }), }; case "kimi": - // Kimi infers stdio from command; it uses `transport`, not `type`. + case "lm_studio": + // Both infer stdio from command; Kimi uses `transport`, not `type`. return { command: executable, args, diff --git a/src/application/ClientRegistrationStatus.ts b/src/application/ClientRegistrationStatus.ts index f1396eefa8..6c1ab64228 100644 --- a/src/application/ClientRegistrationStatus.ts +++ b/src/application/ClientRegistrationStatus.ts @@ -28,6 +28,7 @@ import { import { clientEvidencePaths, clientOwnershipEvidencePaths, + compactInputSchemaByDefault, manualRegistrationRemediation, supportedClients, } from "./SupportedClients.js"; @@ -279,12 +280,14 @@ export const readClientRegistrationInspections = async ( } const registration = parsedRegistration.value; const registrationEnvironment = - client.name === "kimi" || client.name === "codex" + compactInputSchemaByDefault(client) || client.name === "codex" ? await clientRegistrationEnvironmentDetailsWithBudget( parsed, {}, "effective", - options.resolveProviderRequestBudget, + client.name === "kimi" || client.name === "codex" + ? options.resolveProviderRequestBudget + : undefined, options.platform, ) : undefined; @@ -321,7 +324,7 @@ export const readClientRegistrationInspections = async ( command, commandFacts, platform: options.platform ?? process.platform, - kimiInputSchemaProfileSelected: + inputSchemaProfileSelected: registrationEnvironment?.ok === true && snapshotEnvironment( registrationEnvironment.value.values, @@ -449,14 +452,14 @@ const registrationAligned = ({ command, commandFacts, platform, - kimiInputSchemaProfileSelected, + inputSchemaProfileSelected, }: { readonly registration: ClientRegistration; readonly client: SetupClient; readonly command: readonly string[]; readonly commandFacts: ClientRegistrationCommandFacts; readonly platform: NodeJS.Platform; - readonly kimiInputSchemaProfileSelected: boolean; + readonly inputSchemaProfileSelected: boolean; }): "aligned" | "stale" | "unknown" => { const { identity } = commandFacts; if (identity === "unknown") return "unknown"; @@ -478,7 +481,8 @@ const registrationAligned = ({ registration.toolTimeoutMs === undefined) ) return "stale"; - if (client.name === "kimi" && !kimiInputSchemaProfileSelected) return "stale"; + if (compactInputSchemaByDefault(client) && !inputSchemaProfileSelected) + return "stale"; // A Harness registration only connects as the rea stdio server; another // server name or transport leaves the session without REA tools. if ( diff --git a/src/application/SetupClientConfiguration.ts b/src/application/SetupClientConfiguration.ts index 393ebf52c6..242f0b148d 100644 --- a/src/application/SetupClientConfiguration.ts +++ b/src/application/SetupClientConfiguration.ts @@ -67,6 +67,7 @@ import type { SetupProviderEnvironment, } from "./SetupTypes.js"; import { + compactInputSchemaByDefault, registrationWriteBlocker, type SetupClient, } from "./SupportedClients.js"; @@ -936,7 +937,7 @@ const clientConfigurationDesired = async ( ); if (!environment.ok) throw environment.error; const registrationEnvironment = - client.name === "kimi" && + compactInputSchemaByDefault(client) && snapshotEnvironment(environment.value).REA_MCP_INPUT_SCHEMA_PROFILE === undefined ? { diff --git a/src/application/SupportedClients.ts b/src/application/SupportedClients.ts index 2dadce7811..7ee94bf8a0 100644 --- a/src/application/SupportedClients.ts +++ b/src/application/SupportedClients.ts @@ -1,6 +1,6 @@ import { dirname, isAbsolute, join, resolve, win32 } from "node:path"; import { fileURLToPath } from "node:url"; -import { lstatSync } from "node:fs"; +import { existsSync, lstatSync, readFileSync } from "node:fs"; import { err, ok, type Result } from "../domain/result.js"; import { HermesClientPathError, @@ -59,6 +59,7 @@ export interface SetupClient { | "omp" | "pi" | "kimi" + | "lm_studio" | "hermes" | "zcode_fallback" | "zcode" @@ -193,6 +194,40 @@ const codexDirectory = ({ home, env }: ClientPathContext): string => const kimiDirectory = ({ home, env }: ClientPathContext): string => env.KIMI_CODE_HOME || join(home, ".kimi-code"); +/** Clients registered with the compact input-schema profile unless one is selected. */ +export const compactInputSchemaByDefault = ({ + name, +}: Pick): boolean => + name === "kimi" || name === "lm_studio"; + +/** + * LM Studio's findLMStudioHome, read-only: a home pointer file wins, then an + * existing ~/.cache/lm-studio, then ~/.lmstudio. Setup never writes the pointer. + */ +const lmStudioLocation = ({ + home, +}: ClientPathContext): Result => { + const pointerPath = join(home, ".lmstudio-home-pointer"); + let directory: string; + try { + directory = readFileSync(pointerPath, "utf8").trim(); + } catch (cause: unknown) { + if (!(cause instanceof Error && "code" in cause && cause.code === "ENOENT")) + return err({ + path: pointerPath, + message: `${pointerPath} could not be read, so REA cannot establish which LM Studio home holds mcp.json: ${cause instanceof Error ? cause.message : String(cause)}. Repair access to that file, then rerun.`, + }); + const cache = join(home, ".cache", "lm-studio"); + directory = existsSync(cache) ? cache : join(home, ".lmstudio"); + } + if (!isAbsolute(directory)) + return err({ + path: pointerPath, + message: `${pointerPath} does not hold an absolute LM Studio home path. Correct or remove that file, then rerun.`, + }); + return ok({ configPath: join(directory, "mcp.json"), markerPath: directory }); +}; + const qoderDirectory = ({ home, env }: ClientPathContext): string => env.QODER_CONFIG_DIR || join(home, ".qoder"); @@ -652,6 +687,12 @@ export const SUPPORTED_CLIENT_DEFINITIONS = [ markerPath: [".workbuddy"], format: "json", }, + { + name: "lm_studio", + displayName: "LM Studio", + location: lmStudioLocation, + format: "lm_studio", + }, { name: "grok_build", displayName: "Grok Build", diff --git a/tests/boundary/filesystem/clientLifecycle.test.ts b/tests/boundary/filesystem/clientLifecycle.test.ts index 7bb9cc22d5..474bf5be61 100644 --- a/tests/boundary/filesystem/clientLifecycle.test.ts +++ b/tests/boundary/filesystem/clientLifecycle.test.ts @@ -62,6 +62,7 @@ describe("client configuration filesystem lifecycle", () => { "vscode", "trae", "workbuddy", + "lm_studio", "grok_build", "omp", "pi", diff --git a/tests/boundary/filesystem/clientRegistrationStatus.test.ts b/tests/boundary/filesystem/clientRegistrationStatus.test.ts index 269f0ef68a..324689d0a6 100644 --- a/tests/boundary/filesystem/clientRegistrationStatus.test.ts +++ b/tests/boundary/filesystem/clientRegistrationStatus.test.ts @@ -167,6 +167,8 @@ describe("client registration environment validation", () => { const home = await createTestTempDirectory("rea-registration-budget-"); await mkdir(join(home, ".cursor")); await mkdir(join(home, ".codex")); + await mkdir(join(home, ".lmstudio")); + await mkdir(join(home, ".kimi-code")); const entry = { command: "npx", args: ["-y", PRODUCT_IDENTITY.registrationPackageSpecifier, "mcp"], @@ -175,6 +177,11 @@ describe("client registration environment validation", () => { join(home, ".cursor/mcp.json"), JSON.stringify({ mcpServers: { rea: entry } }), ); + for (const path of [".lmstudio/mcp.json", ".kimi-code/mcp.json"]) + await writeFile( + join(home, path), + JSON.stringify({ mcpServers: { rea: entry } }), + ); await writeFile( join(home, ".codex/config.toml"), `[mcp_servers.rea]\ncommand = "npx"\nargs = ["-y", "${PRODUCT_IDENTITY.registrationPackageSpecifier}", "mcp"]\n`, @@ -194,10 +201,13 @@ describe("client registration environment validation", () => { // The failing resolver must not invalidate non-Kimi/Codex registrations. expect(byClient.has("cursor")).toBe(true); expect(byClient.get("cursor")?.state).not.toBe("invalid"); - expect(byClient.get("codex")).toMatchObject({ - state: "invalid", - remediation: expect.stringContaining("budget unavailable"), - }); + expect(byClient.has("lm_studio")).toBe(true); + expect(byClient.get("lm_studio")?.state).not.toBe("invalid"); + for (const client of ["codex", "kimi"]) + expect(byClient.get(client)).toMatchObject({ + state: "invalid", + remediation: expect.stringContaining("budget unavailable"), + }); }); }); diff --git a/tests/boundary/filesystem/lmStudioClientConfiguration.test.ts b/tests/boundary/filesystem/lmStudioClientConfiguration.test.ts new file mode 100644 index 0000000000..1be933b4b6 --- /dev/null +++ b/tests/boundary/filesystem/lmStudioClientConfiguration.test.ts @@ -0,0 +1,201 @@ +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { readClientRegistrationStatuses } from "../../../src/application/ClientRegistrationStatus.js"; +import { currentClientRegistrationCommand } from "../../../src/application/ClientRegistrationIdentity.js"; +import { configureClientConfiguration } from "../../../src/application/SetupClientConfiguration.js"; +import { supportedClients } from "../../../src/application/SupportedClients.js"; +import { systemUninstallHost } from "../../../src/application/Uninstall.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { clearClientLocationEnvironment } from "../../fixtures/clientEnvironment.js"; + +const command = currentClientRegistrationCommand(); + +beforeEach(clearClientLocationEnvironment); +afterEach(() => vi.unstubAllEnvs()); + +const lmStudioClient = (home: string) => + supportedClients(home, process.platform, {}).find( + ({ name }) => name === "lm_studio", + ); + +const exists = async (path: string): Promise => { + try { + await access(path); + return true; + } catch { + return false; + } +}; + +const expectedRegistration = (profile = "compact") => ({ + command: command[0], + args: command.slice(1), + env: { REA_MCP_INPUT_SCHEMA_PROFILE: profile }, +}); + +const doctorState = async (home: string) => + ( + await readClientRegistrationStatuses(home, undefined, { environment: {} }) + ).find(({ client }) => client === "lm_studio"); + +/** An installed LM Studio home; the pointer and cache layouts are opt-in. */ +const installedHome = async (layout: "default" | "pointer" | "cache") => { + const home = await createTestTempDirectory("rea-lm-studio-"); + let directory = join(home, ".lmstudio"); + if (layout === "pointer") { + directory = await createTestTempDirectory("rea-lm-studio-target-"); + await writeFile(join(home, ".lmstudio-home-pointer"), `${directory}\n`); + } + if (layout === "cache") directory = join(home, ".cache", "lm-studio"); + await mkdir(directory, { recursive: true }); + const client = lmStudioClient(home); + if (client === undefined) throw new Error("missing LM Studio client"); + return { home, directory, client }; +}; + +describe("LM Studio home resolution", () => { + it.each(["default", "pointer", "cache"] as const)( + "resolves mcp.json in the %s home", + async (layout) => { + const { directory, client } = await installedHome(layout); + expect(client).toEqual({ + name: "lm_studio", + displayName: "LM Studio", + configPath: join(directory, "mcp.json"), + markerPath: directory, + format: "lm_studio", + }); + }, + ); + + it("prefers the home pointer over an existing cache directory", async () => { + const { home, directory } = await installedHome("pointer"); + await mkdir(join(home, ".cache", "lm-studio"), { recursive: true }); + expect(lmStudioClient(home)?.markerPath).toBe(directory); + }); + + it("reports a relative home pointer instead of guessing a home", async () => { + const home = await createTestTempDirectory("rea-lm-studio-relative-"); + await writeFile(join(home, ".lmstudio-home-pointer"), "lmstudio\n"); + expect(lmStudioClient(home)).toMatchObject({ + configPath: join(home, ".lmstudio-home-pointer"), + configPathError: expect.stringContaining("absolute"), + }); + }); +}); + +describe("LM Studio registration lifecycle", () => { + it.each(["default", "pointer", "cache"] as const)( + "registers rea with the compact profile in the %s home", + async (layout) => { + const { home, directory, client } = await installedHome(layout); + const pointer = join(home, ".lmstudio-home-pointer"); + const pointerBefore = (await exists(pointer)) + ? await readFile(pointer, "utf8") + : undefined; + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + expect( + JSON.parse(await readFile(join(directory, "mcp.json"), "utf8")), + ).toEqual({ mcpServers: { rea: expectedRegistration() } }); + expect(await doctorState(home)).toMatchObject({ + state: "aligned", + command, + }); + expect( + (await exists(pointer)) ? await readFile(pointer, "utf8") : undefined, + ).toBe(pointerBefore); + }, + ); + + it("preserves sibling servers, backs up, is idempotent, and uninstalls only rea", async () => { + const { home, client } = await installedHome("default"); + const original = `{ + "mcpServers": { + "other": { "command": "other-server", "args": ["--stdio"], "cwd": "/srv" } + } +} +`; + await writeFile(client.configPath, original); + + expect(await configureClientConfiguration(client, {}, command)).toEqual({ + status: "configured", + backupPath: `${client.configPath}.rea.backup`, + }); + const text = await readFile(client.configPath, "utf8"); + expect(JSON.parse(text)).toEqual({ + mcpServers: { + other: { command: "other-server", args: ["--stdio"], cwd: "/srv" }, + rea: expectedRegistration(), + }, + }); + expect(text).not.toContain('"type"'); + expect(text).not.toContain('"timeout"'); + expect(await readFile(`${client.configPath}.rea.backup`, "utf8")).toBe( + original, + ); + expect(await configureClientConfiguration(client, {}, command)).toEqual({ + status: "unchanged", + }); + expect(await readFile(client.configPath, "utf8")).toBe(text); + expect(await doctorState(home)).toMatchObject({ state: "aligned" }); + + expect((await systemUninstallHost(home).removeClient(client)).status).toBe( + "removed", + ); + expect(JSON.parse(await readFile(client.configPath, "utf8"))).toEqual({ + mcpServers: { + other: { command: "other-server", args: ["--stdio"], cwd: "/srv" }, + }, + }); + }); + + it("keeps an explicit full profile and reports an omitted profile stale", async () => { + const { home, client } = await installedHome("default"); + await writeFile( + client.configPath, + JSON.stringify({ + mcpServers: { rea: { command: command[0], args: command.slice(1) } }, + }), + ); + expect(await doctorState(home)).toMatchObject({ state: "stale" }); + + expect( + ( + await configureClientConfiguration( + client, + { REA_MCP_INPUT_SCHEMA_PROFILE: "full" }, + command, + ) + ).status, + ).toBe("configured"); + expect( + JSON.parse(await readFile(client.configPath, "utf8")).mcpServers.rea, + ).toEqual(expectedRegistration("full")); + expect(await configureClientConfiguration(client, {}, command)).toEqual({ + status: "unchanged", + }); + expect(await doctorState(home)).toMatchObject({ state: "aligned" }); + }); + + it.each([ + '{"mcpServers": {', + '{/* comment */ "mcpServers": {}}', + '{"mcpServers": {},}', + '{"mcpServers": {}}', + ])("rejects invalid strict JSON %j without writes", async (original) => { + const { client } = await installedHome("default"); + await writeFile(client.configPath, original); + expect(await configureClientConfiguration(client, {}, command)).toEqual({ + status: "failed", + reason: "readback", + }); + expect(await readFile(client.configPath, "utf8")).toBe(original); + expect(await exists(`${client.configPath}.rea.backup`)).toBe(false); + }); +}); From abc647bf6cfe4b7f40343448faccf000adceec32 Mon Sep 17 00:00:00 2001 From: JANG BYUNGKUN Date: Wed, 4 Nov 2026 20:53:00 +0900 Subject: [PATCH 57/94] feat(desktop): suggest managed members for Unity script assemblies (#5511) * feat(desktop): suggest managed members for Unity predefined script assemblies Unity Mono triage listed only inspect_managed_artifact probes, mostly for engine and framework assemblies. Assembly-CSharp.dll and Assembly-CSharp-firstpass.dll directly in a managed directory with a UnityEngine reference witness now get an inspect_managed_members probe, ordered before every managed artifact probe. * docs(desktop): describe Unity Mono managed member probes * refactor(desktop): read Unity managed directory from the witness fact * refactor(desktop): reuse the managed directory index for predefined assemblies * feat(desktop): point Unity member probes at summary delivery first * test(desktop): no member probes without a Unity reference witness * docs(desktop): trim Unity member probe note * feat(desktop): suggest managed members in Unity IL2CPP backup folders Unity IL2CPP builds write _BackUpThisFolder_ButDontShipItWithYourGame next to the player. When shipped, its Managed/ holds pre-IL2CPP assemblies. A path convention now records the folder on Assembly-CSharp*.dll directly in its Managed/, including when the admitted root is the folder itself, and triage offers inspect_managed_members for those files. Correspondence to the shipped native code stays unknown. * docs(desktop): note IL2CPP backup folder member probes * docs(desktop): list IL2CPP metadata versions 24 and 27 for the metadata probe * style(desktop): merge stacked doc comments on desktopPathConventions * test(desktop): cover member probes in a macOS bundle managed directory * docs(desktop): note that member probes need the game folder as root * test(desktop): no member probe when the root is the Unity data folder * test(desktop): check the data-folder root still gets managed probes * test(desktop): check the IL2CPP backup convention against the probe bound --------- Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> --- docs/desktop-applications.md | 9 + src/artifacts/DesktopApplicationReader.ts | 25 ++- src/domain/desktopApplication.ts | 6 + .../desktopApplicationProjection.test.ts | 5 + src/domain/desktopApplicationProjection.ts | 32 ++- src/domain/nativeConvention.ts | 46 ++++- .../filesystem/desktopApplication.test.ts | 194 ++++++++++++++++++ 7 files changed, 307 insertions(+), 10 deletions(-) diff --git a/docs/desktop-applications.md b/docs/desktop-applications.md index 58c8926ee8..64f4bf9e2c 100644 --- a/docs/desktop-applications.md +++ b/docs/desktop-applications.md @@ -110,6 +110,15 @@ not establish full metadata validity or correspondence to a native binary; version 24 inspection still applies only to supported header layouts. See [IL2CPP metadata](il2cpp-metadata.md). +In a managed directory with a Unity Mono witness, `Assembly-CSharp.dll` and +`Assembly-CSharp-firstpass.dll` get an `inspect_managed_members` probe listed +first; project-named asmdef assemblies keep only the ordinary managed probes. +The same probe is offered for those names in `Managed/` of a +`_BackUpThisFolder_ButDontShipItWithYourGame` folder (or when that +folder is the root); whether its IL matches the shipped IL2CPP code is unknown. +Point the scan at the game folder: a root that is the `_Data` or +`Managed` folder itself has no Mono witness path and gets no member probe. + Candidates are derived or inferred from these facts and do not prove loading or execution. BepInEx, MelonLoader, Easy Anti-Cheat and BattlEye labels identify path conventions only; runtime presence stays unknown. No bypass is provided. diff --git a/src/artifacts/DesktopApplicationReader.ts b/src/artifacts/DesktopApplicationReader.ts index 5f05bc03e8..7bf2efc7f9 100644 --- a/src/artifacts/DesktopApplicationReader.ts +++ b/src/artifacts/DesktopApplicationReader.ts @@ -1,5 +1,5 @@ import { desktopPathConventions } from "../domain/nativeConvention.js"; -import { isAbsolute } from "node:path"; +import { basename, isAbsolute } from "node:path"; import { DirectoryArtifactReader, DirectoryArtifactRootFailure, @@ -236,6 +236,7 @@ const scanDirectory = async ( const files: DesktopFile[] = []; const omissions: DesktopOmission[] = []; const unityWitnessRoots = new Set(); + const rootName = basename(root.canonical_path); const budget: DesktopReadBudget = { bytes: 0, maximum: input.max_probe_bytes, @@ -299,9 +300,12 @@ const scanDirectory = async ( entries++; abortIfNeeded(signal); const conventions = - entry.kind === "file" ? pathConventions(entry.path) : undefined; + entry.kind === "file" + ? pathConventions(entry.path, rootName) + : undefined; // Reserve the row and every repeated path before retaining them: each - // convention fact copies the path and its probes copy the local path. + // convention fact copies at most the path and its probes copy the + // local path. reserve( RETAINED_ROW_BYTES + retainedTextBytes(entry.path) + @@ -487,10 +491,19 @@ const fileRow = ( }; }; -/** Path-convention facts of one file; each repeats the path as its value. */ -const pathConventions = (path: string) => { - const conventions = desktopPathConventions(path); +/** Path-convention facts of one file; each value repeats at most the path. */ +const pathConventions = (path: string, rootName: string) => { + const conventions = desktopPathConventions(path, rootName); const facts: DesktopFact[] = []; + if ( + conventions.unityIl2cppBackupAssembly && + conventions.unityIl2cppBackupDirectory !== null + ) + facts.push({ + kind: "unity-il2cpp-backup-convention", + offset: null, + value: { backup_directory: conventions.unityIl2cppBackupDirectory }, + }); if (conventions.electronArchive) facts.push({ kind: "electron-asar-convention", offset: null, value: path }); if (conventions.unrealContainer) diff --git a/src/domain/desktopApplication.ts b/src/domain/desktopApplication.ts index 77cf6c9a95..2d7effd10b 100644 --- a/src/domain/desktopApplication.ts +++ b/src/domain/desktopApplication.ts @@ -93,6 +93,11 @@ const desktopFactSchema = z.discriminatedUnion("kind", [ offset: z.null(), value: z.string(), }), + z.strictObject({ + kind: z.literal("unity-il2cpp-backup-convention"), + offset: z.null(), + value: z.strictObject({ backup_directory: z.string() }), + }), z.strictObject({ kind: z.literal("electron-asar-convention"), offset: z.null(), @@ -162,6 +167,7 @@ export const desktopProbeSchema = z.strictObject({ probe_id: prefixedDigestSchema("dsp"), operation: z.enum([ "inspect_managed_artifact", + "inspect_managed_members", "inspect_il2cpp_metadata", "open_binary", "analyze_javascript_application", diff --git a/src/domain/desktopApplicationProjection.test.ts b/src/domain/desktopApplicationProjection.test.ts index 0ca7ddd1cb..d3adc89173 100644 --- a/src/domain/desktopApplicationProjection.test.ts +++ b/src/domain/desktopApplicationProjection.test.ts @@ -62,6 +62,11 @@ const facts: DesktopFact[] = [ value: { container: "FORM", first_chunk: "GEN8" }, }, { kind: "il2cpp-library-convention", offset: null, value: "p" }, + { + kind: "unity-il2cpp-backup-convention", + offset: null, + value: { backup_directory: "B" }, + }, { kind: "electron-asar-convention", offset: null, value: "p" }, { kind: "unreal-container-convention", offset: null, value: "p" }, ]; diff --git a/src/domain/desktopApplicationProjection.ts b/src/domain/desktopApplicationProjection.ts index b7e47c3a0c..2cd31d218c 100644 --- a/src/domain/desktopApplicationProjection.ts +++ b/src/domain/desktopApplicationProjection.ts @@ -1,5 +1,6 @@ import { IL2CPP_METADATA_VERSIONS } from "./il2cpp/il2cppMetadata.js"; import { digestCanonicalValue } from "./canonicalDigest.js"; +import { desktopPathConventions } from "./nativeConvention.js"; import type { DesktopApplication, DesktopFile } from "./desktopApplication.js"; /** The most next probes this projection derives from any one fact. */ @@ -53,13 +54,42 @@ export const projectDesktopApplication = ( managed, "PE optional headers declare nonempty CLI directories; CLR execution and metadata validity remain unverified.", ); + const mono = managed.filter((file) => has(file, "unity-assembly-reference")); + const monoDirectories = new Set( + mono.flatMap((file) => + file.facts.flatMap((fact) => + fact.kind === "unity-assembly-reference" + ? [fact.value.managed_directory] + : [], + ), + ), + ); + for (const file of managed) { + const conventions = desktopPathConventions(file.path); + if ( + conventions.unityPredefinedAssembly && + conventions.unityManagedDirectory !== null && + monoDirectories.has(conventions.unityManagedDirectory) + ) + probe( + "inspect_managed_members", + file, + 'Inventory types, members and user strings of this Unity predefined script assembly; its references and contents are unverified. Start with detail "summary" and page rows through inspect_analysis_view.', + ); + } + for (const file of managed) + if (has(file, "unity-il2cpp-backup-convention")) + probe( + "inspect_managed_members", + file, + 'Inventory types, members and user strings of this predefined script assembly in a Unity IL2CPP backup folder; its correspondence to the shipped native code is unknown. Start with detail "summary" and page rows through inspect_analysis_view.', + ); for (const file of managed) probe( "inspect_managed_artifact", file, "Inspect the declared managed metadata and references.", ); - const mono = managed.filter((file) => has(file, "unity-assembly-reference")); candidate( "unity-mono", mono, diff --git a/src/domain/nativeConvention.ts b/src/domain/nativeConvention.ts index 081e38d0a0..c36f9d4099 100644 --- a/src/domain/nativeConvention.ts +++ b/src/domain/nativeConvention.ts @@ -22,6 +22,11 @@ const DESKTOP_IL2CPP_LIBRARIES = new Set([ "gameassembly.dylib", "libil2cpp.so", ]); +/** Unity's predefined script assembly names; asmdef assembly names are project-chosen. */ +const UNITY_PREDEFINED_ASSEMBLIES = new Set([ + "assembly-csharp.dll", + "assembly-csharp-firstpass.dll", +]); /** * A runtime family named by the library filename or a containing framework. @@ -48,11 +53,19 @@ export const isSwiftRuntimeLibrary = (path: string): boolean => .split("/") .some((segment) => /^libswift.+\.dylib$/u.test(foldSegment(segment))); -/** Parse desktop layout conventions once while preserving original path segments. */ -export const desktopPathConventions = (path: string) => { +/** Unity names the pre-IL2CPP managed backup `_BackUpThisFolder_ButDontShipItWithYourGame`. */ +const UNITY_IL2CPP_BACKUP_SUFFIX = + "_backupthisfolder_butdontshipitwithyourgame"; + +/** + * Parse desktop layout conventions once while preserving original path segments. + * `rootName` is the admitted root directory name; `.` denotes that root in returned directories. + */ +export const desktopPathConventions = (path: string, rootName?: string) => { const segments = path.split("/"); const folded = segments.map(foldSegment); let unityManagedDirectory: string | null = null; + let unityManagedIndex = -1; for (let index = 0; index + 2 < folded.length; index++) { const segment = folded[index]; const namedDataDirectory = @@ -64,12 +77,39 @@ export const desktopPathConventions = (path: string) => { if ( (namedDataDirectory || bundleDataDirectory) && folded[index + 1] === "managed" - ) + ) { unityManagedDirectory = segments.slice(0, index + 2).join("/"); + unityManagedIndex = index + 1; + } + } + // The admitted root may itself be the backup folder, so its name precedes the relative segments. + const rooted = + rootName === undefined ? folded : [foldSegment(rootName), ...folded]; + const shift = rooted.length - folded.length; + let unityIl2cppBackupDirectory: string | null = null; + let unityIl2cppBackupIndex = -1; + for (let index = 0; index + 1 < rooted.length; index++) { + const segment = rooted[index] ?? ""; + if ( + segment.length > UNITY_IL2CPP_BACKUP_SUFFIX.length && + segment.endsWith(UNITY_IL2CPP_BACKUP_SUFFIX) + ) { + unityIl2cppBackupDirectory = + index < shift ? "." : segments.slice(0, index - shift + 1).join("/"); + unityIl2cppBackupIndex = index - shift; + } } const name = folded.at(-1) ?? ""; return { + unityIl2cppBackupDirectory, + unityIl2cppBackupAssembly: + unityIl2cppBackupIndex === segments.length - 3 && + folded[unityIl2cppBackupIndex + 1] === "managed" && + UNITY_PREDEFINED_ASSEMBLIES.has(name), unityManagedDirectory, + unityPredefinedAssembly: + unityManagedIndex === segments.length - 2 && + UNITY_PREDEFINED_ASSEMBLIES.has(name), il2cppLibrary: DESKTOP_IL2CPP_LIBRARIES.has(name), electronArchive: folded.at(-2) === "resources" && diff --git a/tests/boundary/filesystem/desktopApplication.test.ts b/tests/boundary/filesystem/desktopApplication.test.ts index c144d41503..054157520c 100644 --- a/tests/boundary/filesystem/desktopApplication.test.ts +++ b/tests/boundary/filesystem/desktopApplication.test.ts @@ -500,6 +500,200 @@ it.each(["Game_Data/Managed", "Contents/Resources/Data/Managed"])( }, ); +it("orders member probes for Unity predefined script assemblies before every managed artifact probe", async () => { + const root = await createTestTempDirectory("rea-desktop-mono-members-"); + const unity = buildManagedPeFixture({ + references: ["UnityEngine.CoreModule"], + }); + const plain = buildManagedPeFixture({ references: ["System.Runtime"] }); + await put(root, "Game_Data/Managed/Assembly-CSharp.dll", unity); + await put(root, "Game_Data/Managed/ASSEMBLY-CSHARP-FIRSTPASS.DLL", plain); + await put(root, "Game_Data/Managed/Game.Logic.dll", unity); + await put(root, "Game_Data/Managed/UnityEngine.CoreModule.dll", plain); + await put(root, "Game_Data/Managed/Nested/Assembly-CSharp.dll", plain); + await put(root, "Other_Data/Managed/Assembly-CSharp.dll", plain); + await put(root, "Plugins/Assembly-CSharp.dll", unity); + const { report } = await inspect(root); + const relative = new Map( + report.files.map((file) => [file.file_id, file.path]), + ); + const rows = report.next_probes.map((probe) => [ + probe.operation, + relative.get(probe.file_id), + ]); + expect( + rows.filter(([operation]) => operation === "inspect_managed_members"), + ).toEqual([ + [ + "inspect_managed_members", + "Game_Data/Managed/ASSEMBLY-CSHARP-FIRSTPASS.DLL", + ], + ["inspect_managed_members", "Game_Data/Managed/Assembly-CSharp.dll"], + ]); + expect(rows.slice(0, 2).map(([operation]) => operation)).toEqual([ + "inspect_managed_members", + "inspect_managed_members", + ]); + expect( + rows.filter(([operation]) => operation === "inspect_managed_artifact"), + ).toHaveLength(7); + for (const probe of report.next_probes) + if (probe.operation === "inspect_managed_members") + expect(probe.rationale).toContain( + 'Start with detail "summary" and page rows through inspect_analysis_view.', + ); +}); + +it("offers member probes for predefined script assemblies in a macOS bundle's managed directory", async () => { + const root = await createTestTempDirectory("rea-desktop-mono-bundle-"); + const managed = "Contents/Resources/Data/Managed"; + await put( + root, + `${managed}/Assembly-CSharp.dll`, + buildManagedPeFixture({ references: ["UnityEngine.CoreModule"] }), + ); + await put( + root, + `${managed}/Game.Logic.dll`, + buildManagedPeFixture({ references: ["System.Runtime"] }), + ); + const { report } = await inspect(root); + const relative = new Map( + report.files.map((file) => [file.file_id, file.path]), + ); + expect( + report.next_probes + .filter((probe) => probe.operation === "inspect_managed_members") + .map((probe) => relative.get(probe.file_id)), + ).toEqual([`${managed}/Assembly-CSharp.dll`]); + expect(report.next_probes[0]?.operation).toBe("inspect_managed_members"); +}); + +it.each(["Game_Data", "Game_Data/Managed"])( + "offers no member probe when the root is %s itself", + async (rootDirectory) => { + const root = await createTestTempDirectory("rea-desktop-mono-data-root-"); + await put( + root, + "Game_Data/Managed/Assembly-CSharp.dll", + buildManagedPeFixture({ references: ["UnityEngine.CoreModule"] }), + ); + const { report } = await inspect(join(root, rootDirectory)); + const operations = report.next_probes.map((probe) => probe.operation); + expect(operations).toContain("inspect_managed_artifact"); + expect(operations).not.toContain("inspect_managed_members"); + }, +); + +it("offers no member probes when no managed directory has a Unity reference witness", async () => { + const root = await createTestTempDirectory("rea-desktop-mono-no-witness-"); + const plain = buildManagedPeFixture({ references: ["System.Runtime"] }); + await put(root, "Game_Data/Managed/Assembly-CSharp.dll", plain); + await put(root, "Game_Data/Managed/Assembly-CSharp-firstpass.dll", plain); + await put(root, "Contents/Resources/Data/Managed/Assembly-CSharp.dll", plain); + const { report } = await inspect(root); + expect(report.candidates.map((row) => row.family)).toEqual(["dotnet"]); + expect(report.next_probes.map((probe) => probe.operation)).toEqual([ + "inspect_managed_artifact", + "inspect_managed_artifact", + "inspect_managed_artifact", + ]); +}); + +const BACKUP = "Game_BackUpThisFolder_ButDontShipItWithYourGame"; + +it("offers member probes for predefined script assemblies in a game directory's IL2CPP backup folder", async () => { + const root = await createTestTempDirectory("rea-desktop-il2cpp-backup-"); + const plain = buildManagedPeFixture({ references: ["System.Runtime"] }); + await put(root, `${BACKUP}/Managed/Assembly-CSharp.dll`, plain); + await put( + root, + "OTHER_BACKUPTHISFOLDER_BUTDONTSHIPITWITHYOURGAME/MANAGED/ASSEMBLY-CSHARP-FIRSTPASS.DLL", + plain, + ); + await put(root, `${BACKUP}/Managed/Game.Logic.dll`, plain); + await put(root, `${BACKUP}/Managed/Nested/Assembly-CSharp.dll`, plain); + await put(root, `${BACKUP}/Assembly-CSharp.dll`, plain); + await put( + root, + "_BackUpThisFolder_ButDontShipItWithYourGame/Managed/Assembly-CSharp.dll", + plain, + ); + await put(root, "Game_BackUpThisFolder/Managed/Assembly-CSharp.dll", plain); + const metadata = Buffer.alloc(8); + metadata.writeUInt32LE(0xfab11baf); + metadata.writeUInt32LE(31, 4); + await put( + root, + "Game_Data/il2cpp_data/Metadata/global-metadata.dat", + metadata, + ); + const { report } = await inspect(root); + const relative = new Map(report.files.map((file) => [file.file_id, file])); + const members = report.next_probes.filter( + (probe) => probe.operation === "inspect_managed_members", + ); + expect( + members.map((probe) => relative.get(probe.file_id)?.path).sort(), + ).toEqual( + [ + `${BACKUP}/Managed/Assembly-CSharp.dll`, + "OTHER_BACKUPTHISFOLDER_BUTDONTSHIPITWITHYOURGAME/MANAGED/ASSEMBLY-CSHARP-FIRSTPASS.DLL", + ].sort(), + ); + for (const probe of members) { + expect(probe.rationale).toContain( + "correspondence to the shipped native code is unknown", + ); + expect(relative.get(probe.file_id)?.facts).toContainEqual({ + kind: "unity-il2cpp-backup-convention", + offset: null, + value: { + backup_directory: relative.get(probe.file_id)!.path.split("/")[0], + }, + }); + } + const order = report.next_probes.map((probe) => probe.operation); + expect(order.lastIndexOf("inspect_managed_members")).toBeLessThan( + order.indexOf("inspect_il2cpp_metadata"), + ); + expect(report.candidates.map((row) => row.family)).not.toContain( + "unity-mono", + ); + expect( + report.files.filter((file) => + file.facts.some((fact) => fact.kind === "unity-il2cpp-backup-convention"), + ), + ).toHaveLength(2); +}); + +it("recognizes an admitted root that is itself the IL2CPP backup folder", async () => { + const parent = await createTestTempDirectory( + "rea-desktop-il2cpp-backup-root-", + ); + const plain = buildManagedPeFixture({ references: ["System.Runtime"] }); + await put(parent, `${BACKUP}/Managed/Assembly-CSharp.dll`, plain); + await put(parent, "Ordinary/Managed/Assembly-CSharp.dll", plain); + const backup = await inspect(join(parent, BACKUP)); + expect(backup.report.next_probes.map((probe) => probe.operation)).toEqual([ + "inspect_managed_members", + "inspect_managed_artifact", + ]); + expect( + backup.report.files.find( + (file) => file.path === "Managed/Assembly-CSharp.dll", + )?.facts, + ).toContainEqual({ + kind: "unity-il2cpp-backup-convention", + offset: null, + value: { backup_directory: "." }, + }); + const ordinary = await inspect(join(parent, "Ordinary")); + expect(ordinary.report.next_probes.map((probe) => probe.operation)).toEqual([ + "inspect_managed_artifact", + ]); +}); + it("retains complete observations through cancellation during directory finalization", async () => { const root = await createTestTempDirectory("rea-desktop-finalize-cancel-"); await put(root, "notes.txt", "target content"); From 61bb02d3df18b4271e283760b02c84c9c20a0358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A4=80=ED=98=B8?= <101695482+chordpli@users.noreply.github.com> Date: Wed, 4 Nov 2026 20:53:45 +0900 Subject: [PATCH 58/94] fix(artifacts): keep inventorying an ASAR whose member declares an invalid size (#8719) * fix(artifacts): keep inventorying an ASAR whose member declares an invalid size A member whose declared size is not a non-negative safe integer, such as Figma Desktop's .codesign placeholder (size -1000), made readAsarListing throw and failed inventory of the whole archive without naming the entry. Listing now keeps such a member with a null size and a limitation naming the parsed value, as it already keeps an unreadable offset. Opening a packed member with a null size raises ArtifactEntryContentUnavailable, so inventory records it as unavailable and continues with the other members. ElectronFileScope keeps the strict size rule. Member metadata that still stops inventory now names the member and the rejected fields instead of reporting a generic malformed archive. Fixes #8715 * fix(artifacts): bound and preserve ASAR listing diagnostics Quote an unsafe path component or a string size through a 256-unit prefix plus its length, so one oversized header key cannot inflate the failure or the retained limitation. Render numeric sizes with String so a size that JSON.parse turned into Infinity is reported as Infinity rather than null. * fix(artifacts): keep ASAR listing diagnostics exact and located Report a numeric size beyond the safe-integer range as having no retained exact value instead of the number JSON.parse rounded or overflowed, name object and array sizes correctly, and interpret each declared size once without validating absent directory sizes. Quote member and parent paths through the bounded source quote, name the rejected root fields as member failures already do, and append the archive path to listing failures so a nested archive stays identifiable. * test(artifacts): match ASAR listing archive paths that contain spaces --- src/artifacts/AsarArtifactReader.ts | 28 ++- src/artifacts/AsarListing.ts | 94 +++++++- src/artifacts/AsarNode.ts | 5 +- .../filesystem/asarListingAdmission.test.ts | 209 +++++++++++++++++- 4 files changed, 325 insertions(+), 11 deletions(-) diff --git a/src/artifacts/AsarArtifactReader.ts b/src/artifacts/AsarArtifactReader.ts index 92d7191d3a..27542d1f23 100644 --- a/src/artifacts/AsarArtifactReader.ts +++ b/src/artifacts/AsarArtifactReader.ts @@ -78,7 +78,21 @@ export class AsarArtifactReader implements ArtifactReader { jsonBytes: admitted.jsonBytes, }; assertAsarReadActive(signal); - return readAsarListing(admitted.header, signal); + try { + return await readAsarListing(admitted.header, signal); + } catch (cause: unknown) { + // Listing names members relative to this archive; keep which archive. + if ( + cause instanceof ArtifactReaderFailure && + (cause.reason === "format" || cause.reason === "path") + ) + throw new ArtifactReaderFailure( + cause.reason, + `${cause.message} in ${this.path}`, + { cause }, + ); + throw cause; + } }, signal); } catch (cause: unknown) { throw asarFailure(this.path, "inventory", cause); @@ -202,6 +216,18 @@ export class AsarArtifactReader implements ArtifactReader { offset === undefined || headerSize === undefined ? undefined : 8 + headerSize + offset; + if (facts.size === null) + throw new ArtifactEntryContentUnavailable( + `ASAR entry declares no readable size: ${producedEntry.path}`, + "ASAR packed entry bytes were not read because its declared size cannot bound them; no content hash or child artifact was produced.", + undefined, + artifactIntegrityDetails( + producedEntry.path, + producedEntry.declaredIntegrity, + null, + false, + ), + ); if ( archiveSize === undefined || start === undefined || diff --git a/src/artifacts/AsarListing.ts b/src/artifacts/AsarListing.ts index e8bf49c0bb..f4744ec1c7 100644 --- a/src/artifacts/AsarListing.ts +++ b/src/artifacts/AsarListing.ts @@ -1,6 +1,7 @@ import { setImmediate } from "node:timers/promises"; +import { z } from "zod"; -import { asarNodeSchema } from "./AsarNode.js"; +import { asarDeclaredSizeSchema, asarNodeSchema } from "./AsarNode.js"; import { ArtifactReaderFailure, type ArtifactEntry } from "./ArtifactReader.js"; import { assertAsarReadActive } from "./AsarFailure.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; @@ -9,6 +10,68 @@ import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; const MAX_LISTING_BYTES = 64 * 1024 * 1024; const BLOCK_INTEGRITY_LIMITATION = "ASAR block integrity declarations were not validated. Whole-file digest comparison does not establish block integrity or Electron runtime acceptance."; +/** Source text quoted in listing diagnostics; longer values keep their length. */ +const MAX_QUOTED_SOURCE_UNITS = 256; +const quoteSource = (value: string): string => + value.length <= MAX_QUOTED_SOURCE_UNITS + ? JSON.stringify(value) + : `${JSON.stringify(value.slice(0, MAX_QUOTED_SOURCE_UNITS))}... (${String(value.length)} UTF-16 code units)`; +// Listing keeps a member whose size cannot bound its content, as it keeps an +// unreadable offset; only reading that member fails. +const asarListingNodeSchema = asarNodeSchema.extend({ + size: z.unknown().optional(), +}); + +const malformedMetadata = (subject: string, error: z.ZodError) => { + const fields = [ + ...new Set( + error.issues.map((issue) => + issue.path.length === 0 ? "entry" : issue.path.join("."), + ), + ), + ].join(", "); + return new ArtifactReaderFailure( + "format", + `${subject} metadata is malformed (${fields})`, + { cause: error }, + ); +}; + +/** Interpret one declared size once; an invalid one keeps a source fact, not a number. */ +const readDeclaredSize = ( + value: unknown, +): { readonly size: number | null; readonly limitation?: string } => { + if (value === undefined) return { size: null }; + const parsed = asarDeclaredSizeSchema.safeParse(value); + if (parsed.success) return { size: parsed.data }; + // JSON.parse has already rounded a number beyond the safe-integer range or + // turned it into Infinity, so its source spelling is unknown here. + if ( + typeof value === "number" && + !(Number.isFinite(value) && Math.abs(value) <= Number.MAX_SAFE_INTEGER) + ) + return { + size: null, + limitation: + "ASAR entry declares a numeric size outside the safe-integer range; its exact source value was not retained.", + }; + const declared = + typeof value === "number" + ? `size ${String(value)}` + : typeof value === "string" + ? `size ${quoteSource(value)}` + : value === null + ? "a null size" + : Array.isArray(value) + ? "an array size" + : typeof value === "object" + ? "an object size" + : `a ${typeof value} size`; + return { + size: null, + limitation: `ASAR entry declares ${declared}, which is not a non-negative safe integer.`, + }; +}; type AsarListedFacts = Pick< ArtifactEntry, @@ -19,7 +82,8 @@ export type AsarListedEntry = AsarListedFacts & | { readonly kind: "directory" | "symlink"; readonly size: number | null } | { readonly kind: "file"; - readonly size: number; + /** Null when the declared size cannot bound the member's content. */ + readonly size: number | null; readonly packedOffset: number | undefined; } ); @@ -41,7 +105,10 @@ export const readAsarListing = async ( header: unknown, signal?: AbortSignal, ): Promise => { - const root = asarNodeSchema.parse(header); + const parsedRoot = asarListingNodeSchema.safeParse(header); + if (!parsedRoot.success) + throw malformedMetadata("ASAR root", parsedRoot.error); + const root = parsedRoot.data; if (root.files === undefined) throw new ArtifactReaderFailure("format", "ASAR root has no files table"); const entries: AsarListedEntry[] = []; @@ -81,7 +148,7 @@ export const readAsarListing = async ( ) throw new ArtifactReaderFailure( "path", - "ASAR header has an unsafe path component", + `ASAR header has an unsafe path component under ${frame.parent.length === 0 ? "the archive root" : quoteSource(frame.parent)}: ${quoteSource(name)}`, ); const units = frame.parent.length + name.length + (frame.parent.length === 0 ? 0 : 1); @@ -89,7 +156,10 @@ export const readAsarListing = async ( // occurrence even when the engine can share a string prefix. admitRetainedBytes(units * 2 + 256); const path = frame.parent.length === 0 ? name : `${frame.parent}/${name}`; - const metadata = asarNodeSchema.parse(raw); + const parsed = asarListingNodeSchema.safeParse(raw); + if (!parsed.success) + throw malformedMetadata(`ASAR entry ${quoteSource(path)}`, parsed.error); + const metadata = parsed.data; if ( metadata.files === undefined && metadata.link === undefined && @@ -97,13 +167,21 @@ export const readAsarListing = async ( ) throw new ArtifactReaderFailure( "format", - "ASAR entry has no directory, link or file metadata", + `ASAR entry ${quoteSource(path)} has no directory, link or file metadata`, ); const declaration = metadata.integrity; const supportedIntegrity = declaration?.algorithm === "SHA256" && /^[a-f0-9]{64}$/u.test(declaration.hash); const limitations: string[] = []; + const { size, limitation: sizeLimitation } = readDeclaredSize( + metadata.size, + ); + if (sizeLimitation !== undefined) { + // Quoted source text is already bounded; charge the retained copy. + admitRetainedBytes(sizeLimitation.length * 2 + 64); + limitations.push(sizeLimitation); + } if ( declaration !== undefined && (Object.hasOwn(declaration, "blockSize") || @@ -141,13 +219,13 @@ export const readAsarListing = async ( entries.push({ ...facts, kind: metadata.files !== undefined ? "directory" : "symlink", - size: metadata.size ?? null, + size, }); else if (metadata.size !== undefined) entries.push({ ...facts, kind: "file", - size: metadata.size, + size, packedOffset: parseArchiveOffset(metadata.offset), }); if (metadata.files !== undefined) diff --git a/src/artifacts/AsarNode.ts b/src/artifacts/AsarNode.ts index b47a22a12b..ee2ad4f589 100644 --- a/src/artifacts/AsarNode.ts +++ b/src/artifacts/AsarNode.ts @@ -1,5 +1,8 @@ import { z } from "zod"; +/** A declared ASAR member size that can bound a packed content range. */ +export const asarDeclaredSizeSchema = z.number().nonnegative().safe(); + /** Shared ASAR node declarations; consumers select only the fields they interpret. */ export const asarNodeSchema = z.looseObject({ files: z @@ -9,7 +12,7 @@ export const asarNodeSchema = z.looseObject({ ) .optional(), link: z.string().optional(), - size: z.number().nonnegative().safe().optional(), + size: asarDeclaredSizeSchema.optional(), offset: z.string().optional(), unpacked: z.boolean().optional(), executable: z.boolean().optional(), diff --git a/tests/boundary/filesystem/asarListingAdmission.test.ts b/tests/boundary/filesystem/asarListingAdmission.test.ts index f2b0ee4c0c..2705cac904 100644 --- a/tests/boundary/filesystem/asarListingAdmission.test.ts +++ b/tests/boundary/filesystem/asarListingAdmission.test.ts @@ -1,6 +1,6 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; import { createHash } from "node:crypto"; -import { join } from "node:path"; +import { basename, join } from "node:path"; import { buffer } from "node:stream/consumers"; import { setImmediate } from "node:timers/promises"; import { expect, it } from "vitest"; @@ -88,6 +88,82 @@ it.each(["..", ".", "", "a/../b", "a/b", "a\0b"])( }, ); +/** Listing failures name the member, then the archive that holds it. The + * reader may report the canonical spelling of a temporary directory, which can + * contain spaces, so only the archive name is matched exactly. */ +const inArchive = (message: string, path: string) => + expect.stringMatching( + new RegExp( + `^${message.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")} in .*[\\\\/]${basename(path).replace(".", "\\.")}$`, + "u", + ), + ); + +it.each([ + [ + { size: 0, offset: 0 }, + 'ASAR entry "app/main.js" metadata is malformed (offset)', + ], + [ + { size: 0, offset: "0", unpacked: "yes", executable: 1 }, + 'ASAR entry "app/main.js" metadata is malformed (unpacked, executable)', + ], + ["entry", 'ASAR entry "app/main.js" metadata is malformed (entry)'], + [{}, 'ASAR entry "app/main.js" has no directory, link or file metadata'], +])( + "names the member whose metadata %j stops inventory", + async (metadata, message) => { + const json = JSON.stringify({ + files: { app: { files: { "main.js": metadata } } }, + }); + const path = await archiveWith(asarHeaderBytes(json)); + await expect(inventoryArtifact(path)).rejects.toMatchObject({ + reason: "format", + message: inArchive(message, path), + }); + }, +); + +it("names the rejected root fields", async () => { + const path = await archiveWith( + asarHeaderBytes(JSON.stringify({ files: {}, unpacked: "yes" })), + ); + await expect(inventoryArtifact(path)).rejects.toMatchObject({ + reason: "format", + message: inArchive("ASAR root metadata is malformed (unpacked)", path), + }); +}); + +it("bounds the quoted unsafe ASAR path component", async () => { + const component = `${"a".repeat(300)}/x`; + const json = JSON.stringify({ + files: { [component]: { size: 0, offset: "0" } }, + }); + const path = await archiveWith(asarHeaderBytes(json)); + await expect(inventoryArtifact(path)).rejects.toMatchObject({ + reason: "path", + message: inArchive( + `ASAR header has an unsafe path component under the archive root: ${JSON.stringify("a".repeat(256))}... (302 UTF-16 code units)`, + path, + ), + }); +}); + +it("bounds the quoted parent of an unsafe ASAR path component", async () => { + const parent = "d".repeat(300); + const json = JSON.stringify({ + files: { [parent]: { files: { "..": { size: 0, offset: "0" } } } }, + }); + const path = await archiveWith(asarHeaderBytes(json)); + await expect(inventoryArtifact(path)).rejects.toMatchObject({ + reason: "path", + message: inArchive( + `ASAR header has an unsafe path component under ${JSON.stringify("d".repeat(256))}... (300 UTF-16 code units): ".."`, + path, + ), + }); +}); + it("orders raw names portably and retains prototype-named entries and links", async () => { const path = await archiveWith( asarHeaderBytes( @@ -183,6 +259,137 @@ it.each([undefined, "01", "-1", "9007199254740992"])( }, ); +const invalidSize = (declared: string) => + `ASAR entry declares ${declared}, which is not a non-negative safe integer.`; +const unretainedSize = + "ASAR entry declares a numeric size outside the safe-integer range; its exact source value was not retained."; + +// Header spellings, so values JSON.parse rounds or overflows stay exact here. +it.each([ + ["-1000", invalidSize("size -1000")], + ["1.5", invalidSize("size 1.5")], + ['"12"', invalidSize('size "12"')], + ["null", invalidSize("a null size")], + ["true", invalidSize("a boolean size")], + ["{}", invalidSize("an object size")], + ["[]", invalidSize("an array size")], + [ + JSON.stringify("9".repeat(300)), + invalidSize( + `size ${JSON.stringify("9".repeat(256))}... (300 UTF-16 code units)`, + ), + ], + ["9007199254740992", unretainedSize], + ["9007199254740993", unretainedSize], + ["1e400", unretainedSize], + ["-1e400", unretainedSize], +])( + "inventories the other members when one declares size %s", + async (sizeText, limitation) => { + // Figma Desktop ships `.codesign` with size -1000 and the empty-input digest. + const empty = createHash("sha256").update("").digest("hex"); + const payload = Buffer.from("console.log(1);\n"); + const digest = createHash("sha256").update(payload).digest("hex"); + const bytes = Buffer.concat([ + asarHeaderBytes( + JSON.stringify({ + files: { + ".codesign": { + size: "__SIZE__", + offset: "0", + integrity: { algorithm: "SHA256", hash: empty }, + }, + "main.js": { + size: payload.length, + offset: "0", + integrity: { algorithm: "SHA256", hash: digest }, + }, + }, + }).replace('"__SIZE__"', sizeText), + ), + payload, + ]); + const path = await archiveWith(bytes); + + const reader = new AsarArtifactReader(path); + try { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + const invalid = entries.find((entry) => entry.path === ".codesign"); + expect(invalid).toMatchObject({ + kind: "file", + declaredSize: null, + declaredIntegrity: { algorithm: "sha256", digest: empty }, + limitations: [limitation], + }); + await expect(reader.open(invalid!)).rejects.toMatchObject({ + reason: "unavailable", + message: "ASAR entry declares no readable size: .codesign", + }); + const valid = entries.find((entry) => entry.path === "main.js")!; + expect( + Buffer.compare(await buffer(await reader.open(valid)), payload), + ).toBe(0); + } finally { + await reader.close(); + } + + const inventory = await inventoryArtifact(path); + const occurrence = (name: string) => + inventory.occurrences.find((row) => row.logical_path === name); + expect(occurrence(".codesign")).toMatchObject({ + artifact_id: null, + declared_size: null, + hash_status: "unavailable", + limitations: [ + limitation, + "ASAR packed entry bytes were not read because its declared size cannot bound them; no content hash or child artifact was produced.", + ], + }); + expect(occurrence("main.js")?.hash_status).toBe("verified"); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); + }, +); + +it("reads unpacked members and lists directories whose declared size is invalid", async () => { + const payload = Buffer.from("native"); + const digest = createHash("sha256").update(payload).digest("hex"); + const bytes = asarHeaderBytes( + JSON.stringify({ + files: { + companion: { + size: -1, + unpacked: true, + integrity: { algorithm: "SHA256", hash: digest }, + }, + folder: { size: -1, files: { child: { size: 0, offset: "0" } } }, + }, + }), + ); + const path = await archiveWith(bytes); + await mkdir(`${path}.unpacked`); + await writeFile(join(`${path}.unpacked`, "companion"), payload); + const limitation = + "ASAR entry declares size -1, which is not a non-negative safe integer."; + const inventory = await inventoryArtifact(path); + const occurrence = (name: string) => + inventory.occurrences.find((row) => row.logical_path === name); + // Unpacked bytes come from the companion file, not a packed range. + expect(occurrence("companion")).toMatchObject({ + declared_size: null, + unpacked: true, + hash_status: "verified", + limitations: [limitation], + }); + expect(occurrence("companion")?.artifact_id).not.toBeNull(); + expect(occurrence("folder")).toMatchObject({ + entry_kind: "directory", + declared_size: null, + limitations: [limitation], + }); + expect(occurrence("folder/child")?.hash_status).toBe("verified"); +}); + it("preserves directory and link priority with unrecognized integrity declarations", async () => { const bytes = asarHeaderBytes( JSON.stringify({ From e416ce1983a2276fb7fef4412c29c45b8c7a35c1 Mon Sep 17 00:00:00 2001 From: Wenyu Chiou <162016108+WenyuChiou@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:54:29 -0500 Subject: [PATCH 59/94] fix(cli): preserve Markdown validation error cells (#7978) fix(cli): preserve markdown validation error cells --- src/cli.ts | 6 +- src/cli/argumentErrors.ts | 32 ++ src/cli/cliOutputFormatter.ts | 18 +- src/cli/explicitSensitiveArguments.ts | 2 +- .../cli/markdownArgumentErrors.test.ts | 290 ++++++++++++++++++ 5 files changed, 339 insertions(+), 9 deletions(-) create mode 100644 tests/boundary/cli/markdownArgumentErrors.test.ts diff --git a/src/cli.ts b/src/cli.ts index 0203627b9b..67c072dc04 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -78,7 +78,10 @@ import { registerReverseEngineeringCommands } from "./cli/reverseEngineeringComm import { registerCutterCommands } from "./cli/cutterCommands.js"; import { excludeDeclaredSensitiveArguments } from "./cli/explicitSensitiveArguments.js"; import { rejectSurplusCliArguments } from "./cli/strictPositionals.js"; -import { presentCliArgumentErrors } from "./cli/argumentErrors.js"; +import { + encodeCliArgumentErrors, + presentCliArgumentErrors, +} from "./cli/argumentErrors.js"; import { completionHelpNote, createCliCompletions } from "./cli/completions.js"; import { closeCliGuidanceResources, @@ -245,6 +248,7 @@ export const createCli = ( const serveStrict = rejectSurplusCliArguments(cli); presentCliArgumentErrors(cli); const serveDeclared = excludeDeclaredSensitiveArguments(cli); + encodeCliArgumentErrors(cli); const serve = cli.serve.bind(cli); const complete = createCliCompletions( cli as unknown as Parameters[0], diff --git a/src/cli/argumentErrors.ts b/src/cli/argumentErrors.ts index 813dc4b131..7249c22fa7 100644 --- a/src/cli/argumentErrors.ts +++ b/src/cli/argumentErrors.ts @@ -2,6 +2,7 @@ import { Errors } from "incur"; import { z } from "zod"; import { projectInputIssues } from "../domain/inputIssueProjection.js"; +import { encodeMarkdownTableCell } from "./cliOutputFormatter.js"; import type { CliInstance } from "./types.js"; import { visitRegisteredCommands } from "./visitRegisteredCommands.js"; @@ -46,6 +47,37 @@ export const presentCliArgumentErrors = (cli: CliInstance): void => { }); }; +/** Encode only at the final Markdown presentation boundary, after redaction. */ +export const encodeCliArgumentErrors = (cli: CliInstance): void => { + visitRegisteredCommands(cli, (command) => { + command.middleware = [ + async (context, next) => { + try { + await next(); + } catch (error: unknown) { + if ( + context.format === "md" && + error instanceof Errors.ValidationError + ) { + error.fieldErrors = error.fieldErrors.map((field) => ({ + ...field, + ...(field.code === undefined + ? {} + : { code: encodeMarkdownTableCell(field.code) }), + path: encodeMarkdownTableCell(field.path), + expected: encodeMarkdownTableCell(field.expected), + received: encodeMarkdownTableCell(field.received), + message: encodeMarkdownTableCell(field.message), + })); + } + throw error; + } + }, + ...(command.middleware ?? []), + ]; + }); +}; + /** Retain safe selector constraints dropped by Incur's parser projection. */ const adbSelectorConstraints = ( command: string, diff --git a/src/cli/cliOutputFormatter.ts b/src/cli/cliOutputFormatter.ts index 5fe9ecda17..9553ec6901 100644 --- a/src/cli/cliOutputFormatter.ts +++ b/src/cli/cliOutputFormatter.ts @@ -357,20 +357,22 @@ const escapeMarkdownText = (value: string): string => const markdownHeading = (path: readonly string[]): string => `## ${escapeMarkdownText(path.join("."))}\n\n`; +/** Encode literal text for a Markdown table cell, including edge whitespace. */ +export const encodeMarkdownTableCell = (value: string): string => + escapeTableCell(value).replace(/^[ \t]+|[ \t]+$/g, (whitespace) => + encodeMarkdownWhitespace(whitespace), + ); + const renderTable = ( headers: readonly string[], rows: readonly string[][], ): string => { if (headers.length === 0) return ""; - const encodeTableCell = (value: string): string => - escapeTableCell(value).replace(/^[ \t]+|[ \t]+$/g, (spaces) => - encodeMarkdownWhitespace(spaces), - ); - const escapedHeaders = headers.map(encodeTableCell); + const escapedHeaders = headers.map(encodeMarkdownTableCell); const widths = escapedHeaders.map((header) => header.length); for (const row of rows) { for (let index = 0; index < escapedHeaders.length; index += 1) { - const width = encodeTableCell(row[index] ?? "").length; + const width = encodeMarkdownTableCell(row[index] ?? "").length; if (width > widths[index]!) widths[index] = width; } } @@ -384,7 +386,9 @@ const renderTable = ( for (const row of rows) { lines.push( `| ${escapedHeaders - .map((_, index) => pad(encodeTableCell(row[index] ?? ""), index)) + .map((_, index) => + pad(encodeMarkdownTableCell(row[index] ?? ""), index), + ) .join(" | ")} |`, ); } diff --git a/src/cli/explicitSensitiveArguments.ts b/src/cli/explicitSensitiveArguments.ts index c74a53ff22..5308a544e6 100644 --- a/src/cli/explicitSensitiveArguments.ts +++ b/src/cli/explicitSensitiveArguments.ts @@ -116,7 +116,7 @@ export const excludeDeclaredSensitiveArguments = ( command.options instanceof z.ZodObject && Object.hasOwn(command.options.shape, OPTION) ) - // Outermost, so later command middleware failures are covered as well + // Outside the command middleware, so its failures are covered as well // as the parsing and run that Incur executes inside the chain. command.middleware = [excludeFromFailures, ...(command.middleware ?? [])]; }); diff --git a/tests/boundary/cli/markdownArgumentErrors.test.ts b/tests/boundary/cli/markdownArgumentErrors.test.ts new file mode 100644 index 0000000000..e7cc862641 --- /dev/null +++ b/tests/boundary/cli/markdownArgumentErrors.test.ts @@ -0,0 +1,290 @@ +import { Cli, Errors, z } from "incur"; +import { parseFragment, type DefaultTreeAdapterTypes } from "parse5"; +import { estimateTokenCount } from "tokenx"; +import { createMarkdownRenderer, disposeMdItInstance } from "vitepress"; +import { afterAll, expect, it } from "vitest"; + +import { + encodeCliArgumentErrors, + presentCliArgumentErrors, +} from "../../../src/cli/argumentErrors.js"; +import { + excludeDeclaredSensitiveArguments, + explicitSensitiveValueOptions, +} from "../../../src/cli/explicitSensitiveArguments.js"; + +import { cliTest } from "../../support/cli/cliFixture.js"; + +afterAll(() => disposeMdItInstance()); + +cliTest( + "renders complete validation choices in a Markdown table", + async ({ cli }) => { + const args = [ + "search", + "./missing-input", + "needle", + "--kind", + "unrecognized", + ]; + const markdown = await cli.run({ arguments: [...args, "--format", "md"] }); + const full = await cli.run({ + arguments: [...args, "--format", "md", "--full-output"], + }); + const count = await cli.run({ + arguments: [...args, "--format", "md", "--token-count"], + }); + const window = await cli.run({ + arguments: [ + ...args, + "--format", + "md", + "--full-output", + "--token-limit", + "20", + ], + }); + const json = await cli.run({ arguments: [...args, "--format", "json"] }); + expect(markdown.exitCode).toBe(1); + expect(full.exitCode).toBe(1); + expect(count.exitCode).toBe(1); + expect(window.exitCode).toBe(1); + expect(json.exitCode).toBe(1); + expect(markdown.stderr).toBe(""); + expect(json.stderr).toBe(""); + const failure = JSON.parse(json.stdout); + expect(failure.code).toBe("VALIDATION_ERROR"); + const message = failure.fieldErrors[0].message as string; + expect(message).toContain('"strings"|"procedures"'); + + const renderer = await createMarkdownRenderer(process.cwd(), { + linkify: false, + }); + const renderedCells = (output: string): string[] => { + const cells: string[] = []; + const visit = (node: DefaultTreeAdapterTypes.Node): void => { + if ("tagName" in node && node.tagName === "td") { + const text = (child: DefaultTreeAdapterTypes.Node): string => + child.nodeName === "#text" && "value" in child + ? child.value + : "childNodes" in child + ? child.childNodes.map(text).join("") + : ""; + cells.push(text(node)); + } else if ("childNodes" in node) node.childNodes.forEach(visit); + }; + visit(parseFragment(renderer.render(output))); + return cells; + }; + expect(renderedCells(markdown.stdout)).toContain(message); + expect(renderedCells(full.stdout)).toContain(message); + expect(Number(count.stdout.trim())).toBe( + estimateTokenCount(markdown.stdout.trimEnd()), + ); + expect(window.stdout).toContain("[truncated: showing tokens 0–20 of "); + expect(window.stdout).toContain("nextOffset"); + + // A different parser error has a much longer enum. Compare the visible + // cell with the unmodified machine-readable diagnostic, not a subset. + const doctorArgs = ["doctor", "--client", "x|y"]; + const doctorMd = await cli.run({ + arguments: [...doctorArgs, "--format", "md"], + }); + const doctorJson = await cli.run({ + arguments: [...doctorArgs, "--format", "json"], + }); + expect(doctorMd.exitCode).toBe(1); + expect(doctorJson.exitCode).toBe(1); + const doctorMessage = JSON.parse(doctorJson.stdout).fieldErrors[0] + .message as string; + expect(doctorMessage.split("|").length).toBeGreaterThan(2); + expect(renderedCells(doctorMd.stdout)).toContain(doctorMessage); + }, +); + +cliTest( + "redacts a pipe-containing declaration before Markdown encoding", + async ({ cli }) => { + const secret = '"har"|"mitmproxy"'; + const args = [ + "inspect-web-network-capture", + "/nonexistent/capture.har", + "bogus", + ]; + const run = (format: string, declared: boolean) => + cli.run({ + arguments: [ + ...args, + ...(declared ? ["--sensitive-value", secret] : []), + "--format", + format, + ], + }); + const markdown = await run("md", true); + const json = await run("json", true); + const control = await run("json", false); + expect(markdown.exitCode).toBe(1); + expect(json.exitCode).toBe(1); + expect(markdown.stderr).toBe(""); + expect(json.stderr).toBe(""); + expect(markdown.stdout).not.toContain('"har"\\|"mitmproxy"'); + const renderer = await createMarkdownRenderer(process.cwd(), { + linkify: false, + }); + const rendered = renderer.render(markdown.stdout); + expect(rendered).not.toContain(secret); + expect(rendered).toContain("[REDACTED]"); + const message = JSON.parse(json.stdout).fieldErrors[0].message as string; + expect(message).not.toContain(secret); + expect(message).toContain("[REDACTED]"); + expect(JSON.parse(control.stdout).fieldErrors[0].message).toContain(secret); + }, +); + +it("redacts Markdown punctuation in validation fields before presentation", async () => { + for (const secret of [ + "left|right", + "*markup*", + "left\\right", + "&value", + ]) { + const cli = Cli.create("sensitive-fixture", { sync: false }); + cli.command("inspect", { + options: z.object({ ...explicitSensitiveValueOptions }), + run: () => { + throw new Errors.ValidationError({ + message: "fixture", + fieldErrors: [ + { + code: "invalid_value", + missing: false, + path: "input", + expected: "", + received: secret, + message: `Value ${secret} is invalid`, + }, + ], + }); + }, + }); + presentCliArgumentErrors(cli); + const declared = excludeDeclaredSensitiveArguments(cli); + encodeCliArgumentErrors(cli); + const original = cli.serve.bind(cli); + const run = async (format: string, sensitive: boolean) => { + let stdout = ""; + let exitCode = 0; + await declared( + [ + "inspect", + ...(sensitive ? ["--sensitive-value", secret] : []), + "--format", + format, + ], + { + env: {}, + stdout: (text) => { + stdout += text; + }, + exit: (code) => { + exitCode = code; + }, + }, + original, + ); + expect(exitCode).toBe(1); + return stdout; + }; + const md = await run("md", true); + const json = JSON.parse(await run("json", true)); + const control = JSON.parse(await run("json", false)); + const renderer = await createMarkdownRenderer(process.cwd(), { + linkify: false, + }); + expect(md).not.toContain(secret); + expect(renderer.render(md)).not.toContain(secret); + expect(renderer.render(md)).toContain("[REDACTED]"); + expect(json.fieldErrors[0].received).toBe("[REDACTED]"); + expect(json.fieldErrors[0].message).toBe("Value [REDACTED] is invalid"); + expect(control.fieldErrors[0].received).toBe(secret); + expect(control.fieldErrors[0].message).toBe(`Value ${secret} is invalid`); + } +}); + +it("retains upstream Markdown literal-cell protections after redaction", async () => { + const value = "\t:smile: secret|token # {x}\t"; + const secret = "secret|token"; + const cli = Cli.create("literal-validation-fixture", { sync: false }); + cli.command("inspect", { + options: z.object({ ...explicitSensitiveValueOptions }), + run: () => { + throw new Errors.ValidationError({ + message: "fixture", + fieldErrors: [ + { + code: "invalid_value", + missing: false, + path: "input", + expected: "", + received: value, + message: `Value ${value} is invalid`, + }, + ], + }); + }, + }); + presentCliArgumentErrors(cli); + const declared = excludeDeclaredSensitiveArguments(cli); + encodeCliArgumentErrors(cli); + const original = cli.serve.bind(cli); + const run = async (format: string, sensitive: boolean) => { + let stdout = ""; + let exitCode = 0; + await declared( + [ + "inspect", + ...(sensitive ? ["--sensitive-value", secret] : []), + "--format", + format, + ], + { + env: {}, + stdout: (text) => { + stdout += text; + }, + exit: (code) => { + exitCode = code; + }, + }, + original, + ); + expect(exitCode).toBe(1); + return stdout; + }; + const md = await run("md", false); + const redactedMd = await run("md", true); + const json = JSON.parse(await run("json", false)); + const redactedJson = JSON.parse(await run("json", true)); + expect(md).toContain(" "); + expect(md).toContain(":smile:"); + expect(md).toContain("secret\\|token"); + expect(redactedMd).not.toContain("secret\\|token"); + const renderer = await createMarkdownRenderer(process.cwd(), { + linkify: false, + }); + const text = (node: DefaultTreeAdapterTypes.Node): string => + node.nodeName === "#text" && "value" in node + ? node.value + : "childNodes" in node + ? node.childNodes.map(text).join("") + : ""; + const rendered = text(parseFragment(renderer.render(md))); + const redactedRendered = text(parseFragment(renderer.render(redactedMd))); + expect(rendered).toContain(value); + expect(redactedRendered).not.toContain(secret); + expect(redactedRendered).toContain("[REDACTED]"); + expect(json.fieldErrors[0].received).toBe(value); + expect(redactedJson.fieldErrors[0].received).toBe( + value.replace(secret, "[REDACTED]"), + ); +}); From 33e0bf101ab2afddc35da6dba985ab308dc504c9 Mon Sep 17 00:00:00 2001 From: ShuaixinHuang <1836656514@qq.com> Date: Wed, 4 Nov 2026 19:55:13 +0800 Subject: [PATCH 60/94] fix(adb): admit completed path probes after daemon startup (#8858) * fix(adb): admit completed path probes after daemon startup * test(adb): preserve startup probe and digest evidence --- docs/adb-device-analysis.md | 8 +- src/adb/AdbCommand.test.ts | 110 +++++++++++++++++++++- src/adb/AdbCommand.ts | 63 +++++++++---- src/adb/AdbDeviceOutput.ts | 22 +++++ src/adb/AdbProvider.ts | 19 +++- src/adb/AdbPushProbeEvidence.test.ts | 131 +++++++++++++++++++++++++++ 6 files changed, 330 insertions(+), 23 deletions(-) create mode 100644 src/adb/AdbPushProbeEvidence.test.ts diff --git a/docs/adb-device-analysis.md b/docs/adb-device-analysis.md index 1dfcb826da..efb8bf2b34 100644 --- a/docs/adb-device-analysis.md +++ b/docs/adb-device-analysis.md @@ -203,8 +203,12 @@ and during shutdown. `uninstall_adb_package` declare `mutatesTarget` and never run implicitly: pushes check for existing device files before transfer and refuse them unless `overwrite` is set. A failed or unexpected existence probe stops - transfer even with `overwrite`; only a clean `test -e` exit of 1 with no - output is interpreted as absence. The check is not atomic, so a different + transfer even with `overwrite`. A completed `test -e` exit of 1 establishes + absence only with empty stdout and either empty stderr or a recognized, + completed adb-server startup notice sequence. Startup notices are host + observations, not remote-path diagnostics; successful push receipts retain + them in `raw_result.device_path_probes`. Incomplete startup, extra diagnostics, + truncation and cancellation do not establish absence. The check is not atomic, so a different process can create the path before `adb push` writes it. When available, the device's `sha256sum` verifies the transfer. A missing command or output without a recognizable digest reports `device_digest_source: "unavailable"` with an diff --git a/src/adb/AdbCommand.test.ts b/src/adb/AdbCommand.test.ts index 5fff97243e..89cac7286a 100644 --- a/src/adb/AdbCommand.test.ts +++ b/src/adb/AdbCommand.test.ts @@ -65,7 +65,21 @@ it("recognizes only the completed empty false result as absence", async () => { killed: false, }), ); - await expect(probe()).resolves.toEqual({ ok: true, value: false }); + await expect(probe()).resolves.toEqual({ + ok: true, + value: { + matches: false, + capturedOutput: { + exit_code: 1, + stdout: "", + stderr: "", + stdout_bytes: 0, + stderr_bytes: 0, + signal: null, + truncated: false, + }, + }, + }); }); it.each([ @@ -124,7 +138,21 @@ it("recognizes an empty successful probe as existence", async () => { stdout: Buffer.alloc(0), stderr: Buffer.alloc(0), }); - await expect(probe()).resolves.toEqual({ ok: true, value: true }); + await expect(probe()).resolves.toEqual({ + ok: true, + value: { + matches: true, + capturedOutput: { + exit_code: 0, + stdout: "", + stderr: "", + stdout_bytes: 0, + stderr_bytes: 0, + signal: null, + truncated: false, + }, + }, + }); }); it.each(["stdout", "stderr"] as const)( @@ -265,3 +293,81 @@ it("keeps a physical failure even when the later abort reason is that same error error: { _tag: "AnalysisProtocolError", cause: failure }, }); }); + +const STARTUP_NOTICES = [ + "* daemon not running; starting now at tcp:5037\n* daemon started successfully\n", + "* daemon not running. starting it now on port 5037 *\r\n* daemon started successfully *\r\n", +]; +it.each(STARTUP_NOTICES)( + "recognizes completed true and false probes with successful startup: %j", + async (stderr) => { + vi.mocked(execFileBytes).mockResolvedValueOnce({ + stdout: Buffer.alloc(0), + stderr: Buffer.from(stderr), + }); + await expect(probe()).resolves.toMatchObject({ + ok: true, + value: { + matches: true, + capturedOutput: { + stdout: "", + stderr, + exit_code: 0, + stderr_bytes: Buffer.byteLength(stderr), + }, + }, + }); + vi.mocked(execFileBytes).mockRejectedValueOnce( + Object.assign(new Error("false"), { + code: 1, + stdout: "", + stderr, + signal: null, + killed: false, + }), + ); + await expect(probe()).resolves.toMatchObject({ + ok: true, + value: { + matches: false, + capturedOutput: { + stdout: "", + stderr, + exit_code: 1, + stderr_bytes: Buffer.byteLength(stderr), + }, + }, + }); + }, +); + +it.each([ + "* daemon not running; starting now at tcp:5037\n", + "* daemon failed to start\n", + "* daemon started successfully\n* daemon not running; starting now at tcp:5037\n", + "* daemon started successfully\nunexpected\n", + "\n", + " ", +])("refuses an incomplete or unknown startup output: %j", async (stderr) => { + vi.mocked(execFileBytes).mockResolvedValueOnce({ + stdout: Buffer.alloc(0), + stderr: Buffer.from(stderr), + }); + await expect(probe()).resolves.toMatchObject({ + ok: false, + error: { capturedOutput: { stderr, exit_code: 0 } }, + }); + vi.mocked(execFileBytes).mockRejectedValueOnce( + Object.assign(new Error("false"), { + code: 1, + stdout: "", + stderr, + signal: null, + killed: false, + }), + ); + await expect(probe()).resolves.toMatchObject({ + ok: false, + error: { capturedOutput: { stderr, exit_code: 1 } }, + }); +}); diff --git a/src/adb/AdbCommand.ts b/src/adb/AdbCommand.ts index be544ec3e5..5c627493d6 100644 --- a/src/adb/AdbCommand.ts +++ b/src/adb/AdbCommand.ts @@ -16,7 +16,11 @@ import { type ExecFileOutputFailure, } from "../process/ExecFileOutput.js"; import { packageChangeFailureReason } from "./AdbShellOutput.js"; -import { isSuccessfulDaemonStartNotice } from "./AdbDeviceOutput.js"; +import { + isSuccessfulDaemonStartNotice, + isDaemonStartingNotice, + hasOnlySuccessfulAdbStartupNotices, +} from "./AdbDeviceOutput.js"; import type { AdbBinarySelection } from "./AdbConfiguration.js"; const STDERR_EXCERPT_CHARACTERS = 512; @@ -164,12 +168,7 @@ export const runAdbCapture = async ( const line = stderr .slice(diagnosticStart, newline === -1 ? stderr.length : newline) .trimEnd(); - if ( - !isSuccessfulDaemonStartNotice(line) && - !/^\* daemon not running(?:; starting now at [^\r\n]*|\. starting it now on port \d+ \*)$/u.test( - line, - ) - ) + if (!isSuccessfulDaemonStartNotice(line) && !isDaemonStartingNotice(line)) break; diagnosticStart = newline === -1 ? stderr.length : newline + 1; } @@ -272,7 +271,38 @@ export const packageChangeRefusal = ( ); }; -/** Only the clean false result of test -e establishes absence. */ +/** Parsed predicate and the complete output that supports it. */ +export interface AdbPathProbe { + readonly matches: boolean; + readonly capturedOutput: { + readonly stdout: string; + readonly stderr: string; + readonly stdout_bytes: number; + readonly stderr_bytes: number; + readonly exit_code: 0 | 1; + readonly signal: null; + readonly truncated: false; + }; +} +const completedAdbPathProbe = ( + matches: boolean, + stderr: string, +): AdbPathProbe => ({ + matches, + capturedOutput: { + stdout: "", + stderr, + stdout_bytes: 0, + stderr_bytes: Buffer.byteLength(stderr), + exit_code: matches ? 0 : 1, + signal: null, + truncated: false, + }, +}); +const recognizedProbeStderr = (stderr: string): boolean => + stderr === "" || hasOnlySuccessfulAdbStartupNotices(stderr); + +/** Only a complete false predicate, with no device diagnostics, establishes absence. */ export const probeAdbPathExistence = async ( operation: string, serial: string, @@ -280,11 +310,11 @@ export const probeAdbPathExistence = async ( selection: AdbBinarySelection, limits: AdbRunLimits, signal: AbortSignal | undefined, -): Promise> => { +): Promise> => { return probeAdbPath(operation, serial, path, selection, limits, signal, "-e"); }; -/** Only the clean true/false result of test -d establishes directory status. */ +/** Completed predicate status is independent of recognized host startup notices. */ export const probeAdbPathDirectory = async ( operation: string, serial: string, @@ -292,7 +322,7 @@ export const probeAdbPathDirectory = async ( selection: AdbBinarySelection, limits: AdbRunLimits, signal: AbortSignal | undefined, -): Promise> => +): Promise> => probeAdbPath(operation, serial, path, selection, limits, signal, "-d"); const probeAdbPath = async ( @@ -303,7 +333,7 @@ const probeAdbPath = async ( limits: AdbRunLimits, signal: AbortSignal | undefined, predicate: "-d" | "-e", -): Promise> => { +): Promise> => { const fact = predicate === "-d" ? "destination_directory" : "destination_existence"; const probed = await runAdbCapture( @@ -315,8 +345,9 @@ const probeAdbPath = async ( signal, ); if (probed.ok) { - if (probed.value.stdout.length === 0 && probed.value.stderr.length === 0) - return ok(true); + const stderr = probed.value.stderr.toString("utf8"); + if (probed.value.stdout.length === 0 && recognizedProbeStderr(stderr)) + return ok(completedAdbPathProbe(true, stderr)); return err( new AnalysisProtocolError( `The ADB path ${predicate === "-d" ? "directory" : "existence"} probe returned unexpected output; destination status is unknown`, @@ -345,9 +376,9 @@ const probeAdbPath = async ( !failure.outputTruncated && failure.collectionFailure !== true && failure.stdout === "" && - failure.stderr === "" + recognizedProbeStderr(failure.stderr) ) - return ok(false); + return ok(completedAdbPathProbe(false, failure.stderr)); return err( new AnalysisProtocolError(error.message, { cause: error.cause, diff --git a/src/adb/AdbDeviceOutput.ts b/src/adb/AdbDeviceOutput.ts index dc24522b75..4892e2860f 100644 --- a/src/adb/AdbDeviceOutput.ts +++ b/src/adb/AdbDeviceOutput.ts @@ -40,6 +40,28 @@ export const isSuccessfulDaemonStartNotice = (line: string): boolean => line === "* daemon started successfully" || line === "* daemon started successfully *"; +/** Host-side startup line, shared by failure and successful probe parsing. */ +export const isDaemonStartingNotice = (line: string): boolean => + /^\* daemon not running(?:; starting now at [^\r\n]*|\. starting it now on port \d+ \*)$/u.test( + line, + ); + +/** No diagnostics beyond a completed, recognized startup sequence. */ +export const hasOnlySuccessfulAdbStartupNotices = (output: string): boolean => { + let started = false; + for (let offset = 0; offset < output.length;) { + const newline = output.indexOf("\n", offset); + const line = normalizeAdbOutputLine( + output.slice(offset, newline === -1 ? output.length : newline), + ); + if (isSuccessfulDaemonStartNotice(line)) started = true; + else if (isDaemonStartingNotice(line)) started = false; + else return false; + offset = newline === -1 ? output.length : newline + 1; + } + return started; +}; + const metadataValue = (remainder: string, key: string): string | null => { const match = new RegExp(`(?:^|\\s)${key}:(\\S+)`, "u").exec(remainder); return match === null ? null : match[1]!; diff --git a/src/adb/AdbProvider.ts b/src/adb/AdbProvider.ts index 36724a187a..ac8f80c610 100644 --- a/src/adb/AdbProvider.ts +++ b/src/adb/AdbProvider.ts @@ -2407,7 +2407,7 @@ export class AdbProvider { signal, ); if (!directoryProbe.ok) return directoryProbe; - const devicePath = directoryProbe.value + const devicePath = directoryProbe.value.matches ? `${input.device_path}${input.device_path.endsWith("/") ? "" : "/"}${basename(localPath)}` : input.device_path; const existsProbe = await probeAdbPathExistence( @@ -2419,7 +2419,7 @@ export class AdbProvider { signal, ); if (!existsProbe.ok) return existsProbe; - const deviceExists = existsProbe.value; + const deviceExists = existsProbe.value.matches; if (deviceExists && !input.overwrite) return err( new AnalysisOutputError( @@ -2440,6 +2440,17 @@ export class AdbProvider { signal, ); if (!pushed.ok) return pushed; + const observedProbeStartup = + directoryProbe.value.capturedOutput.stderr !== "" || + existsProbe.value.capturedOutput.stderr !== ""; + const probeOutput = observedProbeStartup + ? { + device_path_probes: { + directory: { ...directoryProbe.value.capturedOutput }, + existence: { ...existsProbe.value.capturedOutput }, + }, + } + : undefined; const receipt = ( deviceDigest: AdbDeviceDigest, rawResult?: { @@ -2463,7 +2474,9 @@ export class AdbProvider { }, provider, { - ...(rawResult === undefined ? {} : { rawResult }), + ...(rawResult === undefined && probeOutput === undefined + ? {} + : { rawResult: { ...rawResult, ...probeOutput } }), limitations: [ "Push writes a private snapshot of the caller's file to the device", ...(deviceDigest.device_digest_source === "unavailable" diff --git a/src/adb/AdbPushProbeEvidence.test.ts b/src/adb/AdbPushProbeEvidence.test.ts new file mode 100644 index 0000000000..099a7edf30 --- /dev/null +++ b/src/adb/AdbPushProbeEvidence.test.ts @@ -0,0 +1,131 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it, onTestFinished, vi } from "vitest"; +import { execFileBytes, execFileOutput } from "../process/ExecFileOutput.js"; +import { AdbProvider } from "./AdbProvider.js"; +vi.mock("../process/ExecFileOutput.js", async (importOriginal) => ({ + ...(await importOriginal()), + execFileBytes: vi.fn(), + execFileOutput: vi.fn(), +})); +const startup = + "* daemon not running; starting now at tcp:5037\n* daemon started successfully\n"; +it.each([false, true])( + "retains startup probe evidence alongside digest observations (recognized=%s)", + async (recognized) => { + vi.clearAllMocks(); + const root = await mkdtemp(join(tmpdir(), "rea-adb-probe-evidence-")); + onTestFinished(() => rm(root, { recursive: true, force: true })); + const source = join(root, "input.bin"); + const bytes = Buffer.from("caller-owned bytes"); + await writeFile(source, bytes); + const digest = createHash("sha256").update(bytes).digest("hex"); + const provider = new AdbProvider({ + environment: { REA_ADB_PATH: process.execPath }, + }); + onTestFinished(() => provider.close()); + vi.mocked(execFileOutput).mockImplementation(async (_binary, args) => { + expect(args).toEqual(["version"]); + const stdout = + "Android Debug Bridge version 1.0.41\nVersion 37.0.0-fixture\n"; + return { + stdout, + stderr: "", + stdoutBytes: Buffer.byteLength(stdout), + stderrBytes: 0, + }; + }); + const digestOutput = recognized + ? `${digest} /sdcard/receiver/input.bin\n` + : "unrecognized sha256sum fixture output\n"; + const commands: string[][] = []; + let staged: string | undefined; + vi.mocked(execFileBytes).mockImplementation(async (_binary, args) => { + const command = [...args]; + commands.push(command); + expect(command.slice(0, 2)).toEqual(["-s", "device"]); + if ( + command[2] === "shell" && + command[3] === "test" && + command[4] === "-d" + ) + return { stdout: Buffer.alloc(0), stderr: Buffer.from(startup) }; + if ( + command[2] === "shell" && + command[3] === "test" && + command[4] === "-e" + ) + throw Object.assign(new Error("absent"), { + code: 1, + signal: null, + killed: false, + stdout: "", + stderr: startup, + }); + if (command[2] === "push") { + staged = command[3]; + expect(staged).not.toBe(source); + expect((await readFile(staged!)).equals(bytes)).toBe(true); + return { + stdout: Buffer.from("1 file pushed\n"), + stderr: Buffer.alloc(0), + }; + } + if (command[2] === "shell" && command[3] === "sha256sum") + return { stdout: Buffer.from(digestOutput), stderr: Buffer.alloc(0) }; + throw new Error("Unexpected adb fixture command"); + }); + const result = await provider.execute({ + operation: "push_adb_file", + input: { + serial: "device", + local_path: source, + device_path: "/sdcard/receiver", + overwrite: false, + }, + }); + if (!result.ok) throw result.error; + expect(result.value.result).toMatchObject({ + sha256: digest, + device_sha256: recognized ? digest : null, + overwritten: false, + device_digest_source: recognized ? "device_sha256sum" : "unavailable", + }); + expect(result.value.rawResult).toEqual({ + device_path_probes: { + directory: { + stdout: "", + stderr: startup, + stdout_bytes: 0, + stderr_bytes: Buffer.byteLength(startup), + exit_code: 0, + signal: null, + truncated: false, + }, + existence: { + stdout: "", + stderr: startup, + stdout_bytes: 0, + stderr_bytes: Buffer.byteLength(startup), + exit_code: 1, + signal: null, + truncated: false, + }, + }, + ...(recognized + ? {} + : { device_sha256sum: { stdout: digestOutput, stderr: "" } }), + }); + expect(commands.map((command) => command.slice(2, 5))).toEqual([ + ["shell", "test", "-d"], + ["shell", "test", "-e"], + ["push", staged, "/sdcard/receiver"], + ["shell", "sha256sum", "/sdcard/receiver/input.bin"], + ]); + expect((await readFile(source)).equals(bytes)).toBe(true); + expect(staged).toBeDefined(); + await expect(stat(staged!)).rejects.toMatchObject({ code: "ENOENT" }); + }, +); From 064bb3f713f6e948c28b40ac3bc2bbe8526085e6 Mon Sep 17 00:00:00 2001 From: Paritosh Lahre <155938586+SandstromPL@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:26:05 +0530 Subject: [PATCH 61/94] fix(firmware): reject staged files cut off at the per-file limit (#4430) * fix(firmware): keep producer reports writable under small staging budgets extract_firmware passed max_output_bytes straight to prlimit --fsize. RLIMIT_FSIZE applies to every regular file the producer writes, including the mandatory report.json and unblob.log outside the staging tree, so a staging budget smaller than the report truncated it and failed the extraction even when every staged file fit. Derive the per-file ceiling as the larger of the staging budget and the 8 MiB report budget; polling and the final inventory still enforce the aggregate staging budget. Fixes #3888 Co-Authored-By: Claude Opus 5.5 * fix(firmware): keep files stopped at the size ceiling over budget The per-file RLIMIT_FSIZE ceiling equalled the largest size its readers accept, and both the staging inventory and the report reader accept a file of exactly their budget. A write the kernel stopped at the ceiling therefore left a truncated file that passed as complete. Raise the ceiling by one byte so such a file exceeds its budget and is rejected. Co-Authored-By: Claude Opus 5.5 * fix(firmware): report the effective file-size ceiling on limit failures The exit path re-derived the staging budget for SIGXFSZ and process-limit recovery, so a small-budget run killed at the 8 MiB + 1 ceiling reported file_size_bytes 64 and said the cap was max_output_bytes. Pass the ceiling runFirmwareCommand installs and describe how it is derived. Co-Authored-By: Claude Opus 5.5 * fix(firmware): reject staged files cut off at the per-file limit RLIMIT_FSIZE stops a write at the limit without extending the file, and the staging inventory admits a file of exactly max_output_bytes. With a budget of 8 MiB or more the producer limit equalled the budget, so a staged file the kernel cut short published as a complete extraction; an inherited ceiling below the budget did the same. Derive the per-run limit once in firmwareFileSizeLimit as one byte above the largest budget, still capped by the inherited ceiling, so a file stopped at the limit exceeds its budget. When the inherited ceiling removes that byte, both staging inventories refuse a regular file that reaches the installed limit with a nonretryable file-size constraint naming the inherited limit. Co-Authored-By: Claude Opus 5.5 * fix(firmware): derive limit adjustability from the per-run limit The one byte of headroom makes the per-run file limit equal an inherited ceiling of exactly max_output_bytes + 1, but adjustability still compared that ceiling with the raw budget. A SIGXFSZ then advised raising max_output_bytes, which the same ceiling clamps back to the same limit. Decide adjustability in firmwareFileSizeAdjustable from firmwareFileSizeLimit itself: recommend a larger budget only when it raises that limit. Co-Authored-By: Claude Opus 5.5 * test(firmware): compare file-size recovery with its owners The SIGXFSZ adjustability cases and the inherited-ceiling workflow matched fragments of remediation prose, so a harmless rewrite would fail them. Compare the emitted guidance with firmwareSignalLimitError for the expected branch, check that the two branches stay distinct, and compare the inventory refusal with firmwareFileSizeLimitReached. The selected limit, resource and retryability stay asserted as structured facts. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> --- docs/firmware-analysis.md | 9 +- src/firmware/FirmwareCommand.test.ts | 112 ++++++++++++++---- src/firmware/FirmwareCommand.ts | 54 +++++++-- src/firmware/FirmwareFailures.ts | 4 +- src/firmware/FirmwareFiles.ts | 28 ++++- src/firmware/FirmwareProvider.ts | 5 + .../firmware/firmwarePublicationPaths.test.ts | 1 + .../firmware/firmwareReportFiles.test.ts | 34 +++++- .../firmwareStagingBudgetWiring.test.ts | 2 + .../boundary/firmware/resourceLimits.test.ts | 60 ++++++++++ tests/fixtures/firmware/fileSizeProducer.ts | 93 +++++++++++++++ .../firmwareInheritedFileSizeCeiling.test.ts | 59 +++++++++ 12 files changed, 418 insertions(+), 43 deletions(-) create mode 100644 tests/boundary/firmware/resourceLimits.test.ts create mode 100644 tests/fixtures/firmware/fileSizeProducer.ts create mode 100644 tests/process-global/firmwareInheritedFileSizeCeiling.test.ts diff --git a/docs/firmware-analysis.md b/docs/firmware-analysis.md index de61db9794..4ff0899e93 100644 --- a/docs/firmware-analysis.md +++ b/docs/firmware-analysis.md @@ -245,9 +245,12 @@ tree semantics, vulnerability scanning, emulation or hardware runtime capture. 3, allowed range 1–10. Staging budgets are polled and rechecked after exit; short-lived overshoot is -possible. The per-file process limit is the larger of the selected payload budget -and the fixed report/diagnostic file allowances, so a small payload selection -still permits the mandatory report. A tighter inherited host limit still applies. +possible. The per-file process limit is one byte above the larger of the selected +payload budget and the fixed report/diagnostic file allowances, so a small payload +selection still permits the mandatory report, and a staged file the kernel stops +at the limit exceeds its budget instead of passing as complete. A tighter +inherited host limit still applies and can remove that byte, so a staged file +that reaches the installed limit is rejected as possibly truncated. The inherited file-size limit bounds each file, not total storage; address-space limits are per process, not an aggregate memory quota. Worker count is not CPU affinity for every external extractor. These limits do not diff --git a/src/firmware/FirmwareCommand.test.ts b/src/firmware/FirmwareCommand.test.ts index 8ab02027fe..b7cb294f14 100644 --- a/src/firmware/FirmwareCommand.test.ts +++ b/src/firmware/FirmwareCommand.test.ts @@ -10,10 +10,12 @@ import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory import { AnalysisCapabilityUnavailableError } from "../domain/analysisErrorCore.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { FIRMWARE_LIMITS } from "./FirmwareRelease.js"; +import { firmwareSignalLimitError } from "./FirmwareFailures.js"; import { FirmwareFileAccess } from "./FirmwareFiles.js"; import { resolveFirmwareCommand, runFirmwareCommand, + type FirmwareLauncher, } from "./FirmwareCommand.js"; import { spawnOwnedProviderProcess, @@ -204,7 +206,11 @@ it("rejects a directory configured as the resource limiter executable", async () }); }); -it("keeps captured tool output when firmware stops on SIGXCPU", async () => { +// A launched producer that writes its output, then stops on a limit signal. +const signalledLauncher = ( + signal: NodeJS.Signals, + output: { stdout: string; stderr: string }, +): FirmwareLauncher => { const stdout = new PassThrough(); const stderr = new PassThrough(); const emitter = new EventEmitter(); @@ -216,20 +222,40 @@ it("keeps captured tool output when firmware stops on SIGXCPU", async () => { stdout, stderr, exitCode: null, - signalCode: "SIGXCPU", + signalCode: signal, kill: () => false, }; let ended = 0; const close = () => { ended += 1; - if (ended === 2) emitter.emit("close", null, "SIGXCPU"); + if (ended === 2) emitter.emit("close", null, signal); }; stdout.once("end", close); stderr.once("end", close); setTimeout(() => { - stdout.end("partial scan\n"); - stderr.end("cpu limit\n"); + stdout.end(output.stdout); + stderr.end(output.stderr); }, 15); + return () => + Promise.resolve({ + process: child, + ownership: { + runId: "rea-firmware-signal", + leaderPid: 4242, + processGroupId: 4242, + }, + cleanup: () => Promise.resolve({ cleaned: true, signaled: false }), + }); +}; + +const analysisDeadline = (signal: AbortSignal) => ({ + phase: "analysis" as const, + signal, + timeoutMs: 30_000, + expiresAtMs: performance.now() + 30_000, +}); + +it("keeps captured tool output when firmware stops on SIGXCPU", async () => { const analysis = new AbortController(); const failure: unknown = await runFirmwareCommand({ fileSizeCeiling: Number.MAX_SAFE_INTEGER, @@ -241,22 +267,11 @@ it("keeps captured tool output when firmware stops on SIGXCPU", async () => { cwd: await createTestTempDirectory("rea-firmware-signal-"), environment: process.env, signal: analysis.signal, - deadline: { - phase: "analysis", - signal: analysis.signal, - timeoutMs: 30_000, - expiresAtMs: performance.now() + 30_000, - }, - launcher: () => - Promise.resolve({ - process: child, - ownership: { - runId: "rea-firmware-signal", - leaderPid: 4242, - processGroupId: 4242, - }, - cleanup: () => Promise.resolve({ cleaned: true, signaled: false }), - }), + deadline: analysisDeadline(analysis.signal), + launcher: signalledLauncher("SIGXCPU", { + stdout: "partial scan\n", + stderr: "cpu limit\n", + }), retainCleanup: vi.fn(), }).catch((cause: unknown) => cause); expect(failure).toMatchObject({ @@ -279,6 +294,54 @@ it("keeps captured tool output when firmware stops on SIGXCPU", async () => { }); }); +const sixteenMiB = 16 * 1024 * 1024; +it.each([ + { + inherited: Number.MAX_SAFE_INTEGER, + limit: sixteenMiB + 1, + adjustable: true, + }, + { inherited: sixteenMiB + 1, limit: sixteenMiB + 1, adjustable: false }, +])( + "recommends a larger max_output_bytes on SIGXFSZ only when it raises the limit (inherited $inherited)", + async ({ inherited, limit, adjustable }) => { + const root = await createTestTempDirectory("rea-firmware-fsize-signal-"); + const analysis = new AbortController(); + const failure: unknown = await runFirmwareCommand({ + fileSizeCeiling: inherited, + engine: "unblob", + operation: "extract_firmware", + command: "/opt/unblob", + limiter: "/usr/bin/prlimit", + args: [], + cwd: root, + environment: process.env, + signal: analysis.signal, + deadline: analysisDeadline(analysis.signal), + launcher: signalledLauncher("SIGXFSZ", { stdout: "", stderr: "" }), + outputBudget: { root, bytes: sixteenMiB, entries: 10 }, + retainCleanup: vi.fn(), + }).catch((cause: unknown) => cause); + expect(failure).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + resource: "file-size", + reportedLimits: { file_size_bytes: limit }, + retryable: false, + }); + // Compare with the recovery owner instead of its wording; the two branches + // must differ for the comparison to pin the selected one. + const recovery = (fileSizeAdjustable: boolean) => + firmwareSignalLimitError( + "extract_firmware", + "SIGXFSZ", + limit, + fileSizeAdjustable, + )?.remediationAction; + expect(recovery(true)).not.toBe(recovery(false)); + expect(failure).toHaveProperty("remediationAction", recovery(adjustable)); + }, +); + const executableFixture = async (root: string, name: string) => { const path = join(root, name); await writeFile(path, "#!/bin/sh\n"); @@ -290,11 +353,16 @@ test.each([ { selected: 1, inherited: Number.MAX_SAFE_INTEGER, - expected: 8 * 1024 * 1024, + expected: 8 * 1024 * 1024 + 1, }, { selected: 16 * 1024 * 1024, inherited: Number.MAX_SAFE_INTEGER, + expected: 16 * 1024 * 1024 + 1, + }, + { + selected: 16 * 1024 * 1024, + inherited: 16 * 1024 * 1024, expected: 16 * 1024 * 1024, }, { selected: 16 * 1024 * 1024, inherited: 1024, expected: 1024 }, diff --git a/src/firmware/FirmwareCommand.ts b/src/firmware/FirmwareCommand.ts index 801cd8b97e..1ec49a9d42 100644 --- a/src/firmware/FirmwareCommand.ts +++ b/src/firmware/FirmwareCommand.ts @@ -153,6 +153,40 @@ export const resolveFirmwareCommand = async ( }; }; +/** + * The RLIMIT_FSIZE one producer run gets. It covers every producer file, + * including mandatory reports and logs; outputBudget inventory still governs + * aggregate payload. One byte above the largest budget makes a file the kernel + * stops at the limit exceed that budget instead of passing as complete. An + * inherited ceiling can remove that byte, so inventory also rejects files + * that reach the returned limit. + */ +export const firmwareFileSizeLimit = ( + fileSizeCeiling: number, + stagingBytes?: number, +): number => + Math.min( + fileSizeCeiling, + Math.max( + stagingBytes ?? 0, + FIRMWARE_LIMITS.reportBytes, + FIRMWARE_LIMITS.diagnosticBytes, + ) + 1, + ); + +/** + * Whether a larger max_output_bytes would raise the per-run file limit. It is + * false when the report and diagnostic allowances or the inherited ceiling, + * including its last byte of headroom, set that limit instead. + */ +const firmwareFileSizeAdjustable = ( + fileSizeCeiling: number, + stagingBytes?: number, +): boolean => + stagingBytes !== undefined && + firmwareFileSizeLimit(fileSizeCeiling, stagingBytes + 1) > + firmwareFileSizeLimit(fileSizeCeiling, stagingBytes); + /** Execute one bounded provider command and verify its owned descendants have stopped. */ export const runFirmwareCommand = async (context: { engine: string; @@ -172,21 +206,14 @@ export const runFirmwareCommand = async (context: { retainCleanup(supervisor: ProviderProcessSupervisor): void; }) => { const { engine, operation, signal } = context; - // RLIMIT_FSIZE covers every producer file, including mandatory reports/logs. - // Aggregate extracted payload remains governed by outputBudget inventory. - const fileSizeBytes = Math.min( + const fileSizeBytes = firmwareFileSizeLimit( context.fileSizeCeiling, - Math.max( - context.outputBudget?.bytes ?? 0, - FIRMWARE_LIMITS.reportBytes, - FIRMWARE_LIMITS.diagnosticBytes, - ), + context.outputBudget?.bytes, + ); + const fileSizeAdjustable = firmwareFileSizeAdjustable( + context.fileSizeCeiling, + context.outputBudget?.bytes, ); - const fileSizeAdjustable = - context.outputBudget !== undefined && - context.fileSizeCeiling > context.outputBudget.bytes && - context.outputBudget.bytes >= - Math.max(FIRMWARE_LIMITS.reportBytes, FIRMWARE_LIMITS.diagnosticBytes); const checkInterruption = () => { const failure = firmwarePhaseInterruption( operation, @@ -325,6 +352,7 @@ export const runFirmwareCommand = async (context: { context.outputBudget.root, context.outputBudget.bytes, context.outputBudget.entries, + fileSizeBytes, operation, signal, ); diff --git a/src/firmware/FirmwareFailures.ts b/src/firmware/FirmwareFailures.ts index f9066251ea..ea79d7a254 100644 --- a/src/firmware/FirmwareFailures.ts +++ b/src/firmware/FirmwareFailures.ts @@ -65,7 +65,7 @@ export const firmwareSignalLimitError = ( if (signal !== "SIGXFSZ") return undefined; return recovery( fileSizeAdjustable - ? `The file-size cap for this extraction is max_output_bytes at ${String(fileSizeBytes)} bytes. analysis_timeout_ms does not raise it. Increase max_output_bytes when it is below its schema maximum and the larger staging budget is still required. Repeating the unchanged request fails the same way.` + ? `The file-size cap for this extraction is ${String(fileSizeBytes)} bytes, one byte above max_output_bytes. analysis_timeout_ms does not raise it. Increase max_output_bytes when it is below its schema maximum and the larger staging budget is still required. Repeating the unchanged request fails the same way.` : firmwareLimitRecovery( `The firmware file-size allowance is ${String(fileSizeBytes)} bytes`, ), @@ -79,7 +79,7 @@ export const firmwareProcessLimitRecovery = ( ): string => `cpuSeconds is ${String(FIRMWARE_LIMITS.cpuSeconds)}. The address-space allowance is ${String(FIRMWARE_LIMITS.addressSpaceBytes)} bytes. This release has no runtime override for those per-process allowances, and analysis_timeout_ms does not raise them. ${ fileSizeAdjustable - ? `The file-size cap for this call is max_output_bytes at ${String(fileSizeBytes)} bytes. analysis_timeout_ms does not raise it.` + ? `The file-size cap for this call is ${String(fileSizeBytes)} bytes, one byte above max_output_bytes. analysis_timeout_ms does not raise it.` : `The file-size allowance is ${String(fileSizeBytes)} bytes. This release has no runtime override for that allowance, and analysis_timeout_ms does not raise it.` }`; diff --git a/src/firmware/FirmwareFiles.ts b/src/firmware/FirmwareFiles.ts index e0eeeb7532..612062b08d 100644 --- a/src/firmware/FirmwareFiles.ts +++ b/src/firmware/FirmwareFiles.ts @@ -356,11 +356,15 @@ const firmwareInputFailure = (operation: string, cause?: unknown) => { ]); }; -/** Inventory private output without following links, enforcing aggregate staging budgets. */ +/** + * Inventory private output without following links, enforcing aggregate staging + * budgets and the producer's per-file limit. + */ export const inventoryFirmwareOutput = async ( root: string, maxBytes: number, maxEntries: number, + fileSizeBytes: number, operation: string, signal?: AbortSignal, ): Promise => { @@ -430,6 +434,8 @@ export const inventoryFirmwareOutput = async ( if (stat.isFile()) bytes += stat.size; if (bytes > maxBytes) throw firmwareStagingLimit(operation, "max_output_bytes", maxBytes); + if (stat.isFile() && stat.size >= fileSizeBytes) + throw firmwareFileSizeLimitReached(operation, fileSizeBytes); const linkTarget = stat.isSymbolicLink() ? observedSymlinkTarget(await readlink(path, { encoding: "buffer" })) : undefined; @@ -490,6 +496,26 @@ const firmwareStagingLimit = ( }, ); +/** + * The kernel stops a write at RLIMIT_FSIZE without extending the file, so a + * staged file that reaches the producer's limit may be truncated. That limit + * sits one byte above every budget unless an inherited ceiling removed the byte. + */ +export const firmwareFileSizeLimitReached = ( + operation: string, + fileSizeBytes: number, +) => + new AnalysisResourceConstraintError( + operation, + "file-size", + `A staged firmware file reached the ${String(fileSizeBytes)}-byte per-file limit and may be truncated`, + { file_size_bytes: fileSizeBytes }, + { + retryable: false, + remediationAction: `REA inherited a ${String(fileSizeBytes)}-byte file-size limit (RLIMIT_FSIZE), so it could not keep one byte of headroom above max_output_bytes and the report and diagnostic allowances. A staged file that reaches the limit cannot be told apart from one the kernel cut short. Raise the file-size limit REA inherits (for example ulimit -f in the launching shell or service) above the largest file the extraction needs. analysis_timeout_ms does not raise it, and repeating the unchanged request fails the same way.`, + }, + ); + /** Bind producer paths to the selected input or private extraction tree. */ export const firmwareLogicalPath = ( path: string, diff --git a/src/firmware/FirmwareProvider.ts b/src/firmware/FirmwareProvider.ts index c0653816ab..4941cb2049 100644 --- a/src/firmware/FirmwareProvider.ts +++ b/src/firmware/FirmwareProvider.ts @@ -39,6 +39,7 @@ import { } from "./FirmwareDeadline.js"; import { admitFirmwareVersion } from "./FirmwareVersion.js"; import { + firmwareFileSizeLimit, firmwareToolUnavailable, resolveFirmwareCommand, runFirmwareCommand, @@ -522,6 +523,10 @@ export class FirmwareProvider implements FirmwareAnalysisPort { outputRoot, request.input.max_output_bytes, request.input.max_output_files, + firmwareFileSizeLimit( + command.fileSizeCeiling, + request.input.max_output_bytes, + ), request.operation, signal, ); diff --git a/tests/boundary/firmware/firmwarePublicationPaths.test.ts b/tests/boundary/firmware/firmwarePublicationPaths.test.ts index 61b26a3c73..e0e1fecf4c 100644 --- a/tests/boundary/firmware/firmwarePublicationPaths.test.ts +++ b/tests/boundary/firmware/firmwarePublicationPaths.test.ts @@ -188,6 +188,7 @@ it("publishes the writer's normalized path while retaining source spelling and u staging, 1024, 10, + Number.MAX_SAFE_INTEGER, "extract_firmware", ), normalized, diff --git a/tests/boundary/firmware/firmwareReportFiles.test.ts b/tests/boundary/firmware/firmwareReportFiles.test.ts index c3067b89d9..02e23eb52e 100644 --- a/tests/boundary/firmware/firmwareReportFiles.test.ts +++ b/tests/boundary/firmware/firmwareReportFiles.test.ts @@ -160,6 +160,7 @@ it("orders extracted firmware paths by code point independently of host locale", root, 1024, 16, + Number.MAX_SAFE_INTEGER, "inspect_firmware_regions", ); // Locale collation would interleave case and place "ä" by language rules. @@ -186,7 +187,13 @@ it.skipIf(process.platform !== "linux")( ); await writeFile(join(root, "good.bin"), "1234"); await expect( - inventoryFirmwareOutput(root, 4, 10, "extract_firmware"), + inventoryFirmwareOutput( + root, + 4, + 10, + Number.MAX_SAFE_INTEGER, + "extract_firmware", + ), ).rejects.toMatchObject({ _tag: "AnalysisOutputError", diagnostics: { raw_name_hex: "626164ff2e62696e" }, @@ -204,7 +211,13 @@ it.each([ await writeFile(join(root, "a"), "1"); await writeFile(join(root, "b"), "2"); await expect( - inventoryFirmwareOutput(root, bytes, files, "extract_firmware"), + inventoryFirmwareOutput( + root, + bytes, + files, + Number.MAX_SAFE_INTEGER, + "extract_firmware", + ), ).rejects.toMatchObject({ _tag: "AnalysisResourceConstraintError", resource: "capacity", @@ -215,6 +228,23 @@ it.each([ }, ); +it("refuses a staged file that reaches the producer's per-file limit", async () => { + const root = await createTestTempDirectory("rea-firmware-file-limit-"); + await writeFile(join(root, "below"), "123"); + await expect( + inventoryFirmwareOutput(root, 1024, 10, 4, "extract_firmware"), + ).resolves.toEqual([expect.objectContaining({ size: 3 })]); + await writeFile(join(root, "reached"), "1234"); + await expect( + inventoryFirmwareOutput(root, 1024, 10, 4, "extract_firmware"), + ).rejects.toMatchObject({ + _tag: "AnalysisResourceConstraintError", + resource: "file-size", + reportedLimits: { file_size_bytes: 4 }, + retryable: false, + }); +}); + it("attributes missing sources to the selected path without relabeling snapshot destination failure", async () => { const root = await createTestTempDirectory("rea-firmware-source-"); const files = new FirmwareFileAccess(); diff --git a/tests/boundary/firmware/firmwareStagingBudgetWiring.test.ts b/tests/boundary/firmware/firmwareStagingBudgetWiring.test.ts index 84b34dea4f..d53126c6e2 100644 --- a/tests/boundary/firmware/firmwareStagingBudgetWiring.test.ts +++ b/tests/boundary/firmware/firmwareStagingBudgetWiring.test.ts @@ -37,6 +37,7 @@ it("refuses staging bytes at the schema ceiling with the ceiling recovery", asyn root, MAX_FIRMWARE_STAGING_BYTES, MAX_FIRMWARE_STAGING_ENTRIES, + Number.MAX_SAFE_INTEGER, "extract_firmware", ).catch((cause: unknown) => cause); @@ -71,6 +72,7 @@ it("still offers a raise for a budget below the schema ceiling", async () => { root, 1, MAX_FIRMWARE_STAGING_ENTRIES, + Number.MAX_SAFE_INTEGER, "extract_firmware", ).catch((cause: unknown) => cause); diff --git a/tests/boundary/firmware/resourceLimits.test.ts b/tests/boundary/firmware/resourceLimits.test.ts new file mode 100644 index 0000000000..390781238a --- /dev/null +++ b/tests/boundary/firmware/resourceLimits.test.ts @@ -0,0 +1,60 @@ +import { access } from "node:fs/promises"; +import { expect, it } from "vitest"; +import { firmwareResultSchemas } from "../../../src/domain/firmware/firmwareAnalysis.js"; +import { FIRMWARE_LIMITS } from "../../../src/firmware/FirmwareRelease.js"; +import { firmwareFileSizeService } from "../../fixtures/firmware/fileSizeProducer.js"; +import { assertFirmwareCleanup } from "../../fixtures/firmware/provider.js"; + +const linux = it.skipIf(process.platform !== "linux"); +const budget = 16 * 1024 * 1024; +const limits = { + probeFileBytes: FIRMWARE_LIMITS.reportBytes + 1, + extractionFileBytes: budget + 1, +}; + +linux("publishes a staged file of exactly the staging budget", async () => { + const fixture = await firmwareFileSizeService({ + ...limits, + attemptedBytes: budget, + }); + const outcome = await fixture.service.execute("extract_firmware", { + path: fixture.path, + output_directory: fixture.output, + max_output_bytes: budget, + }); + if (!outcome.ok) throw outcome.error; + const result = firmwareResultSchemas.extract_firmware.parse( + outcome.value.normalized_result, + ); + expect(result.files).toEqual([ + expect.objectContaining({ relative_path: "rootfs/config", size: budget }), + ]); + await assertFirmwareCleanup(fixture.launches); +}); + +linux( + "rejects a staged file the kernel stops at the per-file limit", + async () => { + const fixture = await firmwareFileSizeService({ + ...limits, + attemptedBytes: budget + 4096, + }); + const outcome = await fixture.service.execute("extract_firmware", { + path: fixture.path, + output_directory: fixture.output, + max_output_bytes: budget, + }); + expect(outcome).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisResourceConstraintError", + resource: "capacity", + reportedLimits: { max_output_bytes: budget }, + }, + }); + await expect(access(fixture.output)).rejects.toMatchObject({ + code: "ENOENT", + }); + await assertFirmwareCleanup(fixture.launches); + }, +); diff --git a/tests/fixtures/firmware/fileSizeProducer.ts b/tests/fixtures/firmware/fileSizeProducer.ts new file mode 100644 index 0000000000..148af384c1 --- /dev/null +++ b/tests/fixtures/firmware/fileSizeProducer.ts @@ -0,0 +1,93 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { onTestFinished } from "vitest"; +import { createFirmwareAnalysisProvider } from "../../../src/composition/firmware.js"; +import { FirmwareAnalysisService } from "../../../src/application/firmware/FirmwareAnalysisService.js"; +import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; +import type { assertFirmwareCleanup } from "./provider.js"; +import { createTestTempDirectory } from "../temporaryDirectory.js"; + +/** + * Run the production firmware provider and real prlimit with a synthetic + * Unblob producer that writes one staged file of attemptedBytes. Real Unblob + * has its own verifier. Python keeps SIGXFSZ ignored, so an over-limit write + * fails with EFBIG and leaves a truncated file, as the real producer would. + * The producer asserts the per-file limits it inherits and that the kernel + * left exactly min(attempted, limit) bytes, then reports what is on disk, so + * only REA's limit and inventory can expose the truncation. Python is used + * because Node cannot start under the 1 GiB address-space limit. + */ +export const firmwareFileSizeService = async (limits: { + attemptedBytes: number; + probeFileBytes: number; + extractionFileBytes: number; +}) => { + const { attemptedBytes, probeFileBytes, extractionFileBytes } = limits; + const directory = await createTestTempDirectory("rea-firmware-limits-"); + const command = join(directory, "unblob-probe"); + await writeFile( + command, + `#!/usr/bin/python3 +import errno, hashlib, json, os, resource, sys +if '--version' in sys.argv: + assert resource.getrlimit(resource.RLIMIT_FSIZE) == (${String(probeFileBytes)}, ${String(probeFileBytes)}) + print('26.6.4') + sys.exit(0) +assert resource.getrlimit(resource.RLIMIT_FSIZE) == (${String(extractionFileBytes)}, ${String(extractionFileBytes)}) +args = sys.argv[1:] +report = args[args.index('--report') + 1] +log = args[args.index('--log') + 1] +source = args[-1] +child = os.path.join(args[args.index('--extract-dir') + 1], 'rootfs', 'config') +with open(source, 'rb') as handle: + data = handle.read() +os.makedirs(os.path.dirname(child)) +with open(child, 'wb', buffering=0) as handle: + pending = memoryview(b'x' * ${String(attemptedBytes)}) + try: + while pending: + pending = pending[handle.write(pending):] + except OSError as error: + if error.errno != errno.EFBIG: + raise +with open(child, 'rb') as handle: + content = handle.read() +assert len(content) == min(${String(attemptedBytes)}, ${String(extractionFileBytes)}) +with open(log, 'w') as handle: + handle.write('resource fixture log\\n') +def stat(path, size): + return {'__typename__': 'StatReport', 'path': path, 'size': size, 'is_file': True, 'is_dir': False, 'is_link': False, 'link_target': None} +def task(path, depth, blob_id): + return {'path': path, 'depth': depth, 'blob_id': blob_id, 'is_multi_file': False} +def digest(value): + return {'__typename__': 'HashReport', 'sha256': hashlib.sha256(value).hexdigest()} +child_task = task(child, 1, 'gzip-id') +chunk = {'__typename__': 'ChunkReport', 'id': 'gzip-id', 'start_offset': 0, 'end_offset': len(data), 'size': len(data), 'handler_name': 'gzip', 'is_encrypted': False, 'extraction_reports': []} +with open(report, 'w') as handle: + json.dump([ + {'task': child_task, 'reports': [stat(child, len(content)), digest(content)], 'subtasks': []}, + {'task': task(source, 0, ''), 'reports': [stat(source, len(data)), digest(data), chunk], 'subtasks': [child_task]}, + ], handle) +`, + { mode: 0o700 }, + ); + const path = join(directory, "input.bin"); + await writeFile(path, "fixture-input-data"); + const launches: Parameters[0] = []; + const service = new FirmwareAnalysisService( + createFirmwareAnalysisProvider( + { ...process.env, REA_UNBLOB_COMMAND: command }, + async (options) => { + const spawned = await spawnOwnedProviderProcess(options); + launches.push({ + cwd: options.cwd, + pid: spawned.process.pid, + args: options.arguments, + }); + return spawned; + }, + ), + ); + onTestFinished(() => service.close()); + return { service, path, output: join(directory, "output"), launches }; +}; diff --git a/tests/process-global/firmwareInheritedFileSizeCeiling.test.ts b/tests/process-global/firmwareInheritedFileSizeCeiling.test.ts new file mode 100644 index 0000000000..502575605d --- /dev/null +++ b/tests/process-global/firmwareInheritedFileSizeCeiling.test.ts @@ -0,0 +1,59 @@ +import { execFile } from "node:child_process"; +import { access, readFile } from "node:fs/promises"; +import { promisify } from "node:util"; +import { expect, it, onTestFinished } from "vitest"; +import { firmwareFileSizeLimitReached } from "../../src/firmware/FirmwareFiles.js"; +import { FIRMWARE_LIMITS } from "../../src/firmware/FirmwareRelease.js"; +import { firmwareFileSizeService } from "../fixtures/firmware/fileSizeProducer.js"; +import { assertFirmwareCleanup } from "../fixtures/firmware/provider.js"; + +const execFileAsync = promisify(execFile); +const budget = 16 * 1024 * 1024; + +// Lowers this isolated fork's soft RLIMIT_FSIZE to the staging budget, which +// removes the one byte of headroom the producer limit normally keeps. +it.skipIf(process.platform !== "linux")( + "rejects a staged file that reaches an inherited file-size ceiling", + async () => { + const limits = await readFile("/proc/self/limits", "utf8"); + const soft = /^Max file size\s+(\S+)/mu.exec(limits)?.[1]; + if (soft === undefined) throw new Error("No inherited file-size limit"); + const setSoftLimit = async (value: string): Promise => { + await execFileAsync("prlimit", [ + "--pid", + String(process.pid), + `--fsize=${value}:`, + ]); + }; + await setSoftLimit(String(budget)); + onTestFinished(() => setSoftLimit(soft)); + const fixture = await firmwareFileSizeService({ + attemptedBytes: budget + 4096, + probeFileBytes: FIRMWARE_LIMITS.reportBytes + 1, + extractionFileBytes: budget, + }); + const outcome = await fixture.service.execute("extract_firmware", { + path: fixture.path, + output_directory: fixture.output, + max_output_bytes: budget, + }); + expect(outcome).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisResourceConstraintError", + resource: "file-size", + reportedLimits: { file_size_bytes: budget }, + retryable: false, + // The inventory refusal, not the producer's SIGXFSZ recovery, owns this. + remediationAction: firmwareFileSizeLimitReached( + "extract_firmware", + budget, + ).remediationAction, + }, + }); + await expect(access(fixture.output)).rejects.toMatchObject({ + code: "ENOENT", + }); + await assertFirmwareCleanup(fixture.launches); + }, +); From bd7002b4410486fe88d8c2da8104e0b91bd3f21f Mon Sep 17 00:00:00 2001 From: Ha1baraA1 <243105435+Ha1baraA11@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:56:50 +0800 Subject: [PATCH 62/94] fix(cli): export source Evidence for native value traces (#8297) * fix(cli): export source Evidence for native value traces * fix(cli): export source Evidence for native value traces * fix(cli): preserve trace completeness in summaries --- docs/native-investigation.md | 5 + src/application/DirectAnalysis.ts | 16 +- src/cli/analysisEvidenceDelivery.ts | 38 ++ src/cli/coreNativeCommands.ts | 79 +++- src/domain/analysisErrorProjection.ts | 3 + src/domain/analysisView/analysisView.ts | 2 + src/domain/analysisView/analysisViewResult.ts | 11 + .../analysisView/nativeValueTraceView.ts | 95 +++++ .../cli/traceNativeValuesEvidenceCli.test.ts | 339 ++++++++++++++++++ 9 files changed, 570 insertions(+), 18 deletions(-) create mode 100644 src/domain/analysisView/nativeValueTraceView.ts create mode 100644 tests/boundary/cli/traceNativeValuesEvidenceCli.test.ts diff --git a/docs/native-investigation.md b/docs/native-investigation.md index 83b414f5bd..a60428100a 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -434,6 +434,11 @@ nodes and edges. Node payloads are bounded to 8 MiB and serialized results to 32 MiB. Pagination returns edges whose source is on the node page and preserves stable IDs for endpoints outside that page. +The one-shot CLI accepts `--evidence_output ` to save the trace and +the transitive source Evidence closure named by its links. The destination is +never overwritten. `--detail summary` requires `--evidence_output`; the bundle +still retains the complete trace and source Evidence. + Within one value trace, each exact procedure selector is attempted at most once, including failed analyses. A failed seed returns the provider's original typed failure and corrective guidance; it cannot establish a trace or target identity. diff --git a/src/application/DirectAnalysis.ts b/src/application/DirectAnalysis.ts index bb4f104ba2..4e60e95348 100644 --- a/src/application/DirectAnalysis.ts +++ b/src/application/DirectAnalysis.ts @@ -17,7 +17,10 @@ import type { ResolvedSessionOpen } from "./binary/BinarySessionOpen.js"; import { silentLogger } from "../logger.js"; import type { Logger } from "pino"; import { withEvidenceCallerInputPath } from "../domain/evidence.js"; -import { createImmutableEvidenceBundle } from "../domain/evidenceBundle.js"; +import { + createImmutableEvidenceBundle, + type EvidenceBundle, +} from "../domain/evidenceBundle.js"; import { retainAnalysisResultSources } from "./investigation/recordEvidenceSources.js"; import type { Evidence } from "../domain/evidence.js"; import type { NativeToolName } from "../contracts/native/nativeToolContracts.js"; @@ -139,6 +142,8 @@ export const runDirectAnalysis = async ( readonly architecture?: BinaryArchitecture; readonly entryPoint?: string; readonly espFlash?: EspFlashSelection; + /** Capture the exact session Evidence available to a one-shot CLI adapter. */ + readonly onEvidenceBundle?: (bundle: EvidenceBundle) => void; } = {}, ): Promise> => withProcessCancellation(options.signal, (signal) => @@ -146,6 +151,9 @@ export const runDirectAnalysis = async ( logger: options.logger ?? silentLogger, snapshotPath: options.snapshotPath, signal, + ...(options.onEvidenceBundle === undefined + ? {} + : { onEvidenceBundle: options.onEvidenceBundle }), ...targetSelection(options), ...(options.providerId === undefined ? {} @@ -364,6 +372,7 @@ const runAnalysis = async ( readonly snapshotPath: string | undefined; readonly signal: AbortSignal; readonly evidenceParameters?: Readonly>; + readonly onEvidenceBundle?: (bundle: EvidenceBundle) => void; readonly createSession?: () => BinarySession | Promise; readonly providerId?: AnalysisProviderSelector; readonly formatHint?: ExecutableFormatHint; @@ -458,8 +467,10 @@ const runAnalysis = async ( : routeProviderIdentity(route, tool), evidenceProfile, }); - if (cached !== undefined) + if (cached !== undefined) { + options.onEvidenceBundle?.(snapshot.evidence_bundle); return ok(withEvidenceCallerInputPath(cached, path)); + } } } const openOptions = { @@ -552,6 +563,7 @@ const runAnalysis = async ( return err(written.error.retainPartialObservation(evidence)); } } + options.onEvidenceBundle?.(session.exportEvidenceBundle()); return ok(evidence); }; const outcome = await withSessionCleanup<{ diff --git a/src/cli/analysisEvidenceDelivery.ts b/src/cli/analysisEvidenceDelivery.ts index 2d0cf5d26c..dbf82b0812 100644 --- a/src/cli/analysisEvidenceDelivery.ts +++ b/src/cli/analysisEvidenceDelivery.ts @@ -14,6 +14,7 @@ import { import type { JsonValue } from "../domain/jsonValue.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { EvidenceReferenceError } from "../domain/evidenceErrors.js"; import { redactExplicitFailure } from "../domain/explicitSensitiveFailure.js"; import { err, ok, type Result } from "../domain/result.js"; import { AnalysisInputError } from "../domain/analysisErrorCore.js"; @@ -36,6 +37,7 @@ import { import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; import type { ZodSafeParseResult } from "zod"; import { bindCliEvidenceCallerPath } from "./cliEvidenceCallerPath.js"; +import { resolveInvestigationEvidenceClosure } from "../application/investigation/InvestigationEvidence.js"; export const analysisEvidenceDeliveryOptions = { detail: z @@ -78,6 +80,42 @@ export const cliEvidenceDeliveryInputError = ( ]) : undefined; +/** Resolve only the authenticated Evidence records linked by one CLI result. */ +export const cliEvidenceSourceClosure = ( + root: Evidence, + available: readonly Evidence[], +): Result => { + const recordsById = new Map( + available.map((record) => [record.evidence_id, record]), + ); + const closure = resolveInvestigationEvidenceClosure( + { evidenceForAnalysis: (evidenceId) => recordsById.get(evidenceId) }, + root.evidence_links, + ); + if (!closure.ok) { + if (closure.error instanceof EvidenceReferenceError) + return err( + new EvidenceReferenceError( + closure.error.evidenceId, + closure.error.reason, + { + cause: closure.error, + expected: closure.error.expected, + actual: closure.error.actual, + referenceScope: closure.error.referenceScope, + diagnostics: { operation: root.operation }, + }, + ), + ); + return err(closure.error); + } + return ok( + closure.value + .filter((evidenceId) => evidenceId !== root.evidence_id) + .map((evidenceId) => recordsById.get(evidenceId)!), + ); +}; + /** Parse delivery JSON once and enforce its owning schema before opening a target. */ export const parseCliAnalysisEvidenceDelivery = < Delivery extends { diff --git a/src/cli/coreNativeCommands.ts b/src/cli/coreNativeCommands.ts index d438e4bb39..72d150e652 100644 --- a/src/cli/coreNativeCommands.ts +++ b/src/cli/coreNativeCommands.ts @@ -30,9 +30,12 @@ import { withCommandCancellation } from "./commandCancellation.js"; import { analysisEvidenceDeliveryOptions, cliEvidenceDeliveryInputError, + cliEvidenceSourceClosure, publishCliAnalysisEvidence, } from "./analysisEvidenceDelivery.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { AnalysisOutputError } from "../domain/analysisErrorCore.js"; +import type { EvidenceBundle } from "../domain/evidenceBundle.js"; const bytePatternInputSchema = z .object(bytePatternFields) @@ -236,25 +239,69 @@ export const registerCoreNativeCommands = ( .max(5000) .default(1000) .describe("Maximum nodes on this page"), + ...analysisEvidenceDeliveryOptions, ...nativeOpenOptionFields, }), run: ({ args, options }) => - logCliCommand(logger, CLI_COMMANDS.traceNativeValues, () => - runDirectAnalysis( - args.path, - "trace_native_values", - { - procedure: args.procedure, - max_depth: options.maxDepth, - max_functions: options.maxFunctions, - max_call_sites: options.maxCallSites, - max_nodes: options.maxNodes, - max_edges: options.maxEdges, - offset: options.offset, - limit: options.limit, - }, - directAnalysisOptionsFromCli(logger, options), - ).then(projectCliAnalysisResult), + withCommandCancellation((signal) => + logCliCommand(logger, CLI_COMMANDS.traceNativeValues, async () => { + const delivery = { + providerId: "auto", + operation: "trace_native_values", + signal, + detail: options.detail, + evidenceOutput: options.evidence_output, + }; + const invalid = cliEvidenceDeliveryInputError(delivery); + if (invalid !== undefined) return projectAnalysisError(invalid); + let sessionBundle: EvidenceBundle | undefined; + const complete = await runDirectAnalysis( + args.path, + "trace_native_values", + { + procedure: args.procedure, + max_depth: options.maxDepth, + max_functions: options.maxFunctions, + max_call_sites: options.maxCallSites, + max_nodes: options.maxNodes, + max_edges: options.maxEdges, + offset: options.offset, + limit: options.limit, + }, + { + ...directAnalysisOptionsFromCli(logger, options), + signal, + ...(options.evidence_output === undefined + ? {} + : { + onEvidenceBundle: (bundle: EvidenceBundle) => { + sessionBundle = bundle; + }, + }), + }, + ); + if (!complete.ok || options.evidence_output === undefined) + return publishCliAnalysisEvidence(complete, delivery); + if (sessionBundle === undefined) + return projectAnalysisError( + new AnalysisOutputError( + "trace_native_values", + "The CLI session did not return its retained Evidence bundle.", + ).retainPartialObservation(complete.value), + ); + const sources = cliEvidenceSourceClosure( + complete.value, + sessionBundle.records, + ); + if (!sources.ok) + return projectAnalysisError( + sources.error.retainPartialObservation(complete.value), + ); + return publishCliAnalysisEvidence(complete, { + ...delivery, + sourceEvidence: sources.value, + }); + }), ), }); cli.command(CLI_COMMANDS.inspectNativeDataType, { diff --git a/src/domain/analysisErrorProjection.ts b/src/domain/analysisErrorProjection.ts index 9211830bdf..b01c4a5893 100644 --- a/src/domain/analysisErrorProjection.ts +++ b/src/domain/analysisErrorProjection.ts @@ -472,6 +472,9 @@ const requestErrorDetails = ( return { evidence_id: error.evidenceId, reason: error.reason, + ...(typeof error.diagnostics?.operation === "string" + ? { operation: error.diagnostics.operation } + : {}), ...(error.referenceScope === undefined ? {} : { reference_scope: error.referenceScope }), diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index 91937ee723..1b56a6b222 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -3,6 +3,7 @@ import { NATIVE_OBSERVATION_OPERATIONS, } from "../native/nativeObservation.js"; import { projectNativeObservationView } from "./nativeObservationView.js"; +import { projectNativeValueTraceView } from "./nativeValueTraceView.js"; import { projectDevicetreeView } from "./devicetreeView.js"; import { projectApkSignatureView } from "./apkSignatureView.js"; import { byteEntropySchema } from "../native/byteEntropy.js"; @@ -303,6 +304,7 @@ const retainedViewProjectors = new Map([ ...NATIVE_OBSERVATION_OPERATIONS.map((operation) => retain(operation, projectNativeObservationView), ), + retain("trace_native_values", projectNativeValueTraceView), retain("inspect_erlang_beam", projectErlangBeamView), retain("inspect_devicetree", projectDevicetreeView), retain("inspect_native_exports", projectNativeExportsView), diff --git a/src/domain/analysisView/analysisViewResult.ts b/src/domain/analysisView/analysisViewResult.ts index 2b5e9d3cd3..c7240d0ce8 100644 --- a/src/domain/analysisView/analysisViewResult.ts +++ b/src/domain/analysisView/analysisViewResult.ts @@ -16,6 +16,7 @@ import { NATIVE_OBSERVATION_OPERATIONS, type NativeObservationOperation, } from "../native/nativeObservation.js"; +import { nativeValueTraceSummarySchema } from "./nativeValueTraceView.js"; import { devicetreeViewRequestSchema, devicetreeViewItemsSchema, @@ -348,6 +349,7 @@ const viewResultBase = { "inspect_gpu_code_objects", ...NATIVE_INVENTORY_OPERATIONS, ...NATIVE_OBSERVATION_OPERATIONS, + "trace_native_values", "match_native_functions", ]), parent_digest: digestSchema, @@ -388,6 +390,14 @@ export const nativeObservationSummaryViewResultSchemas = { ), get_call_graph: nativeObservationSummaryFor("get_call_graph"), }; + +export const nativeValueTraceSummaryViewResultSchema = z.strictObject({ + ...summaryViewResultBase, + artifact: artifactSchema.nullable(), + parent_operation: z.literal("trace_native_values"), + summary: nativeValueTraceSummarySchema, +}); + const nativeObservationSummaryViewResultSchema = z.discriminatedUnion( "parent_operation", [ @@ -1120,6 +1130,7 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ }), z.discriminatedUnion("parent_operation", [ nativeObservationSummaryViewResultSchema, + nativeValueTraceSummaryViewResultSchema, erlangBeamSummaryViewResultSchema, devicetreeSummaryViewResultSchema, pythonPycSummaryViewResultSchema, diff --git a/src/domain/analysisView/nativeValueTraceView.ts b/src/domain/analysisView/nativeValueTraceView.ts new file mode 100644 index 0000000000..a26215a5cc --- /dev/null +++ b/src/domain/analysisView/nativeValueTraceView.ts @@ -0,0 +1,95 @@ +import { z } from "zod"; +import { nativeValueTraceSchema } from "../native/nativeValueTrace.js"; +import { err, ok } from "../result.js"; +import type { AnalysisViewParent } from "./analysisViewParent.js"; +import { createAnalysisViewResultReader } from "./analysisViewParent.js"; +import { + assertOwnedView, + type AdmittedView, + type SummaryAnalysisView, +} from "./analysisViewShape.js"; +import { analysisViewInputError } from "./analysisViewErrors.js"; +import { + boundedViewMessages, + completeWithinViewCoverage, +} from "./analysisViewCoverage.js"; + +/** Counts and pagination state for a bounded native value trace page. */ +export const nativeValueTraceSummarySchema = z.strictObject({ + seed: z.string(), + target_sha256: z.string(), + node_count: z.int().nonnegative(), + total_nodes: z.int().nonnegative(), + edge_count: z.int().nonnegative(), + total_edges: z.int().nonnegative(), + procedure_count: z.int().nonnegative(), + unknown_count: z.int().nonnegative(), + decompilations: z.int().nonnegative(), + next_offset: z.int().nonnegative().nullable(), + page_truncated: nativeValueTraceSchema.shape.page_truncated, + traversal: nativeValueTraceSchema.shape.traversal, + truncated: z.boolean(), + limitation_count: z.int().nonnegative(), +}); + +const readTrace = createAnalysisViewResultReader(nativeValueTraceSchema); + +/** Keep the bounded graph page and its completeness state in source Evidence. */ +export const projectNativeValueTraceView = ( + parent: AnalysisViewParent, + view: AdmittedView, +) => { + if (view.kind !== "summary") + return err( + analysisViewInputError([ + { + path: ["view"], + reason: "invalid_value", + message: + "Native value trace Evidence supports the summary view; retrieve parent Evidence for graph nodes and edges.", + }, + ]), + ); + assertOwnedView(view, "summary"); + const parsed = readTrace(parent); + if (!parsed.ok) return parsed; + const trace = parsed.value; + return ok({ + kind: "summary" as const, + view, + parent_evidence_id: parent.evidenceId, + parent_operation: "trace_native_values" as const, + parent_digest: parent.evidenceId.slice(3), + artifact: parent.artifact ?? null, + summary: { + seed: trace.seed, + target_sha256: trace.target_sha256, + node_count: trace.nodes.length, + total_nodes: trace.total_nodes, + edge_count: trace.edges.length, + total_edges: trace.total_edges, + procedure_count: trace.procedures.length, + unknown_count: trace.unknowns.length, + decompilations: trace.decompilations, + next_offset: trace.next_offset, + page_truncated: trace.page_truncated, + traversal: trace.traversal, + truncated: trace.truncated, + limitation_count: trace.limitations.length, + }, + coverage: completeWithinViewCoverage(1, 1), + limitations: boundedViewMessages( + [...parent.limitations, ...trace.limitations], + parent.evidenceId, + "limitations", + ), + unknowns: boundedViewMessages( + trace.unknowns.map( + ({ procedure, address, reason }) => + `${procedure}${address === null ? "" : ` at ${address}`}: ${reason}`, + ), + parent.evidenceId, + "unknowns", + ), + }); +}; diff --git a/tests/boundary/cli/traceNativeValuesEvidenceCli.test.ts b/tests/boundary/cli/traceNativeValuesEvidenceCli.test.ts new file mode 100644 index 0000000000..aab76306b6 --- /dev/null +++ b/tests/boundary/cli/traceNativeValuesEvidenceCli.test.ts @@ -0,0 +1,339 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { Cli } from "incur"; +import pino from "pino"; +import { expect, it, vi } from "vitest"; + +import { runDirectAnalysis } from "../../../src/application/DirectAnalysis.js"; +import { registerCoreNativeCommands } from "../../../src/cli/coreNativeCommands.js"; +import { projectCliAnalysisResult } from "../../../src/cli/projectCliAnalysisResult.js"; +import type { CliInstance } from "../../../src/cli/types.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { + createEvidenceBundle, + parseEvidenceBundle, + type EvidenceBundle, +} from "../../../src/domain/evidenceBundle.js"; +import { REA_WORKFLOW_PROVIDER } from "../../../src/domain/investigationProviders.js"; +import { nativeValueTraceSchema } from "../../../src/domain/native/nativeValueTrace.js"; +import { ok } from "../../../src/domain/result.js"; +import { composedWorkflowFixture } from "../../fixtures/composedWorkflowEvidence.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const sourceProvider = { id: "fixture", name: "Fixture", version: "1" }; +const sourceTarget = { + path: "/fixtures/native.exe", + format: "pe" as const, + sha256: "a".repeat(64), +}; + +const createTraceFixture = () => { + const leaf = createEvidence(sourceTarget, sourceProvider, { + operation: "read_native_source", + parameters: { address: "0x1000" }, + result: { text: "return 0;" }, + }); + const procedure = createEvidence(sourceTarget, sourceProvider, { + operation: "analyze_function", + parameters: { procedure: "main" }, + result: { procedure: { address: "0x1000" } }, + evidenceLinks: [leaf.evidence_id], + }); + const traceResult = nativeValueTraceSchema.parse({ + seed: "main", + target_sha256: sourceTarget.sha256, + nodes: [ + { + id: "node-1", + procedure: "0x1000", + evidence_id: procedure.evidence_id, + kind: "operation", + operation: null, + parameter: null, + }, + ], + edges: [], + procedures: [ + { + address: "0x1000", + depth: 0, + evidence_id: procedure.evidence_id, + provider: "fixture", + analysis_profile_digest: null, + }, + ], + unknowns: [ + { procedure: "0x1000", address: null, reason: "fixture uncertainty" }, + ], + total_nodes: 2, + total_edges: 0, + decompilations: 1, + next_offset: 1, + page_truncated: true, + traversal: { status: "complete" }, + truncated: true, + limitations: ["fixture trace limitation"], + }); + const trace = createEvidence(sourceTarget, REA_WORKFLOW_PROVIDER, { + operation: "trace_native_values", + parameters: { procedure: "main" }, + result: traceResult, + confidence: "derived", + evidenceLinks: [procedure.evidence_id], + }); + const unrelated = createEvidence(sourceTarget, sourceProvider, { + operation: "address_name", + parameters: { address: "0x2000" }, + result: { name: "unrelated" }, + }); + const sessionBundle = createEvidenceBundle([ + trace, + procedure, + leaf, + unrelated, + ]); + return { leaf, procedure, trace, unrelated, sessionBundle }; +}; + +type DirectAnalysisRunner = Parameters[2]; + +const createTraceCli = (run: DirectAnalysisRunner) => { + const cli = Cli.create("rea-trace-evidence-test", { + version: "1", + sync: false, + }); + registerCoreNativeCommands( + cli as unknown as CliInstance, + pino({ enabled: false }), + run, + ); + return cli; +}; + +const runJsonCli = async ( + cli: ReturnType, + args: string[], +) => { + let stdout = ""; + await cli.serve(args, { + env: {}, + stdout: (text: string) => { + stdout += text; + }, + exit: () => undefined, + }); + return JSON.parse(stdout) as Record; +}; + +const verifyDirectAnalysisEvidenceBundleCallback = async ( + snapshotPath: string, +) => { + const fixture = await composedWorkflowFixture(); + let captured: EvidenceBundle | undefined; + const direct = parseEvidence( + await runDirectAnalysis( + fixture.dependencies, + fixture.path, + "binary_overview", + {}, + { snapshotPath, onEvidenceBundle: (bundle) => (captured = bundle) }, + ).then(projectCliAnalysisResult), + ); + expect(captured?.records.map(({ evidence_id }) => evidence_id)).toContain( + direct.evidence_id, + ); + + let replayedBundle: EvidenceBundle | undefined; + const replayed = parseEvidence( + await runDirectAnalysis( + fixture.dependencies, + fixture.path, + "binary_overview", + {}, + { + snapshotPath, + onEvidenceBundle: (bundle) => (replayedBundle = bundle), + }, + ).then(projectCliAnalysisResult), + ); + expect(replayed).toEqual(direct); + expect( + replayedBundle?.records.map(({ evidence_id }) => evidence_id), + ).toContain(replayed.evidence_id); +}; + +it("exports only the transitive source Evidence closure for a native value trace", async () => { + const { leaf, procedure, trace, unrelated, sessionBundle } = + createTraceFixture(); + const run = vi.fn( + async (_path, _tool, _arguments, options) => { + options?.onEvidenceBundle?.(sessionBundle); + return ok(trace); + }, + ); + const cli = createTraceCli(run); + const directory = await createTestTempDirectory("rea-trace-evidence-"); + const destination = join(directory, "trace-evidence.json"); + const delivered = await runJsonCli(cli, [ + "trace-native-values", + sourceTarget.path, + "main", + "--evidence_output", + destination, + "--json", + ]); + + expect(delivered.evidence_id).toBe(trace.evidence_id); + expect(run).toHaveBeenCalledOnce(); + const exported = parseEvidenceBundle( + JSON.parse(await readFile(destination, "utf8")), + ); + expect(exported.records.map(({ evidence_id }) => evidence_id).sort()).toEqual( + [trace.evidence_id, procedure.evidence_id, leaf.evidence_id].sort(), + ); + expect( + exported.records.some( + ({ evidence_id }) => evidence_id === unrelated.evidence_id, + ), + ).toBe(false); + + const summaryOutput = await runJsonCli(cli, [ + "trace-native-values", + "/missing-target", + "main", + "--detail", + "summary", + "--json", + ]); + expect(run).toHaveBeenCalledOnce(); + expect(JSON.stringify(summaryOutput)).toContain("evidence_output"); +}); + +it("publishes a bounded trace summary with a complete evidence bundle", async () => { + const { leaf, procedure, trace, sessionBundle } = createTraceFixture(); + const run = vi.fn( + async (_path, _tool, _arguments, options) => { + options?.onEvidenceBundle?.(sessionBundle); + return ok(trace); + }, + ); + const cli = createTraceCli(run); + const directory = await createTestTempDirectory("rea-trace-summary-"); + const destination = join(directory, "trace-summary-evidence.json"); + const receipt = await runJsonCli(cli, [ + "trace-native-values", + sourceTarget.path, + "main", + "--detail", + "summary", + "--evidence_output", + destination, + "--json", + ]); + + const summaryResult = ( + receipt.summary as { normalized_result?: unknown } | undefined + )?.normalized_result; + const expectedSummary = { + kind: "summary", + parent_operation: "trace_native_values", + parent_evidence_id: trace.evidence_id, + summary: { + seed: "main", + node_count: 1, + total_nodes: 2, + edge_count: 0, + procedure_count: 1, + unknown_count: 1, + decompilations: 1, + next_offset: 1, + page_truncated: true, + traversal: { status: "complete" }, + truncated: true, + limitation_count: 1, + }, + }; + expect(summaryResult).toMatchObject(expectedSummary); + const summaryEvidence = parseEvidence(receipt.summary); + expect(summaryEvidence.operation).toBe("inspect_analysis_view"); + const bundle = parseEvidenceBundle( + JSON.parse(await readFile(destination, "utf8")), + ); + expect(bundle.records.map(({ evidence_id }) => evidence_id).sort()).toEqual( + [trace.evidence_id, procedure.evidence_id, leaf.evidence_id].sort(), + ); + expect(run).toHaveBeenCalledOnce(); +}); + +it("retains completed trace Evidence when the session returns no bundle", async () => { + const { trace } = createTraceFixture(); + const run = vi.fn(async () => ok(trace)); + const cli = createTraceCli(run); + const directory = await createTestTempDirectory("rea-trace-no-bundle-"); + const destination = join(directory, "missing-bundle.json"); + const error = await runJsonCli(cli, [ + "trace-native-values", + sourceTarget.path, + "main", + "--evidence_output", + destination, + "--json", + ]); + + expect(error).toMatchObject({ + code: "unreadable_output", + details: { + operation: "trace_native_values", + partial_observation: { + operation: "trace_native_values", + normalized_result: { seed: "main" }, + }, + }, + }); + await expect(readFile(destination, "utf8")).rejects.toThrow(); + expect(run).toHaveBeenCalledOnce(); +}); + +it("reports a missing linked source with its identity and completed trace", async () => { + const { leaf, procedure, trace } = createTraceFixture(); + const run = vi.fn( + async (_path, _tool, _arguments, options) => { + options?.onEvidenceBundle?.(createEvidenceBundle([trace, procedure])); + return ok(trace); + }, + ); + const cli = createTraceCli(run); + const directory = await createTestTempDirectory("rea-trace-missing-source-"); + const destination = join(directory, "missing-source.json"); + const error = await runJsonCli(cli, [ + "trace-native-values", + sourceTarget.path, + "main", + "--evidence_output", + destination, + "--json", + ]); + + expect(error).toMatchObject({ + code: "evidence_integrity_mismatch", + details: { + evidence_id: leaf.evidence_id, + reason: "missing", + operation: "trace_native_values", + reference_scope: "retained-closure", + partial_observation: { + operation: "trace_native_values", + normalized_result: { seed: "main" }, + }, + }, + }); + await expect(readFile(destination, "utf8")).rejects.toThrow(); + expect(run).toHaveBeenCalledOnce(); +}); + +it("replays direct-analysis Evidence bundle callbacks without rerunning", async () => { + const directory = await createTestTempDirectory("rea-trace-callback-"); + await verifyDirectAnalysisEvidenceBundleCallback( + join(directory, "analysis-snapshot.json"), + ); +}); From 85be0f70a69a3a0ae7ea3a965f9c3621ebdb4b9e Mon Sep 17 00:00:00 2001 From: Dhruvilkumar Ashokbhai Chodvadiya <50516925+dhruvac29@users.noreply.github.com> Date: Wed, 4 Nov 2026 03:58:06 -0800 Subject: [PATCH 63/94] fix(artifacts): guide retries for symlinked files (#8725) * fix(go): suggest resolved path for rejected symlinks * test(go): cover unresolved symlink recovery paths * fix(go): omit schema recovery for resolved symlinks * fix(artifacts): share resolved-path recovery across inspectors * fix(sqlite): retain resolved database path through snapshot reads * fix(captures): retain symlink retry guidance * fix(artifacts): preserve retry hints through redaction and selected reads --------- Co-authored-by: dhruvac29 --- src/android/AndroidMappingFile.ts | 2 + src/apkSignature/apkSignatureFailures.ts | 9 ++- src/artifacts/ArtifactReader.ts | 5 ++ .../ArtifactSourceResolution.test.ts | 32 +++++++++ src/artifacts/StableArtifactFailure.ts | 9 +++ src/artifacts/readStableArtifact.ts | 23 ++++++- src/browser/assets/WebArtifactReads.ts | 14 +++- src/browser/history/CaptureFailures.test.ts | 32 +++++++++ src/browser/history/CaptureFailures.ts | 45 +++++++------ src/domain/explicitSensitiveFailure.test.ts | 17 +++++ src/domain/explicitSensitiveFailure.ts | 11 +++- src/erlang/BeamProvider.ts | 19 ++++-- src/evm/EvmoleFailures.ts | 2 + src/firmware/devicetree/DevicetreeProvider.ts | 19 ++++-- src/go/GoBinaryProvider.ts | 9 ++- src/il2cpp/Il2cppMetadataProvider.ts | 9 ++- src/native/apple/AppleIpsProvider.ts | 9 ++- src/native/elf/ElfNotesProvider.ts | 19 ++++-- src/native/elf/KernelModuleProvider.ts | 19 ++++-- src/native/exports/NativeExportsProvider.ts | 9 ++- src/native/pe/PeCertificatesProvider.ts | 19 ++++-- src/native/pe/PeImportsProvider.ts | 9 ++- src/native/pe/PeLoadConfigProvider.ts | 9 ++- src/native/pe/PeResourcesProvider.ts | 19 ++++-- src/native/pwntools/PwntoolsFailures.ts | 2 + src/process/ArtifactAnalysisResources.test.ts | 38 ++++++++++- src/process/ArtifactAnalysisResources.ts | 14 +++- src/python/PythonPycProvider.ts | 19 ++++-- src/rust/RustBinaryProvider.ts | 9 ++- src/sqlite/SqliteDatabaseFailures.ts | 19 ++++-- src/sqlite/SqliteDatabaseSnapshot.ts | 20 ++++-- src/wasm/WabtFailures.ts | 9 ++- tests/boundary/go/goBinaryProvider.test.ts | 7 +- tests/boundary/mcp/goBinaryMcp.test.ts | 65 ++++++++++++++++++- .../stableArtifactSymlinkRecoveryMcp.test.ts | 56 ++++++++++++++++ .../sqlite/sqliteDatabaseSnapshot.test.ts | 14 ++++ 36 files changed, 525 insertions(+), 116 deletions(-) create mode 100644 tests/boundary/mcp/stableArtifactSymlinkRecoveryMcp.test.ts diff --git a/src/android/AndroidMappingFile.ts b/src/android/AndroidMappingFile.ts index 46577f1bbc..a0f5b4cc72 100644 --- a/src/android/AndroidMappingFile.ts +++ b/src/android/AndroidMappingFile.ts @@ -4,6 +4,7 @@ import { bindArtifactFileCleanup, } from "../artifacts/ArtifactFileOperation.js"; import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import { stableArtifactInputRecovery } from "../artifacts/StableArtifactFailure.js"; import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { readStableArtifact } from "../artifacts/readStableArtifact.js"; import { @@ -99,6 +100,7 @@ const readFailurePrimary = ( if (owned?.reason === "path") return invalidMapping(read, "invalid_value", owned.message, { cause: owned, + ...stableArtifactInputRecovery(owned), }); if (primary instanceof Error && "code" in primary) { if (primary.code === "EACCES" || primary.code === "EPERM") diff --git a/src/apkSignature/apkSignatureFailures.ts b/src/apkSignature/apkSignatureFailures.ts index 659506ecc9..d63a2942b2 100644 --- a/src/apkSignature/apkSignatureFailures.ts +++ b/src/apkSignature/apkSignatureFailures.ts @@ -1,5 +1,6 @@ import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import { stableArtifactInputRecovery } from "../artifacts/StableArtifactFailure.js"; import { artifactFilePrimaryFacts, type ArtifactFilePrimaryFacts, @@ -230,9 +231,11 @@ const artifactReaderFailure = ( }, ); case "path": - return new AnalysisInputError(OPERATION, { cause }, [ - { path: ["path"], reason: "invalid_format", message: cause.message }, - ]); + return new AnalysisInputError( + OPERATION, + { cause, ...stableArtifactInputRecovery(cause) }, + [{ path: ["path"], reason: "invalid_format", message: cause.message }], + ); default: return adapterFailure(cause, path); } diff --git a/src/artifacts/ArtifactReader.ts b/src/artifacts/ArtifactReader.ts index e2fc1b6219..8709121b3d 100644 --- a/src/artifacts/ArtifactReader.ts +++ b/src/artifacts/ArtifactReader.ts @@ -26,6 +26,8 @@ import type { export interface ArtifactReaderFailureOptions extends ErrorOptions { /** Recovery guidance established by the producing adapter, separate from diagnostics. */ readonly userMessage?: string | undefined; + /** Regular-file path observed behind a rejected selected symlink; retry still revalidates it. */ + readonly resolvedPath?: string | undefined; readonly cancellation?: { readonly reason: unknown } | undefined; readonly cleanup?: AnalysisCleanupObservation | undefined; /** Exact authority for all reported cleanup; undefined leaves ownership unknown. */ @@ -230,6 +232,7 @@ export interface ArtifactReader { /** Typed adapter failure translated at provider boundary. */ export class ArtifactReaderFailure extends Error { readonly userMessage: string | undefined; + readonly resolvedPath: string | undefined; readonly cancellation: { readonly reason: unknown } | undefined; readonly cleanup: AnalysisCleanupObservation | undefined; readonly cleanupLeases: readonly AnalysisCleanupLease[] | undefined; @@ -255,6 +258,7 @@ export class ArtifactReaderFailure extends Error { super(message, options); this.name = "ArtifactReaderFailure"; this.userMessage = options?.userMessage; + this.resolvedPath = options?.resolvedPath; this.cancellation = reason === "cancelled" ? (options?.cancellation ?? { reason: options?.cause }) @@ -306,6 +310,7 @@ export class ArtifactReaderFailure extends Error { { cause: primary, userMessage: primary.userMessage, + resolvedPath: primary.resolvedPath, cancellation: primary.cancellation, cleanup: combineCleanup(primary.cleanup, cleanup), cleanupLeases: diff --git a/src/artifacts/ArtifactSourceResolution.test.ts b/src/artifacts/ArtifactSourceResolution.test.ts index 7101a31382..3a87e57f0b 100644 --- a/src/artifacts/ArtifactSourceResolution.test.ts +++ b/src/artifacts/ArtifactSourceResolution.test.ts @@ -156,3 +156,35 @@ it.each(["ENOENT", "ENOTDIR", "ELOOP", "ENAMETOOLONG", "EACCES", "EIO"])( }); }, ); + +it.skipIf(process.platform === "win32")( + "keeps a rejected symlink without a hint when its resolved target lookup fails", + async () => { + const root = await createTestTempDirectory("rea-stable-symlink-lookup-"); + const target = join(root, "target.bin"); + const selected = join(root, "selected"); + await writeFile(target, "carrier"); + await symlink(target, selected); + const selectedStats = await lstat(selected, { bigint: true }); + const failure = Object.assign(new Error("target lookup failed"), { + code: "EACCES", + }); + + await expect( + readStableArtifact(selected, 1024, undefined, { + lstat: async (path) => { + if (path === selected) return selectedStats; + throw failure; + }, + realpath: async () => target, + open: async () => { + throw new Error("A rejected symlink must not be opened"); + }, + }), + ).rejects.toMatchObject({ + reason: "path", + resolvedPath: undefined, + message: `Expected a regular file without a symlink: ${selected}`, + }); + }, +); diff --git a/src/artifacts/StableArtifactFailure.ts b/src/artifacts/StableArtifactFailure.ts index 0c4fb62751..4d326b2927 100644 --- a/src/artifacts/StableArtifactFailure.ts +++ b/src/artifacts/StableArtifactFailure.ts @@ -5,6 +5,15 @@ import { ArtifactReaderFailure } from "./ArtifactReader.js"; import { bindArtifactFileCleanup } from "./ArtifactFileOperation.js"; import { completedStableArtifactObservation } from "./StableArtifactObservation.js"; +/** Offer the verified regular-file target after a selected symlink is rejected. */ +export const stableArtifactInputRecovery = (cause: ArtifactReaderFailure) => + cause.reason === "path" && cause.resolvedPath !== undefined + ? { + remediationAction: `Retry with the resolved regular file path ${cause.resolvedPath}. The selected symlink is not admitted; the retry will validate the target again.`, + conformsToInputContract: true, + } + : {}; + const artifactCleanupFailure = ( cause: unknown, providerId: string, diff --git a/src/artifacts/readStableArtifact.ts b/src/artifacts/readStableArtifact.ts index ad3371d3f7..ec20830575 100644 --- a/src/artifacts/readStableArtifact.ts +++ b/src/artifacts/readStableArtifact.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { constants, type BigIntStats } from "node:fs"; -import { lstat, open, type FileHandle } from "node:fs/promises"; +import { lstat, open, realpath, type FileHandle } from "node:fs/promises"; import { sameRegularFileState } from "../filesystem/RegularFile.js"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; import type { ArtifactResourceScope } from "./ArtifactResourceScope.js"; @@ -38,11 +38,29 @@ export const completedStableArtifact = ( export interface StableArtifactFileSystem { lstat(path: string): Promise; open(path: string, flags: number): Promise; + realpath?(path: string): Promise; } const FILE_SYSTEM: StableArtifactFileSystem = { lstat: (path) => lstat(path, { bigint: true }), open, + realpath, +}; + +/** Suggest a retry only when the rejected link currently resolves to a regular file. */ +const resolvedRegularFile = async ( + path: string, + fileSystem: StableArtifactFileSystem, +): Promise => { + try { + const resolved = await fileSystem.realpath?.(path); + if (resolved === undefined) return undefined; + const target = await fileSystem.lstat(resolved); + return target.isFile() && !target.isSymbolicLink() ? resolved : undefined; + } catch { + // A missing or unreadable target changes only the recovery hint. + return undefined; + } }; const afterValidation = async ( @@ -77,6 +95,9 @@ export const readStableArtifact = async ( throw new ArtifactReaderFailure( "path", `Expected a regular file without a symlink: ${path}`, + before.isSymbolicLink() + ? { resolvedPath: await resolvedRegularFile(path, fileSystem) } + : {}, ); const maximumSize = BigInt(maximumBytes); if (before.size > maximumSize) diff --git a/src/browser/assets/WebArtifactReads.ts b/src/browser/assets/WebArtifactReads.ts index 009fe9e901..a0b469160e 100644 --- a/src/browser/assets/WebArtifactReads.ts +++ b/src/browser/assets/WebArtifactReads.ts @@ -1,6 +1,7 @@ import type { WebArtifactAcquisition as Acquisition } from "../../domain/webArtifactAcquisition.js"; import { z } from "zod"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; +import { stableArtifactInputRecovery } from "../../artifacts/StableArtifactFailure.js"; import { artifactFileCancellationCause, artifactFilePrimaryFacts, @@ -250,9 +251,16 @@ const selectedFileError = ( context: WebArtifactReadContext, ): AnalysisError | undefined => { const selected = (message: string) => - new AnalysisInputError(context.operation, { cause }, [ - { path: context.field, reason: "invalid_value", message }, - ]); + new AnalysisInputError( + context.operation, + { + cause, + ...(cause instanceof ArtifactReaderFailure + ? stableArtifactInputRecovery(cause) + : {}), + }, + [{ path: context.field, reason: "invalid_value", message }], + ); if ( origin === "owned-or-classified" && cause instanceof ArtifactReaderFailure diff --git a/src/browser/history/CaptureFailures.test.ts b/src/browser/history/CaptureFailures.test.ts index 260c630a1e..72eb16f407 100644 --- a/src/browser/history/CaptureFailures.test.ts +++ b/src/browser/history/CaptureFailures.test.ts @@ -12,6 +12,38 @@ import { NODE_CAPTURE_PROCESS_MEMORY_LIMIT_BYTES } from "./CaptureRelease.js"; import { redactExplicitFailure } from "../../domain/explicitSensitiveFailure.js"; import { OwnedCommandFailure } from "../../process/OwnedCommand.js"; +it.each([ + { phase: "capture-read", field: ["capture_path"] }, + { + phase: "descriptor-read", + field: ["protobuf", "schema", "descriptor_set_path"], + }, +] as const)("offers the resolved-file retry for $phase", ({ phase, field }) => { + const input = inspectWebNetworkCaptureInputSchema.parse({ + capture_path: "/selected/capture.har", + format: "har", + protobuf: { + schema: { + descriptor_set_path: "/selected/schema.pb", + message_type: "capture.Message", + }, + }, + }); + const resolvedPath = "/resolved/regular-file"; + const cause = new ArtifactReaderFailure("path", "Selected a symlink", { + resolvedPath, + }); + const projected = projectAnalysisError( + historicalCaptureFailure(input, cause, phase), + ); + expect(projected).toMatchObject({ + code: "invalid_request", + details: { issues: [{ path: field }] }, + remediation: { action: expect.stringContaining(resolvedPath) }, + }); + expect(projected).not.toHaveProperty("recovery"); +}); + it.each([ { label: "POSIX SIGABRT", exitCode: null, signal: "SIGABRT" }, { label: "Windows exit 134", exitCode: 134, signal: null }, diff --git a/src/browser/history/CaptureFailures.ts b/src/browser/history/CaptureFailures.ts index 0b51e4780d..cd02548a55 100644 --- a/src/browser/history/CaptureFailures.ts +++ b/src/browser/history/CaptureFailures.ts @@ -1,6 +1,7 @@ import { analysisSettingLimit } from "../../domain/analysisSettingDefinitions.js"; import type { ExecutionOptions } from "../../application/AnalysisProvider.js"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; +import { stableArtifactInputRecovery } from "../../artifacts/StableArtifactFailure.js"; import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorCleanup.js"; import { bindArtifactFileCleanup, @@ -168,26 +169,30 @@ export const historicalCaptureFailure = ( cause.reason === "format" || cause.reason === "path") ) - return new AnalysisInputError(OPERATION, { cause }, [ - { - path: inputPath, - reason: cause.reason === "limit" ? "out_of_range" : "invalid_format", - message: - cause.reason === "limit" - ? `${cause.message}. Select a ${sourceLabel} within the ${String(WEB_NETWORK_CAPTURE_LIMITS.inputBytes)}-byte input budget. This release has no runtime override. capture_timeout_ms does not raise it.` - : cause.message, - ...(cause.reason === "limit" - ? { - expected: { - [descriptorRead - ? "maximum_descriptor_set_bytes" - : "maximum_capture_bytes"]: - WEB_NETWORK_CAPTURE_LIMITS.inputBytes, - }, - } - : {}), - }, - ]); + return new AnalysisInputError( + OPERATION, + { cause, ...stableArtifactInputRecovery(cause) }, + [ + { + path: inputPath, + reason: cause.reason === "limit" ? "out_of_range" : "invalid_format", + message: + cause.reason === "limit" + ? `${cause.message}. Select a ${sourceLabel} within the ${String(WEB_NETWORK_CAPTURE_LIMITS.inputBytes)}-byte input budget. This release has no runtime override. capture_timeout_ms does not raise it.` + : cause.message, + ...(cause.reason === "limit" + ? { + expected: { + [descriptorRead + ? "maximum_descriptor_set_bytes" + : "maximum_capture_bytes"]: + WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + }, + } + : {}), + }, + ], + ); if ( phase !== "decoder" && cause instanceof Error && diff --git a/src/domain/explicitSensitiveFailure.test.ts b/src/domain/explicitSensitiveFailure.test.ts index c02093672e..3a1d81b1f7 100644 --- a/src/domain/explicitSensitiveFailure.test.ts +++ b/src/domain/explicitSensitiveFailure.test.ts @@ -278,6 +278,23 @@ it("coarsens ordinary sensitive issue-path segments to their real parent", () => expect(JSON.stringify(projectAnalysisError(result))).not.toContain("secret"); }); +it("retains input retry guidance while redacting marked text", () => { + const error = new AnalysisInputError( + "inspect_web_network_capture", + { + userMessage: "Selected private-marker capture is a symlink", + remediationAction: "Retry /resolved/private-marker/capture.har", + conformsToInputContract: true, + }, + [{ path: ["capture_path"], reason: "invalid_format" }], + ); + const redacted = redactExplicitFailure(error, ["private-marker"]); + const projected = projectAnalysisError(redacted); + expect(projected.remediation.action).toContain("/resolved/"); + expect(projected).not.toHaveProperty("recovery"); + expect(JSON.stringify(projected)).not.toContain("private-marker"); +}); + it("does not reinterpret an unknown argument name as a JSON pointer", () => { const input = new AnalysisInputError( "inspect_web_network_capture", diff --git a/src/domain/explicitSensitiveFailure.ts b/src/domain/explicitSensitiveFailure.ts index e55be448a7..a7e4e5f226 100644 --- a/src/domain/explicitSensitiveFailure.ts +++ b/src/domain/explicitSensitiveFailure.ts @@ -258,7 +258,16 @@ const redactFailure = ( if (error instanceof AnalysisInputError) return new AnalysisInputError( error.operation, - context, + { + ...context, + ...(error.userMessage === undefined + ? {} + : { userMessage: text(error.userMessage) }), + ...(error.remediationAction === undefined + ? {} + : { remediationAction: text(error.remediationAction) }), + conformsToInputContract: error.conformsToInputContract, + }, error.issues.map(issueProjection), ); if (error instanceof AnalysisOutputError) diff --git a/src/erlang/BeamProvider.ts b/src/erlang/BeamProvider.ts index bbee47ca82..a68d45b004 100644 --- a/src/erlang/BeamProvider.ts +++ b/src/erlang/BeamProvider.ts @@ -1,6 +1,7 @@ import { ERLANG_BEAM_PROVIDER_IDENTITY } from "./ErlangBeamProviderIdentity.js"; import { projectStableArtifactFailure, + stableArtifactInputRecovery, projectArtifactCloseFailure, } from "../artifacts/StableArtifactFailure.js"; import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; @@ -154,13 +155,17 @@ const beamFailurePrimary = ( ); } if (cause.reason === "path" || cause.reason === "format") - return new AnalysisInputError(OPERATION, { cause }, [ - { - path: ["path"], - reason: "invalid_format", - message: cause.message, - }, - ]); + return new AnalysisInputError( + OPERATION, + { cause, ...stableArtifactInputRecovery(cause) }, + [ + { + path: ["path"], + reason: "invalid_format", + message: cause.message, + }, + ], + ); } if (cause instanceof Error && "code" in cause) { if (cause.code === "EACCES" || cause.code === "EPERM") diff --git a/src/evm/EvmoleFailures.ts b/src/evm/EvmoleFailures.ts index 3ae76c1b21..bfa320f067 100644 --- a/src/evm/EvmoleFailures.ts +++ b/src/evm/EvmoleFailures.ts @@ -1,5 +1,6 @@ import type { EvmWorkerLimits } from "./EvmWorkerLimits.js"; import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import { stableArtifactInputRecovery } from "../artifacts/StableArtifactFailure.js"; import { artifactFileCancellationCause, artifactSelectedPathErrno, @@ -218,6 +219,7 @@ export const evmInterfaceFailure = ( { cause, ...cleanupOptions, + ...stableArtifactInputRecovery(cause), }, [ { diff --git a/src/firmware/devicetree/DevicetreeProvider.ts b/src/firmware/devicetree/DevicetreeProvider.ts index 848f9bec51..26ba90e132 100644 --- a/src/firmware/devicetree/DevicetreeProvider.ts +++ b/src/firmware/devicetree/DevicetreeProvider.ts @@ -1,5 +1,6 @@ import { projectStableArtifactFailure, + stableArtifactInputRecovery, projectArtifactCloseFailure, } from "../../artifacts/StableArtifactFailure.js"; import { isSelectedRegularFileError } from "../../filesystem/RegularFile.js"; @@ -153,13 +154,17 @@ const devicetreeFailurePrimary = ( } if (cause.reason === "path" || cause.reason === "format") - return new AnalysisInputError(OPERATION, { cause }, [ - { - path: ["path"], - reason: "invalid_format", - message: cause.message, - }, - ]); + return new AnalysisInputError( + OPERATION, + { cause, ...stableArtifactInputRecovery(cause) }, + [ + { + path: ["path"], + reason: "invalid_format", + message: cause.message, + }, + ], + ); } if (cause instanceof Error && "code" in cause) { if (cause.code === "EACCES" || cause.code === "EPERM") diff --git a/src/go/GoBinaryProvider.ts b/src/go/GoBinaryProvider.ts index 5f59bf7f75..15c5413a15 100644 --- a/src/go/GoBinaryProvider.ts +++ b/src/go/GoBinaryProvider.ts @@ -2,6 +2,7 @@ import { GO_BINARY_PROVIDER_IDENTITY } from "./GoBinaryProviderIdentity.js"; import { projectStableArtifactFailure, projectArtifactCloseFailure, + stableArtifactInputRecovery, } from "../artifacts/StableArtifactFailure.js"; import { isSelectedRegularFileError } from "../filesystem/RegularFile.js"; import { readStableArtifact } from "../artifacts/readStableArtifact.js"; @@ -113,9 +114,11 @@ const failureForPrimary = ( }, ); if (cause.reason === "path") - return new AnalysisInputError(OPERATION, { cause }, [ - { path: ["path"], reason: "invalid_format", message: cause.message }, - ]); + return new AnalysisInputError( + OPERATION, + { cause, ...stableArtifactInputRecovery(cause) }, + [{ path: ["path"], reason: "invalid_format", message: cause.message }], + ); } if (cause instanceof Error && "code" in cause) { if (cause.code === "EACCES" || cause.code === "EPERM") diff --git a/src/il2cpp/Il2cppMetadataProvider.ts b/src/il2cpp/Il2cppMetadataProvider.ts index 0cdd5b8e2e..996e0a480b 100644 --- a/src/il2cpp/Il2cppMetadataProvider.ts +++ b/src/il2cpp/Il2cppMetadataProvider.ts @@ -11,6 +11,7 @@ import { } from "../artifacts/readStableArtifact.js"; import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import { stableArtifactInputRecovery } from "../artifacts/StableArtifactFailure.js"; import { artifactFilePrimaryFacts, type ArtifactFilePrimaryFacts, @@ -145,9 +146,11 @@ const failureForPrimary = ( }, ); if (cause.reason === "path") - return new AnalysisInputError(OPERATION, { cause }, [ - { path: [field], reason: "invalid_format", message: cause.message }, - ]); + return new AnalysisInputError( + OPERATION, + { cause, ...stableArtifactInputRecovery(cause) }, + [{ path: [field], reason: "invalid_format", message: cause.message }], + ); } if (cause instanceof Error && "code" in cause) { if (cause.code === "EACCES" || cause.code === "EPERM") diff --git a/src/native/apple/AppleIpsProvider.ts b/src/native/apple/AppleIpsProvider.ts index 26a1fc901f..913186c749 100644 --- a/src/native/apple/AppleIpsProvider.ts +++ b/src/native/apple/AppleIpsProvider.ts @@ -1,5 +1,6 @@ import { projectStableArtifactFailure, + stableArtifactInputRecovery, projectArtifactCloseFailure, } from "../../artifacts/StableArtifactFailure.js"; import { isSelectedRegularFileError } from "../../filesystem/RegularFile.js"; @@ -236,9 +237,11 @@ const appleIpsFailurePrimary = ( }, ); if (cause.reason === "path" || cause.reason === "format") - return new AnalysisInputError(OPERATION, { cause }, [ - { path: ["path"], reason: "invalid_format", message: cause.message }, - ]); + return new AnalysisInputError( + OPERATION, + { cause, ...stableArtifactInputRecovery(cause) }, + [{ path: ["path"], reason: "invalid_format", message: cause.message }], + ); } if (cause instanceof Error && "code" in cause) { if (cause.code === "EACCES" || cause.code === "EPERM") diff --git a/src/native/elf/ElfNotesProvider.ts b/src/native/elf/ElfNotesProvider.ts index 9efe2d648e..e96de9954f 100644 --- a/src/native/elf/ElfNotesProvider.ts +++ b/src/native/elf/ElfNotesProvider.ts @@ -1,5 +1,6 @@ import { projectStableArtifactFailure, + stableArtifactInputRecovery, projectArtifactCloseFailure, } from "../../artifacts/StableArtifactFailure.js"; import { isSelectedRegularFileError } from "../../filesystem/RegularFile.js"; @@ -195,13 +196,17 @@ const elfNotesFailurePrimary = ( ); } if (cause.reason === "path" || cause.reason === "format") - return new AnalysisInputError(OPERATION, { cause }, [ - { - path: ["path"], - reason: "invalid_format", - message: cause.message, - }, - ]); + return new AnalysisInputError( + OPERATION, + { cause, ...stableArtifactInputRecovery(cause) }, + [ + { + path: ["path"], + reason: "invalid_format", + message: cause.message, + }, + ], + ); } if (cause instanceof Error && "code" in cause) { if (cause.code === "EACCES" || cause.code === "EPERM") diff --git a/src/native/elf/KernelModuleProvider.ts b/src/native/elf/KernelModuleProvider.ts index d1a4d4113e..1c83e695cb 100644 --- a/src/native/elf/KernelModuleProvider.ts +++ b/src/native/elf/KernelModuleProvider.ts @@ -1,5 +1,6 @@ import { projectStableArtifactFailure, + stableArtifactInputRecovery, projectArtifactCloseFailure, } from "../../artifacts/StableArtifactFailure.js"; import { isSelectedRegularFileError } from "../../filesystem/RegularFile.js"; @@ -187,13 +188,17 @@ const kernelModuleFailurePrimary = ( ); } if (cause.reason === "path" || cause.reason === "format") - return new AnalysisInputError(OPERATION, { cause }, [ - { - path: ["path"], - reason: "invalid_format", - message: cause.message, - }, - ]); + return new AnalysisInputError( + OPERATION, + { cause, ...stableArtifactInputRecovery(cause) }, + [ + { + path: ["path"], + reason: "invalid_format", + message: cause.message, + }, + ], + ); } if (cause instanceof Error && "code" in cause) { if (cause.code === "EACCES" || cause.code === "EPERM") diff --git a/src/native/exports/NativeExportsProvider.ts b/src/native/exports/NativeExportsProvider.ts index e5682bc369..d564ed8826 100644 --- a/src/native/exports/NativeExportsProvider.ts +++ b/src/native/exports/NativeExportsProvider.ts @@ -1,5 +1,6 @@ import { projectStableArtifactFailure, + stableArtifactInputRecovery, projectArtifactCloseFailure, } from "../../artifacts/StableArtifactFailure.js"; import { isSelectedRegularFileError } from "../../filesystem/RegularFile.js"; @@ -160,9 +161,11 @@ const nativeExportsFailurePrimary = ( }, ); if (cause.reason === "path" || cause.reason === "format") - return new AnalysisInputError(OPERATION, { cause }, [ - { path: ["path"], reason: "invalid_format", message: cause.message }, - ]); + return new AnalysisInputError( + OPERATION, + { cause, ...stableArtifactInputRecovery(cause) }, + [{ path: ["path"], reason: "invalid_format", message: cause.message }], + ); } if (cause instanceof Error && "code" in cause) { if (cause.code === "EACCES" || cause.code === "EPERM") diff --git a/src/native/pe/PeCertificatesProvider.ts b/src/native/pe/PeCertificatesProvider.ts index 47c95deb42..26c0d9a2ca 100644 --- a/src/native/pe/PeCertificatesProvider.ts +++ b/src/native/pe/PeCertificatesProvider.ts @@ -1,5 +1,6 @@ import { projectStableArtifactFailure, + stableArtifactInputRecovery, projectArtifactCloseFailure, } from "../../artifacts/StableArtifactFailure.js"; import { isSelectedRegularFileError } from "../../filesystem/RegularFile.js"; @@ -176,13 +177,17 @@ const peCertificatesFailurePrimary = ( }, ); if (cause.reason === "path" || cause.reason === "format") - return new AnalysisInputError(OPERATION, { cause }, [ - { - path: ["path"], - reason: "invalid_format", - message: cause.message, - }, - ]); + return new AnalysisInputError( + OPERATION, + { cause, ...stableArtifactInputRecovery(cause) }, + [ + { + path: ["path"], + reason: "invalid_format", + message: cause.message, + }, + ], + ); } if (cause instanceof Error && "code" in cause) { if (cause.code === "EACCES" || cause.code === "EPERM") diff --git a/src/native/pe/PeImportsProvider.ts b/src/native/pe/PeImportsProvider.ts index 2e4cf5d452..7324ff68e6 100644 --- a/src/native/pe/PeImportsProvider.ts +++ b/src/native/pe/PeImportsProvider.ts @@ -1,5 +1,6 @@ import { projectStableArtifactFailure, + stableArtifactInputRecovery, projectArtifactCloseFailure, } from "../../artifacts/StableArtifactFailure.js"; import { isSelectedRegularFileError } from "../../filesystem/RegularFile.js"; @@ -203,9 +204,11 @@ const peImportsFailurePrimary = ( }, ); if (cause.reason === "path" || cause.reason === "format") - return new AnalysisInputError(OPERATION, { cause }, [ - { path: ["path"], reason: "invalid_format", message: cause.message }, - ]); + return new AnalysisInputError( + OPERATION, + { cause, ...stableArtifactInputRecovery(cause) }, + [{ path: ["path"], reason: "invalid_format", message: cause.message }], + ); } if (cause instanceof Error && "code" in cause) { if (cause.code === "EACCES" || cause.code === "EPERM") diff --git a/src/native/pe/PeLoadConfigProvider.ts b/src/native/pe/PeLoadConfigProvider.ts index 7087a7e0ef..9131c763d0 100644 --- a/src/native/pe/PeLoadConfigProvider.ts +++ b/src/native/pe/PeLoadConfigProvider.ts @@ -1,5 +1,6 @@ import { projectStableArtifactFailure, + stableArtifactInputRecovery, projectArtifactCloseFailure, } from "../../artifacts/StableArtifactFailure.js"; import { isSelectedRegularFileError } from "../../filesystem/RegularFile.js"; @@ -180,9 +181,11 @@ const peLoadConfigFailurePrimary = ( }, ); if (cause.reason === "path" || cause.reason === "format") - return new AnalysisInputError(OPERATION, { cause }, [ - { path: ["path"], reason: "invalid_format", message: cause.message }, - ]); + return new AnalysisInputError( + OPERATION, + { cause, ...stableArtifactInputRecovery(cause) }, + [{ path: ["path"], reason: "invalid_format", message: cause.message }], + ); } if (cause instanceof Error && "code" in cause) { if (cause.code === "EACCES" || cause.code === "EPERM") diff --git a/src/native/pe/PeResourcesProvider.ts b/src/native/pe/PeResourcesProvider.ts index 8bb56154c3..291adb7d51 100644 --- a/src/native/pe/PeResourcesProvider.ts +++ b/src/native/pe/PeResourcesProvider.ts @@ -1,5 +1,6 @@ import { projectStableArtifactFailure, + stableArtifactInputRecovery, projectArtifactCloseFailure, } from "../../artifacts/StableArtifactFailure.js"; import { isSelectedRegularFileError } from "../../filesystem/RegularFile.js"; @@ -165,13 +166,17 @@ const peResourcesFailurePrimary = ( ); } if (cause.reason === "path" || cause.reason === "format") - return new AnalysisInputError(operation, { cause }, [ - { - path: ["path"], - reason: "invalid_format", - message: cause.message, - }, - ]); + return new AnalysisInputError( + operation, + { cause, ...stableArtifactInputRecovery(cause) }, + [ + { + path: ["path"], + reason: "invalid_format", + message: cause.message, + }, + ], + ); } if (cause instanceof Error && "code" in cause) { diff --git a/src/native/pwntools/PwntoolsFailures.ts b/src/native/pwntools/PwntoolsFailures.ts index 10da1660c5..721a4869a3 100644 --- a/src/native/pwntools/PwntoolsFailures.ts +++ b/src/native/pwntools/PwntoolsFailures.ts @@ -3,6 +3,7 @@ import { bindArtifactFileCleanup, } from "../../artifacts/ArtifactFileOperation.js"; import { completedStableArtifactObservation } from "../../artifacts/StableArtifactObservation.js"; +import { stableArtifactInputRecovery } from "../../artifacts/StableArtifactFailure.js"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; import { AnalysisError, @@ -274,6 +275,7 @@ const classifyPwntoolsDecoderFailure = ( context.operation, { cause, + ...stableArtifactInputRecovery(cause), ...(cause.reason === "limit" ? { userMessage: `${cause.message} Admitted artifact bytes use REA_PWNTOOLS_MAX_INPUT_BYTES (default ${String(PWNTOOLS_LIMITS.inputBytes)} bytes, maximum 1073741824). The 1073741824-byte maximum has no runtime override. Restart REA after changing the environment value. pwntools_timeout_ms does not raise this allowance. ${fixedPwntoolsBudgets}`, diff --git a/src/process/ArtifactAnalysisResources.test.ts b/src/process/ArtifactAnalysisResources.test.ts index 421621f7d1..b48bc8e023 100644 --- a/src/process/ArtifactAnalysisResources.test.ts +++ b/src/process/ArtifactAnalysisResources.test.ts @@ -4,7 +4,7 @@ import { ArtifactResourceScope, } from "../artifacts/ArtifactResourceScope.js"; import { withArtifactRegularFile } from "../artifacts/ArtifactFileOperation.js"; -import { writeFile } from "node:fs/promises"; +import { symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js"; @@ -28,6 +28,42 @@ import { ArtifactAnalysisResources } from "./ArtifactAnalysisResources.js"; import { readLinuxResourceLimits } from "./LinuxResourceLimits.js"; import { ProviderProcessSupervisor } from "./ProviderProcess.js"; +it.runIf(process.platform !== "win32")( + "offers a resolved-file retry from the shared selected-artifact boundary", + async () => { + const root = await createTestTempDirectory("rea-selected-artifact-"); + const target = join(root, "target.bin"); + const selected = join(root, "selected"); + await writeFile(target, "fixture"); + await symlink(target, selected); + const resources = new ArtifactAnalysisResources( + "fixture-provider", + "decode_signal_recording", + readStableArtifact, + readLinuxResourceLimits, + { message: "busy", details: {} }, + ); + try { + const result = await resources.run(async () => + ok( + await resources.readSelectedArtifact({ + path: selected, + maximumBytes: 1024, + allowanceName: "fixture input allowance", + observedOutput: undefined, + }), + ), + ); + if (result.ok) throw new Error("Expected selected symlink refusal"); + const projected = projectAnalysisError(result.error); + expect(projected.remediation.action).toContain(target); + expect(projected).not.toHaveProperty("recovery"); + } finally { + await resources.close(); + } + }, +); + it.each(["command", "limiter"])( "attributes post-run %s read failure to its exact artifact and retains producer output", async (failedPath) => { diff --git a/src/process/ArtifactAnalysisResources.ts b/src/process/ArtifactAnalysisResources.ts index aebf66d041..30dc6ae23a 100644 --- a/src/process/ArtifactAnalysisResources.ts +++ b/src/process/ArtifactAnalysisResources.ts @@ -10,6 +10,7 @@ import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { readStableArtifact } from "../artifacts/readStableArtifact.js"; import { completedStableArtifactObservation } from "../artifacts/StableArtifactObservation.js"; +import { stableArtifactInputRecovery } from "../artifacts/StableArtifactFailure.js"; import { AnalysisError, type AnalysisCapturedOutput, @@ -221,9 +222,16 @@ export class ArtifactAnalysisResources { return new AnalysisAccessDeniedError(operation, path, cause.code, { cause, }); - return new AnalysisInputError(operation, { cause }, [ - { path: ["path"], reason: "invalid_value", message: reason }, - ]); + return new AnalysisInputError( + operation, + { + cause, + ...(cause instanceof ArtifactReaderFailure + ? stableArtifactInputRecovery(cause) + : {}), + }, + [{ path: ["path"], reason: "invalid_value", message: reason }], + ); } readLimits(limits: LinuxResourceLimits): Promise { diff --git a/src/python/PythonPycProvider.ts b/src/python/PythonPycProvider.ts index a95fc5cb6c..488e44f625 100644 --- a/src/python/PythonPycProvider.ts +++ b/src/python/PythonPycProvider.ts @@ -1,6 +1,7 @@ import { PYTHON_PYC_PROVIDER_IDENTITY } from "./PythonPycProviderIdentity.js"; import { projectStableArtifactFailure, + stableArtifactInputRecovery, projectArtifactCloseFailure, } from "../artifacts/StableArtifactFailure.js"; import { isSelectedRegularFileError } from "../filesystem/RegularFile.js"; @@ -126,13 +127,17 @@ const failureForPrimary = ( }, ); if (cause.reason === "path" || cause.reason === "format") - return new AnalysisInputError(OPERATION, { cause }, [ - { - path: ["path"], - reason: "invalid_format", - message: cause.message, - }, - ]); + return new AnalysisInputError( + OPERATION, + { cause, ...stableArtifactInputRecovery(cause) }, + [ + { + path: ["path"], + reason: "invalid_format", + message: cause.message, + }, + ], + ); } if (cause instanceof Error && "code" in cause) { if (cause.code === "EACCES" || cause.code === "EPERM") diff --git a/src/rust/RustBinaryProvider.ts b/src/rust/RustBinaryProvider.ts index 5a90462c0e..69d48477e3 100644 --- a/src/rust/RustBinaryProvider.ts +++ b/src/rust/RustBinaryProvider.ts @@ -1,6 +1,7 @@ import { RUST_BINARY_PROVIDER_IDENTITY } from "./RustBinaryProviderIdentity.js"; import { projectStableArtifactFailure, + stableArtifactInputRecovery, projectArtifactCloseFailure, } from "../artifacts/StableArtifactFailure.js"; import { isSelectedRegularFileError } from "../filesystem/RegularFile.js"; @@ -343,9 +344,11 @@ const artifactReaderFailure = ( }, ); if (cause.reason === "path") - return new AnalysisInputError(OPERATION, { cause }, [ - { path: ["path"], reason: "invalid_format", message: cause.message }, - ]); + return new AnalysisInputError( + OPERATION, + { cause, ...stableArtifactInputRecovery(cause) }, + [{ path: ["path"], reason: "invalid_format", message: cause.message }], + ); return new ProviderAdapterError(RUST_BINARY_PROVIDER_IDENTITY.id, OPERATION, { cause, diagnostics: { path, reason: cause.message }, diff --git a/src/sqlite/SqliteDatabaseFailures.ts b/src/sqlite/SqliteDatabaseFailures.ts index e049b61318..0bf8620e58 100644 --- a/src/sqlite/SqliteDatabaseFailures.ts +++ b/src/sqlite/SqliteDatabaseFailures.ts @@ -1,4 +1,5 @@ import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import { stableArtifactInputRecovery } from "../artifacts/StableArtifactFailure.js"; import { ArtifactFileCleanupFailure, bindArtifactFileCleanup, @@ -172,13 +173,17 @@ export const sqliteDatabaseFailure = ( return new AnalysisCancelledError(operation, { cause: artifactFileCancellationCause(cause), }); - return new AnalysisInputError(operation, { cause }, [ - { - path: ["path"], - reason: "invalid_format", - message: cause.message, - }, - ]); + return new AnalysisInputError( + operation, + { cause, ...stableArtifactInputRecovery(cause) }, + [ + { + path: ["path"], + reason: "invalid_format", + message: cause.message, + }, + ], + ); } if (cause instanceof Error && cause.name === "AbortError") return new AnalysisCancelledError(operation, { diff --git a/src/sqlite/SqliteDatabaseSnapshot.ts b/src/sqlite/SqliteDatabaseSnapshot.ts index 79daeec560..b36763829a 100644 --- a/src/sqlite/SqliteDatabaseSnapshot.ts +++ b/src/sqlite/SqliteDatabaseSnapshot.ts @@ -1,4 +1,4 @@ -import { lstat, open, writeFile } from "node:fs/promises"; +import { lstat, open, realpath, writeFile } from "node:fs/promises"; import { constants, type BigIntStats } from "node:fs"; import { join } from "node:path"; import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; @@ -24,6 +24,7 @@ import { SQLITE_DATABASE_LIMITS } from "./SqliteDatabaseLimits.js"; const fileSystem: StableArtifactFileSystem = { lstat: (path) => lstat(path, { bigint: true }), open, + realpath, }; type SnapshotMemberRole = "database" | "wal"; @@ -165,14 +166,17 @@ const duringAcquisition = async ( }; const readObservedArtifact = ( + role: SnapshotMemberRole, path: string, observed: BigIntStats, maximumBytes: number, selectedFileSystem: StableArtifactFileSystem, resourceScope: ArtifactResourceScope, signal?: AbortSignal, -) => - duringAcquisition(path, () => +) => { + const resolveSelectedPath = + role === "database" ? selectedFileSystem.realpath : undefined; + return duringAcquisition(path, () => readStableArtifact( path, maximumBytes, @@ -180,9 +184,15 @@ const readObservedArtifact = ( { open: (selectedPath, flags) => selectedFileSystem.open(selectedPath, flags), + ...(resolveSelectedPath === undefined + ? {} + : { + realpath: (selectedPath: string) => + resolveSelectedPath.call(selectedFileSystem, selectedPath), + }), lstat: async (selectedPath) => { const current = await selectedFileSystem.lstat(selectedPath); - if (!sameState(observed, current)) + if (selectedPath === path && !sameState(observed, current)) throw new ArtifactReaderFailure( "integrity", `SQLite snapshot file set changed during acquisition: ${selectedPath}`, @@ -193,6 +203,7 @@ const readObservedArtifact = ( resourceScope, ), ); +}; const verifyInactiveJournal = async ( path: string, @@ -301,6 +312,7 @@ export const captureSqliteDatabaseSnapshot = async ( return recordMember( role, await readObservedArtifact( + role, role === "database" ? path : walPath, observed, maximumBytes, diff --git a/src/wasm/WabtFailures.ts b/src/wasm/WabtFailures.ts index 27ba86a1bc..d7bc63aaed 100644 --- a/src/wasm/WabtFailures.ts +++ b/src/wasm/WabtFailures.ts @@ -1,5 +1,6 @@ import { analysisSettingLimit } from "../domain/analysisSettingDefinitions.js"; import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import { stableArtifactInputRecovery } from "../artifacts/StableArtifactFailure.js"; import { artifactFileCancellationCause, bindArtifactFileCleanup, @@ -307,7 +308,13 @@ const classifyWabtFailure = ( if (phase === "artifact") return new AnalysisInputError( "inspect_wasm_artifact", - { cause, ...artifactOptions }, + { + cause, + ...artifactOptions, + ...(cause instanceof ArtifactReaderFailure + ? stableArtifactInputRecovery(cause) + : {}), + }, [ { path: [...(execution?.inputField ?? ["path"])], diff --git a/tests/boundary/go/goBinaryProvider.test.ts b/tests/boundary/go/goBinaryProvider.test.ts index 077f2ecbf9..cfda1f065e 100644 --- a/tests/boundary/go/goBinaryProvider.test.ts +++ b/tests/boundary/go/goBinaryProvider.test.ts @@ -131,7 +131,12 @@ it.skipIf(process.platform === "win32")( await symlink(original, selected); const result = await new GoBinaryProvider().inspect({ path: selected }); expect(result.ok).toBe(false); - if (!result.ok) expect(result.error).toBeInstanceOf(AnalysisInputError); + if (!result.ok) { + expect(result.error).toBeInstanceOf(AnalysisInputError); + const projected = projectAnalysisError(result.error); + expect(projected.remediation.action).toContain(original); + expect(projected).not.toHaveProperty("recovery"); + } }, ); diff --git a/tests/boundary/mcp/goBinaryMcp.test.ts b/tests/boundary/mcp/goBinaryMcp.test.ts index 909071e375..0ccb8c3cd9 100644 --- a/tests/boundary/mcp/goBinaryMcp.test.ts +++ b/tests/boundary/mcp/goBinaryMcp.test.ts @@ -1,7 +1,7 @@ import { goBinarySchema } from "../../../src/domain/go/goBinary.js"; import type { Client } from "@modelcontextprotocol/client"; import { Ajv2020 } from "ajv/dist/2020.js"; -import { writeFile } from "node:fs/promises"; +import { mkdir, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; @@ -184,6 +184,69 @@ for (const expectedErrno of ["ELOOP", "ENAMETOOLONG"] as const) { ); } +goMcpTest.skipIf(process.platform === "win32")( + "suggests a resolved regular file after rejecting a selected symlink", + async ({ go: { client } }) => { + const root = await createTestTempDirectory("rea-go-mcp-symlink-"); + const original = join(root, "original.elf"); + const selected = join(root, "selected"); + await writeFile(original, createGoBinaryFixture().bytes); + await symlink(original, selected); + + const rejected = await client.callTool({ + name: "inspect_go_binary", + arguments: { path: selected }, + }); + const diagnostic = parseMcpToolError(rejected).error; + expect(diagnostic).toMatchObject({ + code: "invalid_request", + details: { + issues: [{ path: ["path"], reason: "invalid_format" }], + }, + remediation: { action: expect.stringContaining(original) }, + }); + expect(diagnostic).not.toHaveProperty("recovery"); + + const retried = await client.callTool({ + name: "inspect_go_binary", + arguments: { path: original }, + }); + expect(retried.isError).not.toBe(true); + }, +); + +for (const targetKind of ["dangling", "directory"] as const) { + goMcpTest.skipIf(process.platform === "win32")( + `keeps a ${targetKind} symlink rejection without a resolved-file hint`, + async ({ go: { client } }) => { + const root = await createTestTempDirectory("rea-go-mcp-symlink-"); + const target = join(root, "target"); + const selected = join(root, "selected"); + if (targetKind === "directory") await mkdir(target); + await symlink(target, selected); + + const rejected = await client.callTool({ + name: "inspect_go_binary", + arguments: { path: selected }, + }); + const diagnostic = parseMcpToolError(rejected).error; + expect(diagnostic).toMatchObject({ + code: "invalid_request", + details: { + issues: [ + { + path: ["path"], + reason: "invalid_format", + message: expect.stringContaining(selected), + }, + ], + }, + }); + expect(diagnostic.remediation.action).not.toContain(target); + }, + ); +} + goMcpTest( "preserves invalid embedded Go text bytes through MCP", async ({ go: { client, session } }) => { diff --git a/tests/boundary/mcp/stableArtifactSymlinkRecoveryMcp.test.ts b/tests/boundary/mcp/stableArtifactSymlinkRecoveryMcp.test.ts new file mode 100644 index 0000000000..c1611a1021 --- /dev/null +++ b/tests/boundary/mcp/stableArtifactSymlinkRecoveryMcp.test.ts @@ -0,0 +1,56 @@ +import { symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; + +import { createServer } from "../../../src/server/createServer.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { mcpTest } from "../../support/mcp/mcpFixture.js"; + +mcpTest.skipIf(process.platform === "win32")( + "offers one resolved-file retry for sibling stable-artifact inspectors", + async ({ mcp }) => { + const root = await createTestTempDirectory("rea-stable-recovery-mcp-"); + const target = join(root, "target.bin"); + const selected = join(root, "selected"); + await writeFile(target, "carrier"); + await symlink(target, selected); + const session = createTestBinarySession(() => { + throw new Error("Target-free inspection must not start a deep provider"); + }); + try { + const client = await mcp.connect( + createServer({ kind: "session", session }), + ); + for (const name of [ + "inspect_rust_binary", + "inspect_elf_notes", + "inspect_devicetree", + "inspect_sqlite_database", + ]) { + const response = await client.callTool({ + name, + arguments: { path: selected }, + }); + const diagnostic = parseMcpToolError(response).error; + expect(diagnostic.remediation.action, name).toContain(target); + expect(diagnostic, name).not.toHaveProperty("recovery"); + } + const captureResponse = await client.callTool({ + name: "inspect_web_network_capture", + arguments: { + capture_path: selected, + format: "har", + sensitive_values: ["unrelated marked value"], + }, + }); + const captureError = parseMcpToolError(captureResponse).error; + expect(captureError.remediation.action).toContain(target); + expect(captureError).not.toHaveProperty("recovery"); + } finally { + const closed = await session.close(); + if (!closed.ok) throw closed.error; + } + }, +); diff --git a/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts b/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts index 5cfd3a6f8d..09d8710062 100644 --- a/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts @@ -206,6 +206,20 @@ it.runIf(process.platform !== "win32")( }, ); +it.runIf(process.platform !== "win32")( + "does not offer a resolved-file retry for a derived WAL symlink", + async () => { + const { database, workspace, capture } = await fixture(true); + const target = workspace.path("wal-target"); + await rename(database.walPath, target); + await symlink(target, database.walPath); + await expect(capture()).rejects.toMatchObject({ + operationCause: { reason: "path", resolvedPath: undefined }, + completedMembers: [{ role: "database", path: database.path }], + }); + }, +); + it.each(["activate", "replace", "remove"])( "rejects a cold journal changed during database acquisition: %s", async (change) => { From 2d16a40a491067ef0bbe541feef2feeb5f3110ad Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:58:50 +0800 Subject: [PATCH 64/94] fix(ghidra): report selected project identity failures (#8379) * fix(ghidra): report selected project identity failures * fix(ghidra): retain refusal until client shutdown --- bridge/ghidra/ReaGhidraBridge.java | 29 +- docs/installation.md | 7 +- scripts/lib/native-value-e2e.mjs | 7 + scripts/lib/real-ghidra-project-identity.mjs | 249 ++++++++++++++++++ src/ghidra/GhidraAdmissionError.ts | 2 + src/ghidra/GhidraClientStartup.ts | 20 ++ src/ghidra/GhidraProvider.behavior.test.ts | 41 +++ src/ghidra/GhidraProviderClient.ts | 27 ++ .../providers/ghidra/ghidraClient.fixture.ts | 3 + .../providers/ghidra/ghidraClient.test.ts | 246 ++++++++++++----- tests/fixtures/fakeGhidra.mjs | 51 ++++ 11 files changed, 605 insertions(+), 77 deletions(-) create mode 100644 scripts/lib/real-ghidra-project-identity.mjs diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index efed6af767..c815e8edf6 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -179,6 +179,8 @@ public final class ReaGhidraBridge extends HeadlessScript { private List procedureInventory; private List stringInventory; private DecompInterface decompiler; + // A refused Program may only report its admission failure to the client. + private String identityAdmissionFailure; /** Per-function native decompiler limit from the session; 0 leaves it unbounded. */ private int decompileTimeoutSeconds; // DecompileProcess converts seconds to an int millisecond timer delay. @@ -224,14 +226,17 @@ public final class ReaGhidraBridge extends HeadlessScript { throw new IllegalStateException("Ghidra provider version does not match the session"); } String importedSha256 = currentProgram.getExecutableSHA256(); - if (importedSha256 == null || - !constantTimeEquals( - importedSha256.toLowerCase(Locale.ROOT), - descriptor.targetSha256 - )) { - throw new IllegalStateException( - "Ghidra imported-byte digest does not match the admitted target" - ); + if (importedSha256 == null || !importedSha256.matches("(?i)[0-9a-f]{64}")) { + identityAdmissionFailure = "import_identity_unavailable"; + } + else if (!constantTimeEquals(importedSha256.toLowerCase(Locale.ROOT), + descriptor.targetSha256)) { + identityAdmissionFailure = "import_identity_mismatch"; + } + if (identityAdmissionFailure != null) { + end(false); + serve(descriptor); + return; } admittedTargetSha256 = descriptor.targetSha256; // GhidraScript wraps run() in a transaction. End it before serving so @@ -405,6 +410,14 @@ public final class ReaGhidraBridge extends HeadlessScript { writeSuccess(writer, request.id, result); return; } + if (identityAdmissionFailure != null) { + // Keep the producer alive until the client has consumed the + // refusal and closed or sent shutdown. Every other request + // remains refused, including further startup attempts. + writeFailure(writer, request.id, identityAdmissionFailure, + "Ghidra imported identity admission failed"); + continue; + } decompileTimeoutSeconds = request.decompileTimeoutSeconds() == null ? descriptor.decompileTimeoutSeconds() : request.decompileTimeoutSeconds(); writeSuccess(writer, request.id, handleRequest(request, descriptor)); diff --git a/docs/installation.md b/docs/installation.md index 322a0a55aa..a2cdd53a4a 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1450,8 +1450,11 @@ the original database; REA reads selected files only for fingerprinting and copy This mode does not copy unrelated sibling projects, rerun repair scripts, or admit bridge database edits. Cleanup removes the owned copy. The bridge checks the program's recorded import SHA-256 against the admitted -target digest; mismatches and project changes fail explicitly. Reopen the target -to resolve a new profile after intentionally changing the source project. +target digest. An unavailable or malformed recorded digest is reported as an +unverified identity, distinct from a known mismatch; neither admits the program +or triggers a fresh import. Select a matching target or explicitly configure a +different project or program. Changes to the source project also fail admission; +reopen the target to resolve a new profile after intentionally changing it. The profile records the caller path, resolved project/name, program and content fingerprint. Original analysis settings and completeness remain **unknown**; diff --git a/scripts/lib/native-value-e2e.mjs b/scripts/lib/native-value-e2e.mjs index a20ccb2e3a..959a62b6d8 100644 --- a/scripts/lib/native-value-e2e.mjs +++ b/scripts/lib/native-value-e2e.mjs @@ -17,6 +17,7 @@ import { requireMcpOutput, } from "./mcp-verifier-results.mjs"; import { verifyGhidraBoundaries } from "./real-ghidra-boundaries.mjs"; +import { verifyGhidraProjectIdentity } from "./real-ghidra-project-identity.mjs"; const defaultEntrypoint = fileURLToPath(new URL("../rea.mjs", import.meta.url)); const environment = () => ({ @@ -317,6 +318,11 @@ async function verifyNativeValueAdapters( await client.close(); await transport.close(); } + const projectIdentity = await verifyGhidraProjectIdentity({ + target, + entrypoint, + env, + }); return { mocked: false, cli: true, @@ -324,6 +330,7 @@ async function verifyNativeValueAdapters( inline_evidence: true, direct_function_comparison: true, boundaries, + project_identity: projectIdentity, nodes: cliGraph.total_nodes, edges: cliGraph.total_edges, decompilations: cliGraph.decompilations, diff --git a/scripts/lib/real-ghidra-project-identity.mjs b/scripts/lib/real-ghidra-project-identity.mjs new file mode 100644 index 0000000000..1c068ab2ed --- /dev/null +++ b/scripts/lib/real-ghidra-project-identity.mjs @@ -0,0 +1,249 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + appendFile, + copyFile, + mkdtemp, + readFile, + readdir, + rm, + writeFile, +} from "node:fs/promises"; +import { basename, join } from "node:path"; +import { promisify } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { requireMcpToolError } from "./mcp-verifier-results.mjs"; + +const exec = promisify(execFile); + +/** Verify selected-project identity refusal through real CLI and MCP adapters. */ +export async function verifyGhidraProjectIdentity({ target, entrypoint, env }) { + if (process.platform === "win32") + return { + status: "not-applicable", + reason: "Existing Ghidra projects are not admitted on Windows.", + }; + assert.ok(env.GHIDRA_INSTALL_DIR, "Ghidra installation is required"); + const root = await mkdtemp(join(env.TMPDIR, "rea-project-identity-")); + const targetPath = target.path; + const different = join(root, "different-target"); + const headless = join(env.GHIDRA_INSTALL_DIR, "support", "analyzeHeadless"); + const fixtureEnv = { + ...env, + HOME: root, + XDG_CONFIG_HOME: join(root, "config"), + XDG_CACHE_HOME: join(root, "cache"), + XDG_DATA_HOME: join(root, "data"), + JDK_JAVA_OPTIONS: `-Duser.home="${root}"`, + }; + const program = basename(targetPath); + const known = join(root, "known.gpr"); + const unknown = join(root, "unknown.gpr"); + const runtimeRootsBefore = (await readdir(env.TMPDIR)) + .filter((name) => name.startsWith("rea-ghidra-")) + .sort(); + const sha = async (path) => + createHash("sha256") + .update(await readFile(path)) + .digest("hex"); + try { + await copyFile(targetPath, different); + await appendFile(different, "\nREA_IDENTITY_VERIFIER\n"); + assert.notEqual(await sha(different), await sha(targetPath)); + await writeFile( + join(root, "ClearImportDigest.java"), + `import ghidra.app.script.GhidraScript; +public class ClearImportDigest extends GhidraScript { + @Override public void run() { + currentProgram.setExecutableSHA256(null); + println("REA_DIGEST_AFTER=" + currentProgram.getExecutableSHA256()); + } +}\n`, + ); + await exec( + headless, + [root, "known", "-import", targetPath, "-noanalysis"], + { env: fixtureEnv, timeout: 180000, maxBuffer: 8 * 1024 * 1024 }, + ); + const cleared = await exec( + headless, + [ + root, + "unknown", + "-import", + targetPath, + "-noanalysis", + "-scriptPath", + root, + "-postScript", + "ClearImportDigest.java", + ], + { env: fixtureEnv, timeout: 180000, maxBuffer: 8 * 1024 * 1024 }, + ); + assert.match(cleared.stdout, /REA_DIGEST_AFTER=unknown/u); + const before = await projectState([known, unknown, targetPath, different]); + + const knownEnv = { + ...env, + REA_GHIDRA_PROJECT_PATH: known, + REA_GHIDRA_PROGRAM_NAME: program, + }; + const unknownEnv = { + ...env, + REA_GHIDRA_PROJECT_PATH: unknown, + REA_GHIDRA_PROGRAM_NAME: program, + }; + const mismatch = await selectedProjectMcp({ + entrypoint, + env: knownEnv, + first: different, + matching: targetPath, + expected: "import_identity_mismatch", + }); + const unavailable = await selectedProjectMcp({ + entrypoint, + env: unknownEnv, + first: targetPath, + expected: "import_identity_unavailable", + }); + for (const [label, selectedEnv, input, expected] of [ + ["mismatch", knownEnv, different, "import_identity_mismatch"], + ["unavailable", unknownEnv, targetPath, "import_identity_unavailable"], + ]) { + let failed; + try { + await exec( + process.execPath, + [ + entrypoint, + "decompile", + input, + "0x0", + "--provider", + "ghidra", + "--format", + "json", + ], + { env: selectedEnv, timeout: 180000, maxBuffer: 8 * 1024 * 1024 }, + ); + } catch (cause) { + assert.equal(cause.code, 1, `${label} CLI exit`); + failed = JSON.parse(cause.stdout); + } + assert.ok(failed, `${label} CLI must refuse`); + assert.equal(failed.code, "configuration_invalid"); + assert.equal(failed.details.diagnostics.remote_code, expected); + } + assert.deepEqual( + await projectState([known, unknown, targetPath, different]), + before, + ); + assert.deepEqual( + (await readdir(env.TMPDIR)) + .filter((name) => name.startsWith("rea-ghidra-")) + .sort(), + runtimeRootsBefore, + "Selected-project checks left a Ghidra runtime directory", + ); + return { + status: "verified", + mismatch, + unavailable, + cli_refusals: true, + source_files_unchanged: true, + }; + } finally { + await rm(root, { recursive: true, force: true }); + } +} + +async function selectedProjectMcp({ + entrypoint, + env, + first, + matching, + expected, +}) { + const client = new Client({ + name: "rea-project-identity-verifier", + version: "1", + }); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [entrypoint, "mcp"], + env, + stderr: "pipe", + }); + try { + await client.connect(transport); + const tools = new Map( + (await client.listTools()).tools.map((tool) => [tool.name, tool]), + ); + const call = (name, arguments_) => + client.callTool( + { name, arguments: arguments_ }, + { timeout: 180000, toolDefinition: tools.get(name) }, + ); + assert.notEqual( + (await call("open_binary", { path: first, provider_id: "ghidra" })) + .isError, + true, + ); + const rejected = requireMcpToolError(await call("list_documents", {})); + assert.equal( + rejected.code, + "configuration_invalid", + JSON.stringify({ + code: rejected.code, + message: rejected.message, + failure_kind: rejected.details?.diagnostics?.failure_kind, + failure_message: rejected.details?.diagnostics?.failure_message, + stdout: rejected.details?.diagnostics?.stdout?.text?.slice(-2500), + stderr: rejected.details?.diagnostics?.stderr?.text?.slice(-1000), + }), + ); + assert.equal(rejected.details.diagnostics.remote_code, expected); + if (matching !== undefined) { + assert.notEqual( + (await call("open_binary", { path: matching, provider_id: "ghidra" })) + .isError, + true, + ); + assert.notEqual((await call("list_documents", {})).isError, true); + } + assert.notEqual((await call("close_binary", {})).isError, true); + return { + refused: expected, + matching_target_reopened: matching !== undefined, + }; + } finally { + await client.close(); + await transport.close(); + } +} + +async function projectState(paths) { + const files = []; + const walk = async (dir) => { + for (const entry of await readdir(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) await walk(path); + else if (entry.isFile()) files.push(path); + } + }; + for (const path of paths) { + files.push(path); + if (path.endsWith(".gpr")) await walk(`${path.slice(0, -4)}.rep`); + } + files.sort(); + return Promise.all( + files.map(async (path) => [ + path, + createHash("sha256") + .update(await readFile(path)) + .digest("hex"), + ]), + ); +} diff --git a/src/ghidra/GhidraAdmissionError.ts b/src/ghidra/GhidraAdmissionError.ts index f6ecbf7890..428c0170fa 100644 --- a/src/ghidra/GhidraAdmissionError.ts +++ b/src/ghidra/GhidraAdmissionError.ts @@ -28,6 +28,8 @@ const GHIDRA_ADMISSION_CODES = [ "prepared_options_exceed_bound", "output_unobserved", "handshake_identity", + "import_identity_unavailable", + "import_identity_mismatch", "response_shape", "response_json", "seed_path_differs", diff --git a/src/ghidra/GhidraClientStartup.ts b/src/ghidra/GhidraClientStartup.ts index 092e2d7840..bab00101eb 100644 --- a/src/ghidra/GhidraClientStartup.ts +++ b/src/ghidra/GhidraClientStartup.ts @@ -8,6 +8,7 @@ import { type GhidraSessionFailureFactory, } from "./GhidraSessionError.js"; import type { GhidraSessionInfo } from "./GhidraSessionValues.js"; +import { GhidraAdmissionError } from "./GhidraAdmissionError.js"; /** Inputs needed to complete the post-connection bridge handshake. */ export interface GhidraStartupHandshakeOptions { @@ -42,6 +43,25 @@ export async function completeGhidraStartupHandshake( options.deadline.interruption !== undefined ? interruptionFailure(options) : ping.error; + if ( + failure.kind === "remote" && + (failure.remoteCode === "import_identity_unavailable" || + failure.remoteCode === "import_identity_mismatch") + ) { + const admitted = options.failure( + "admission", + failure.message, + new GhidraAdmissionError(failure.remoteCode, undefined, { + cause: failure, + }), + { remoteCode: failure.remoteCode }, + ); + return err( + failure.cleanupFailure === undefined + ? admitted + : admitted.withCleanupFailure(failure.cleanupFailure), + ); + } return err(failure); } const parsed = options.parseSessionInfo(ping.value); diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index b5c94aaf9a..4018194eb9 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -89,6 +89,7 @@ import { } from "./GhidraSessionValues.js"; import type { Result } from "../domain/result.js"; import { GhidraSessionError } from "./GhidraSessionError.js"; +import { GhidraAdmissionError } from "./GhidraAdmissionError.js"; import { silentLogger } from "../logger.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; @@ -4173,6 +4174,46 @@ describe("Ghidra deadline guidance", () => { }); }); +it.each([ + ["import_identity_unavailable", "did not report a valid imported SHA-256"], + ["import_identity_mismatch", "differs from the selected target"], +] as const)( + "reports fresh-import %s without blaming project configuration", + async (code, message) => { + const failure = new GhidraSessionError( + "admission", + "Identity admission failed", + {}, + { + cause: new GhidraAdmissionError(code), + remoteCode: code, + }, + ); + const ghidra = provider(installationHost(), () => ({ + start: async () => err(failure), + callTool: async () => err(failure), + close: async () => ok(null), + })); + onTestFinished(async () => { + await ghidra.closeOwnedResources(); + }); + const target = executableTarget("elf", "x86_64"); + const profile = await ghidra.resolveAnalysisProfile(target); + if (!profile.ok || profile.value.profile === null) + throw new Error("Expected profile"); + const result = await ghidra + .createClient(target, profile.value.profile) + .execute("list_documents", {}); + if (result.ok) throw new Error("Expected identity admission failure"); + const projected = projectAnalysisError(result.error); + expect(projected.code).toBe("execution_failure"); + expect(projected.message).toContain(message); + expect(JSON.stringify(projected.details)).not.toContain( + "REA_GHIDRA_PROJECT_PATH", + ); + }, +); + describe("Ghidra startup deadline for a finished invocation", () => { const startupLimit = { setting: "ghidra.startup_timeout_ms", diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index 0468cd4a36..c7b69896ad 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -9,6 +9,7 @@ import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup. import { artifactFilePrimaryFacts } from "../artifacts/ArtifactFileOperation.js"; import { fileURLToPath } from "node:url"; import { GHIDRA_BRIDGE_SOURCES } from "./GhidraBridgeSources.js"; +import { GhidraAdmissionError } from "./GhidraAdmissionError.js"; import { createAnalysisExecution, @@ -1239,6 +1240,32 @@ const projectPrimarySessionError = ( physicalCause.reason, context, ); + if ( + failure.kind === "admission" && + failure.cause instanceof GhidraAdmissionError && + (failure.cause.code === "import_identity_unavailable" || + failure.cause.code === "import_identity_mismatch") + ) { + const unavailable = failure.cause.code === "import_identity_unavailable"; + const constraint = unavailable + ? "The Ghidra program did not report a valid imported SHA-256, so REA cannot verify its target identity" + : "The Ghidra program's imported SHA-256 differs from the selected target's SHA-256"; + const remediationAction = unavailable + ? "Select a Ghidra program whose imported identity can be verified, or investigate the missing identity in Ghidra. Configure a different project or program when starting REA. REA did not fall back to a fresh import." + : "Select a target matching the configured program and retry, or configure a matching project or program when starting REA. Resolve any reported cleanup before reopening. REA did not fall back to a fresh import."; + if (limits.ghidraExistingProject !== undefined) + return new ConfigurationError(constraint, { + ...context, + settings: [{ setting: "REA_GHIDRA_PROJECT_PATH", constraint }], + remediationAction, + }); + return new ProviderAdapterError("ghidra", operation, { + ...context, + userMessage: `${constraint}.`, + remediationAction: + "Inspect the retained Ghidra diagnostics and target identity before retrying the import.", + }); + } // A well-formed address that lacks one exact source offset keeps the // bridge's reason and the caller's exact spelling, not schema advice. const fileOffset = diff --git a/tests/boundary/providers/ghidra/ghidraClient.fixture.ts b/tests/boundary/providers/ghidra/ghidraClient.fixture.ts index 045a9065db..91e9f5bbe9 100644 --- a/tests/boundary/providers/ghidra/ghidraClient.fixture.ts +++ b/tests/boundary/providers/ghidra/ghidraClient.fixture.ts @@ -29,6 +29,9 @@ type FixtureMode = | "remote_error" | "decompile_timed_out" | "remote_startup_error" + | "import_identity_unavailable" + | "import_identity_mismatch" + | "late_identity_failure" | "shutdown_error" | "invalid_shutdown_ack" | "hang_after_start" diff --git a/tests/boundary/providers/ghidra/ghidraClient.test.ts b/tests/boundary/providers/ghidra/ghidraClient.test.ts index a07c00c5c4..6e481ad4af 100644 --- a/tests/boundary/providers/ghidra/ghidraClient.test.ts +++ b/tests/boundary/providers/ghidra/ghidraClient.test.ts @@ -29,6 +29,7 @@ import type { ProgressUpdate } from "../../../../src/application/ProgressReporte import { err, ok } from "../../../../src/domain/result.js"; import * as GhidraTransport from "../../../../src/ghidra/GhidraTransport.js"; import { GhidraClient } from "../../../../src/ghidra/GhidraClient.js"; +import { GhidraAdmissionError } from "../../../../src/ghidra/GhidraAdmissionError.js"; import { ProviderRunLineage } from "../../../../src/process/ProviderRunLineage.js"; import { PrivateRuntimeRoot } from "../../../../src/process/PrivateRuntimeRoot.js"; import { OwnedFileHandle } from "../../../../src/filesystem/OwnedFileHandle.js"; @@ -1017,6 +1018,45 @@ it("retains launcher diagnostics when an input error rejects the startup handsha expect(result.error.diagnostics).not.toHaveProperty("diagnostic_scope"); }); +it.each( + ( + ["import_identity_unavailable", "import_identity_mismatch"] as const + ).flatMap((code) => + (HOST_TRANSPORT === "unix-socket" + ? ([HOST_TRANSPORT, "authenticated-loopback-tcp"] as const) + : ([HOST_TRANSPORT] as const) + ).map((transport) => [code, transport] as const), + ), +)( + "retains structured %s from %s startup and cleans the failed bridge", + async (code, transport) => { + const launcher = new FixtureLauncher(code); + const client = clientFor(launcher, { transport }); + const result = await client.start(); + if (result.ok) throw new Error("Expected identity admission failure"); + expect(result.error.kind).toBe("admission"); + expect(result.error.cause).toBeInstanceOf(GhidraAdmissionError); + expect(result.error.cause).toMatchObject({ code }); + expect(client.isReady()).toBe(false); + await expect(waitForExit(launcher.processes[0])).resolves.toBe(true); + expect(launcher.requests).toEqual(["ping", "shutdown"]); + await expect(access(launcher.runtimeRoots[0] ?? "")).rejects.toMatchObject({ + code: "ENOENT", + }); + }, +); + +it("does not classify a later remote error as an identity admission failure", async () => { + const client = clientFor(new FixtureLauncher("late_identity_failure")); + expect((await client.start()).ok).toBe(true); + const result = await client.callTool("list_strings", {}); + if (result.ok) throw new Error("Expected the fixture's later remote failure"); + expect(result.error.kind).toBe("remote"); + expect(result.error.remoteCode).toBe("import_identity_mismatch"); + expect(result.error.cause).not.toBeInstanceOf(GhidraAdmissionError); + expect(client.isReady()).toBe(true); +}); + describe("GhidraClient", () => { it("completes an exact, fragmented post-analysis handshake", async () => { const launcher = new FixtureLauncher("fragmented"); @@ -2073,10 +2113,73 @@ describe("GhidraClient shutdown diagnostics", () => { ); }); -it("preserves failed startup and retained Ghidra cleanup capabilities until a verified retry", async () => { - const fixture = new FixtureLauncher("wrong_identity"); - let cleanupAllowed = false; - let cleanupAttempts = 0; +it.each([ + ["wrong_identity", "protocol"], + ["import_identity_unavailable", "admission"], +] as const)( + "preserves %s and retained Ghidra cleanup until a verified retry", + async (mode, kind) => { + const fixture = new FixtureLauncher(mode); + let cleanupAllowed = false; + let cleanupAttempts = 0; + const launcher: GhidraLauncher = { + launch: async (session) => { + const launched = await fixture.launch(session); + if (!launched.ok) return launched; + return ok({ + ...launched.value, + cleanup: async () => { + cleanupAttempts += 1; + return cleanupAllowed + ? { cleaned: true as const, signaled: false } + : { + cleaned: false as const, + reason: "fixture process cleanup unconfirmed", + }; + }, + }); + }, + }; + const client = clientFor(launcher); + onTestFinished(async () => { + cleanupAllowed = true; + await client.close(); + }); + const failed = await client.start(); + expect(failed).toMatchObject({ + ok: false, + error: { kind, cleanupFailure: { cleanupIncomplete: true } }, + }); + if (failed.ok) throw new Error("Expected failed startup"); + if (mode === "import_identity_unavailable") + expect(failed.error.cause).toMatchObject({ code: mode }); + const runtime = fixture.runtimeRoots[0]; + if (runtime === undefined) + throw new Error("Fixture runtime was not captured"); + await access(runtime); + expect(cleanupAttempts).toBe(1); + expect(await client.close()).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + cleanupAllowed = true; + expect(await client.close()).toEqual(ok(null)); + expect(failed.error.cleanupFailure?.cleanupIncomplete).toBe(false); + expect(cleanupAttempts).toBe(3); + await expect(access(runtime)).rejects.toMatchObject({ code: "ENOENT" }); + }, +); + +it("retains the identity refusal when cancellation arrives during final startup cleanup", async () => { + const fixture = new FixtureLauncher("import_identity_mismatch"); + let entered: (() => void) | undefined; + let release: (() => void) | undefined; + const cleanupStarted = new Promise((resolve) => { + entered = resolve; + }); + const gate = new Promise((resolve) => { + release = resolve; + }); const launcher: GhidraLauncher = { launch: async (session) => { const launched = await fixture.launch(session); @@ -2084,39 +2187,40 @@ it("preserves failed startup and retained Ghidra cleanup capabilities until a ve return ok({ ...launched.value, cleanup: async () => { - cleanupAttempts += 1; - return cleanupAllowed - ? { cleaned: true as const, signaled: false } - : { - cleaned: false as const, - reason: "fixture process cleanup unconfirmed", - }; + entered?.(); + await gate; + return { cleaned: true as const, signaled: false }; }, }); }, }; - const client = clientFor(launcher); - onTestFinished(async () => { - cleanupAllowed = true; - await client.close(); - }); - expect(await client.start()).toMatchObject({ - ok: false, - error: { kind: "protocol", cleanupFailure: { cleanupIncomplete: true } }, - }); - const runtime = fixture.runtimeRoots[0]; - if (runtime === undefined) - throw new Error("Fixture runtime was not captured"); - await access(runtime); - expect(cleanupAttempts).toBe(1); - expect(await client.close()).toMatchObject({ - ok: false, - error: { cleanupIncomplete: true }, - }); - cleanupAllowed = true; - expect(await client.close()).toEqual(ok(null)); - expect(cleanupAttempts).toBe(3); - await expect(access(runtime)).rejects.toMatchObject({ code: "ENOENT" }); + const client = clientFor(launcher, { startupTimeoutMs: 10_000 }); + const controller = new AbortController(); + const started = client.start(controller.signal); + await withVerifierCleanup(async () => { + await awaitAcknowledgement( + cleanupStarted, + started, + "identity refusal cleanup", + ); + controller.abort("cancel during final cleanup"); + release?.(); + const result = await started; + if (result.ok) throw new Error("Expected the earlier identity refusal"); + expect(result.error.kind).toBe("admission"); + expect(result.error.cause).toMatchObject({ + code: "import_identity_mismatch", + }); + expect(client.isReady()).toBe(false); + await expect(access(fixture.runtimeRoots[0] ?? "")).rejects.toMatchObject({ + code: "ENOENT", + }); + }, [ + () => release?.(), + async () => { + await client.close(); + }, + ]); }); describe.skipIf(process.platform === "win32")( @@ -2370,42 +2474,50 @@ it.each([null, "caller stopped", new Error("caller stopped")])( }, ); -it("preserves a handshake protocol error observed before the final startup caller cancels", async () => { - const controller = new AbortController(); - let cancelled = false; - const logger = pino( - { - level: "warn", - hooks: { - logMethod(args, method) { - const fields = args[0]; - if ( - typeof fields === "object" && - fields !== null && - "method" in fields && - fields.method === "ping" && - !cancelled - ) { - cancelled = true; - controller.abort(null); - } - method.apply(this, args); +it.each([ + ["malformed", "protocol"], + ["import_identity_unavailable", "admission"], +] as const)( + "preserves %s observed before the final startup caller cancels", + async (mode, kind) => { + const controller = new AbortController(); + let cancelled = false; + const logger = pino( + { + level: "warn", + hooks: { + logMethod(args, method) { + const fields = args[0]; + if ( + typeof fields === "object" && + fields !== null && + "method" in fields && + fields.method === "ping" && + !cancelled + ) { + cancelled = true; + controller.abort(null); + } + method.apply(this, args); + }, }, }, - }, - { write: () => {} }, - ); - const launcher = new FixtureLauncher("malformed"); - const client = clientFor(launcher, { logger }); - const result = await client.start(controller.signal); - expect(cancelled).toBe(true); - if (result.ok) throw new Error("Expected handshake protocol failure"); - expect(result.error.kind).toBe("protocol"); - expect(await waitForExit(launcher.processes[0])).toBe(true); - await expect(access(launcher.runtimeRoots[0] ?? "")).rejects.toMatchObject({ - code: "ENOENT", - }); -}); + { write: () => {} }, + ); + const launcher = new FixtureLauncher(mode); + const client = clientFor(launcher, { logger }); + const result = await client.start(controller.signal); + expect(cancelled).toBe(true); + if (result.ok) throw new Error("Expected handshake protocol failure"); + expect(result.error.kind).toBe(kind); + if (mode === "import_identity_unavailable") + expect(result.error.cause).toMatchObject({ code: mode }); + expect(await waitForExit(launcher.processes[0])).toBe(true); + await expect(access(launcher.runtimeRoots[0] ?? "")).rejects.toMatchObject({ + code: "ENOENT", + }); + }, +); it("redacts the transport credential when refreshing startup diagnostics", async () => { let token = ""; diff --git a/tests/fixtures/fakeGhidra.mjs b/tests/fixtures/fakeGhidra.mjs index 5ef111d92b..22b8336b23 100644 --- a/tests/fixtures/fakeGhidra.mjs +++ b/tests/fixtures/fakeGhidra.mjs @@ -182,6 +182,40 @@ const handleRequest = (socket, server, request, state) => { handlePing(socket, request, state); return; } + if ( + ["import_identity_unavailable", "import_identity_mismatch"].includes( + state.mode, + ) && + request.method !== "shutdown" + ) { + socket.write( + `${JSON.stringify({ + id: request.id, + ok: false, + error: { + code: state.mode, + message: "Ghidra imported identity admission failed", + }, + })}\n`, + ); + return; + } + if ( + state.mode === "late_identity_failure" && + request.method === "list_strings" + ) { + socket.write( + `${JSON.stringify({ + id: request.id, + ok: false, + error: { + code: "import_identity_mismatch", + message: "Unexpected later failure", + }, + })}\n`, + ); + return; + } if ( state.mode === "old_oom_then_malformed" && request.method === "list_documents" && @@ -341,6 +375,23 @@ const handleRequest = (socket, server, request, state) => { }; const handlePing = (socket, request, state) => { + if ( + ["import_identity_unavailable", "import_identity_mismatch"].includes( + state.mode, + ) + ) { + socket.write( + `${JSON.stringify({ + id: request.id, + ok: false, + error: { + code: state.mode, + message: "Ghidra project identity admission failed", + }, + })}\n`, + ); + return; + } if (state.mode === "remote_startup_error") { socket.write( `${JSON.stringify({ From 89317a39fab23df81efa4331311d09d2110377bd Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 4 Nov 2026 20:43:04 +0800 Subject: [PATCH 65/94] fix(javascript): remove duplicate childNodes import (#8886) * fix(javascript): remove duplicate childNodes import The squash of #7967 re-added the import already introduced in afc18131f, so typecheck failed with TS2300. * test(gdb): expect the command-output cap limitation The retained-record limitation now names the command-output setting. The coverage assertion still required the previous fixed-cap sentence. * test(cli): keep rejected mcp add values off Amp settings mcp add now rejects a flag in the --command value position. The Amp registration test still treated that invocation as a successful selection. --------- Co-authored-by: Cursor Agent --- .../electronCallPathBlockPrefixes.ts | 2 -- .../cli/defaultAmpRegistration.test.ts | 25 +++++++++++++++---- .../GdbSessionManagerCoverage.test.ts | 9 +++++-- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/domain/javascript/electronCallPathBlockPrefixes.ts b/src/domain/javascript/electronCallPathBlockPrefixes.ts index 05887cee92..8df1bc04e4 100644 --- a/src/domain/javascript/electronCallPathBlockPrefixes.ts +++ b/src/domain/javascript/electronCallPathBlockPrefixes.ts @@ -1,8 +1,6 @@ import * as t from "@babel/types"; import { childNodes } from "./javascriptSemanticTraversal.js"; -import { childNodes } from "./javascriptSemanticTraversal.js"; - /** Syntax alone can establish an exit, but never a branch's contextual truth. */ export type ElectronStatementFallthrough = | { readonly kind: "exit" } diff --git a/tests/boundary/cli/defaultAmpRegistration.test.ts b/tests/boundary/cli/defaultAmpRegistration.test.ts index 0531cbba8d..e52271e21a 100644 --- a/tests/boundary/cli/defaultAmpRegistration.test.ts +++ b/tests/boundary/cli/defaultAmpRegistration.test.ts @@ -170,13 +170,12 @@ cliTest( "non-Amp selections, command values and discovery do not touch Amp settings", async ({ cli }) => { const f = await fixture("broken JSON\n"); - for (const selection of [ + const selections = [ ["--agent", "claude-code"], ["--agent=claude-code"], ["--agent", "cursor", "--agent", "claude-code"], - ["--command", "--agent", "amp", "--agent", "claude-code"], - ["-c", "--agent", "amp", "--agent", "claude-code"], - ]) { + ]; + for (const selection of selections) { const result = await cli.run({ arguments: ["mcp", "add", ...selection, "--json"], cwd: f.home, @@ -187,8 +186,24 @@ cliTest( expect(await readFile(f.settings, "utf8")).toBe("broken JSON\n"); expect(await readdir(f.directory)).toEqual(["settings.json"]); } + // A following flag is not a command value. Reject it before Amp settings. + for (const selection of [ + ["--command", "--agent", "amp", "--agent", "claude-code"], + ["-c", "--agent", "amp", "--agent", "claude-code"], + ]) { + const result = await cli.run({ + arguments: ["mcp", "add", ...selection, "--json"], + cwd: f.home, + environment: f.environment, + timeoutMs: 20_000, + }); + expect(result.exitCode, result.stdout).toBe(1); + expect(result.stdout).toContain("Missing value for flag:"); + expect(await readFile(f.settings, "utf8")).toBe("broken JSON\n"); + expect(await readdir(f.directory)).toEqual(["settings.json"]); + } const calls = (await readFile(f.log, "utf8")).trim().split("\n"); - expect(calls).toHaveLength(5); + expect(calls).toHaveLength(selections.length); for (const line of calls) expect(JSON.parse(line)).toContain("claude-code"); await writeFile(f.log, ""); for (const flag of [ diff --git a/tests/boundary/providers/GdbSessionManagerCoverage.test.ts b/tests/boundary/providers/GdbSessionManagerCoverage.test.ts index 5d0adde808..bf0d0754e3 100644 --- a/tests/boundary/providers/GdbSessionManagerCoverage.test.ts +++ b/tests/boundary/providers/GdbSessionManagerCoverage.test.ts @@ -28,10 +28,14 @@ import type { import { DEBUGGER_COMMAND_DEADLINE_SETTING, DEBUGGER_MAX_COMMAND_OUTPUT_BYTES_SETTING, + DEFAULT_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES_LIMIT, } from "../../../src/config/debuggerSessionLimits.js"; import { DEFAULT_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES } from "../../../src/domain/analysisResourceLimits.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; -import { stopOwnedDebuggerProcess } from "../../../src/process/ownedDebuggerSession.js"; +import { + debuggerAdmissionRecovery, + stopOwnedDebuggerProcess, +} from "../../../src/process/ownedDebuggerSession.js"; import { parseLinuxProcessStat } from "../../../src/process/LinuxProcessStat.js"; import { expectGdbSessionStarted } from "./gdbTestHelpers.js"; @@ -3672,8 +3676,9 @@ it("keeps the session usable after an oversized stream record is dropped", async expect(executed.value.result.console).toContain("before"); expect(executed.value.result.completion_status).toBe("complete"); expect(executed.value.result.output_truncated).toBe(true); + const output = DEFAULT_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES_LIMIT; expect(executed.value.evidence.limitations.join(" ")).toContain( - "Retained MI records reached the 16777216 byte or 16384 record cap. This release has no runtime override. Output completeness is unknown.", + `Retained MI records reached the ${String(output.value)} byte command-output cap or the 16384 record cap. ${debuggerAdmissionRecovery(output)} The record cap has no separate setting. Output completeness is unknown.`, ); } const second = await manager.execute(started.value.session_id, "after-drop"); From c13dd36d9508e3b05b51340eb8188391c9fe08dd Mon Sep 17 00:00:00 2001 From: Entrpi <133157614+Entrpi@users.noreply.github.com> Date: Thu, 5 Nov 2026 02:53:01 +1100 Subject: [PATCH 66/94] fix(artifacts): decide how compare_artifacts compares every format_metadata kind (#8231) * feat(artifacts): inventory and extract ESP32 firmware images ESP32-family app and bootloader images (magic 0xE9) had no reader, so open_binary refused OTA downloads, app partitions and bootloaders. Add the esp-image container format behind inspect_artifact and extract_artifact. A pure decoder reads each byte once and keeps only fixed-size fields. Derived facts cover the chip, from a table cited to ESP-IDF v6.1; segment classification as ESP-IDF's should_map and should_load; checksum and appended SHA-256 results; app and bootloader descriptors; Secure Boot v2 signature blocks, observed and unverified; and undescribed trailing byte ranges. Truncation and undescribed bytes make coverage unknown. ESP8266 images are refused with a specific reason, and ciphertext stays unrecognized. Add the format to every target, Evidence, snapshot, live-summary and artifact vocabulary, document it, and add an optional verify:esp-images lane that compares esptool-built images with esptool image-info and espsecure signature-info-v2. Refs #3615 * test(artifacts): run the ESP image lane on esptool v5.6.0 The lane compared descriptor fields only when esptool image-info printed one, so a descriptor that REA reports and esptool does not print passed without comparison. Compare the descriptor kind in both directions. Record that the lane passed for all 13 images with esptool and espsecure v5.6.0, the version the layout facts cite, as well as with 4.12.0. * refactor(artifacts): define ESP image layout sizes once The decoder and the derived facts each defined the segment-header, appended-digest and flash-sector sizes, the erased-byte scan and the offset formatter. Define them once with the other fixed-layout structures and import them in both places. The signature slot count is the one constant that drives the slot scan. Correct the observedEnd comment: it is the end of the last chunk read. * fix(artifacts): report ESP-IDF's segment length limit by chip ESP-IDF's verify_segment_header rejects a segment of 16 MiB or more unless the chip maps flash through 32-bit cache addresses (SOC_SPI_MEM_SUPPORT_CACHE_32BIT_ADDR_MAP: ESP32-S3, P4, C5, C61, H4 and S31 in v6.1). Record that capability in the chip table and add a limitation for such segments; unknown chips get none. `loaded` repeats should_load. It does not mean the bootloader accepts the segment: verify_load_addresses still rejects one outside the chip's RAM, which REA does not model. Say so in the docs, the skill reference and the inventory limitation. * test(artifacts): check ESP lane tools first and verify extraction The lane started work before checking its tools, so a missing RISC-V target, ld.lld, esptool or espsecure surfaced as a raw spawn or command error. Check each one before building and name the variable to set. Warn when esptool is not v5.6.0, the release the layout facts cite. Assert each segment's class for the ESP32-C3 app, its variants and the bootloader, as the Xtensa image already did. Extract the app and the signed image through the CLI and compare every file with the bytes at esptool's segment offsets or in the appended signature sector; the source's SHA-256 must be unchanged and equal the reported root_sha256. * docs(artifacts): state what the ESP image lane does not establish The Verification section listed the lane's checks but not its gaps: only ESP32-C3 and ESP32 images come from a real producer, no real project build, OTA download or flash dump is inspected, no signature is verified, and the lane does not run in CI. Record the timing of the final lane run on esptool v5.6.0. * fix(artifacts): compare ESP segment declarations Segment entries hash only their data, so compare_artifacts reported a segment that moved to another load address with the same bytes as unchanged; only the image itself changed. Compare each segment's load_addr and data_len as its declaration, as Node SEA and PyInstaller members are compared, and count a changed declaration as a metadata change, also when content is unavailable. Header offsets are layout and the classification is derived, so neither is compared. * test(artifacts): run ESP lane commands as owned process trees The lane ran its tools through execFile. A timeout killed only the direct child, so a linker or assembler that Clang or GCC had started could keep running while the lane removed its workspace. Run every command through createVerifierCommands, check the tools inside withVerifierCleanup, and remove the workspace only after commands.close() has stopped every tree; a failed close keeps the workspace. Only a nonzero exit now counts as a refusal or an expected esptool failure; timeouts and spawn failures propagate. Supervising each command roughly doubles the lane's run time. * test(artifacts): cancel ESP decoding after its first decoded chunk The cancellation test requested cancel or close on setImmediate, which can land while the file is still being opened and checked, so it never showed that decoding had started. It also passed against a reader that ignored both requests while decoding. Acknowledge the decoder's first chunk, then cancel or close, join the operation, and assert that no later chunk reaches the decoder and that the reader ends closed. * docs(artifacts): record which ESP metadata comparison ignores Segments are compared by their declaration. The image's and the signature blocks' format_metadata are not: every field comes from their own bytes, which content comparison already covers, and a block's image_digest_matches follows the image's content. * fix(artifacts): decide declaration comparison for every metadata kind compare_artifacts listed the format_metadata kinds it compared by hand, so a new kind went uncompared without notice; ESP segment load addresses were first missed this way. One table beside the schemas now gives every kind a declaration projection or the reason comparison uses none of its facts, and a kind without an entry does not compile. Consumers read one result per pair instead of naming formats. - Godot PCK entries stay uncompared. Godot looks members up by the simplified path without res://, which logical_path holds apart from Unicode normalization. Godot 4.3 exports store res:// and 4.4.1 exports do not, so comparing source_path would mark every member changed. The reader admits only the encryption flag, which the occurrence's encrypted field records. - A changed PyInstaller declaration is now a metadata change also when the member's content is unavailable, as for ASAR, Node SEA and ESP segments. It was unknown before, for no recorded reason. - Root kinds and ESP signature blocks stay uncompared: their fields come from their own bytes, which content comparison covers. - A declaration that only one side records now adds one generic limitation for every compared kind. Only Node SEA added one before, with its own text. * fix(artifacts): keep content comparison for metadata records with content The record rule compared two metadata occurrences by their declaration alone. A hand-built record that names content, with an equal declaration, then hid a content change; PyInstaller records already did so before the table. The rule now needs both records without an artifact_id, which is every metadata record a reader emits. Review changes in the same area: - The unknown-declaration limitation also covers declarations of different kinds and non-container observations, and tests compare it with its constant. - docs/godot-pck.md gives each Godot version's lookup path, the backslash case, and that a member that becomes encrypted compares unknown. The table points to it instead of restating it. - The PyInstaller regression moves to an acceptance test whose MCP client validates the advertised output schema, and the domain tests keep one representative case per behavior. * fix(artifacts): compare encryption changes when content is unavailable An encrypted member has no content. Comparison then took the content-unavailable fallback, which counted only ASAR storage and format declarations, so a Godot PCK, ZIP or PYZ member that became encrypted compared unknown although both sides record `encrypted`. The record rule compared node-less metadata records by format declaration alone, so a PYZ namespace record whose encryption flipped compared unchanged. `OccurrenceMetadata` now owns the encryption comparison beside ASAR storage and the format declaration. Every occurrence records `encrypted`, so a pair's comparison is never unknown. The fallback, the record rule and the metadata dimension read it there; `metadataChanged` no longer compares `encrypted` itself. The record rule compares all of `OccurrenceMetadata`: a changed value is changed in metadata, with availability when another is unknown. Decision: occurrence limitations still do not count for node-less records. Readers derive them from the declared facts (the PYZ caveat follows `encrypted`), and their wording can change between REA versions while the record stays the same. The fallback does not compare them either. Behavior changes: - A pair with unavailable content whose `encrypted` differs is changed [metadata, availability], not unknown [availability]. - A node-less record whose `encrypted` differs is changed [metadata], not unchanged. - A hand-built node-less record that records ASAR storage now compares it; no reader emits such records. Remaining: - The fallback still ignores `entry_kind` and `executable`, which readers also declare, so a file that becomes a symlink, or a mode change on an encrypted ZIP member, still compares unknown. `executable` has its own dimension, so its fix needs content-free checks split from node checks. Each is its own behavior decision. - Node-bearing pairs still compare limitation wording; records do not. Whether limitation wording is ever a comparison fact needs one decision. * fix(artifacts): compare entry kind and executable when content is unavailable When either side lacked verified content, comparison took a fallback that counted only ASAR storage, encryption and format declarations. It compared `entry_kind` only in the metadata check of pairs with content, and `executable` only beside the node checks. A ZIP member that became a symlink, whose target REA does not read, and an encrypted ZIP member whose Unix mode gained an execute bit therefore compared unknown, although readers declare both fields on every occurrence. `OccurrenceMetadata` now owns the entry kind comparison beside encryption; it is never unknown. Comparison checks fall into two groups: declared facts that need no content (`executable` and the metadata declarations), and facts that need both content nodes (content, kind, format, size, relations and node metadata). The fallback, the record rule and pairs with content share the declared checks and one rule for unknown declarations. Kind and format stay content checks: without content they are guesses from the path. Behavior changes: - Unless both sides have content, a pair whose entry kind differs is changed [metadata, availability], not unknown [availability]. - Such a pair whose `executable` differs is changed [executable, availability], with metadata when another declared fact also differs. - A hand-built node-less record whose `executable` differs is changed [executable]. No reader emits such records. Remaining: declared sizes and integrity digests still compare only through observed bytes, so they do not count when content is unavailable. * fix(artifacts): never mark a symlink executable On POSIX hosts the directory reader recorded a symlink's own permission bits as `executable`. Linux reports 0777 for every symlink and macOS usually 0755, so directory, app bundle and DMG symlinks read true. zip.js, the PyInstaller reader and the Windows branch record false. The previous commit compares `executable` without content, so a symlink that two readers or two hosts record differently compared changed in `executable`, and a non-executable file that became a symlink also reported `executable`. `hostExecutableMarking` now gives a symlink no marking on any host: running a symlink runs its target, and the link's own bits do not control that. Behavior change: directory, app bundle and DMG inventories record `executable: false` for symlinks. Occurrence IDs do not change, but the graph digests and manifest IDs of inventories with symlinks do. Comparing an inventory stored before this change with a new one therefore reports changed [executable, availability] once for each such symlink. * fix(artifacts): stop comparing limitation wording Pairs with content compared node and occurrence limitations as metadata, while node-less records, the content-unavailable fallback and inventory limitations did not. Limitations explain what an observation could establish. Readers mostly derive them from fields that comparison already reads, from other occurrences or from what REA could read, and their wording changes between REA versions. So adding `A` beside an unchanged ZIP member `a` marked `a` changed in metadata through its case-collision note, and an older and a newer inventory of equal bytes could differ wherever a reader reworded a note. No comparison path reads limitations now. Inventory limitations still appear in the result, and occurrence limitations stay in the inventory Evidence. Readers write no node limitations, so only hand-built inventories compared them. Behavior change: for a pair with content, limitation text no longer adds metadata. A pair that differs only there compares unchanged, or unknown [availability] when a declaration is unknown. Remaining: a few facts appear only in limitation text, or also in fields that comparison does not read, so their changes are no longer reported while the compared bytes stay equal: an ASAR block integrity table or an unsupported integrity declaration, a Godot member's all-zero MD5, a ZIP name's leading `./` (also in `source_path`), and a PyInstaller PYZ image digest (also in `pyz_sha256`; the archive root still reports the change). Each needs a field or a comparison decision of its own if it matters. * fix(artifacts): compare declared digests when content is unavailable Without observed bytes on both sides, comparison ignored the containers' declared integrity claims. Two encrypted Godot PCK members therefore compared unknown even when their declared MD5s showed that the script had changed. Godot's exporter stores the MD5 of a member's original bytes, so the MD5 does not depend on the encryption IV. This comes from the 4.3, 4.4.1 and 4.7.1 exporter sources; no real encrypted export was compared. When either side has no content, claims of one algorithm that differ now make the path changed in metadata and availability. Equal claims add no change, because they are unverified, and observed bytes still decide when both sides have them. Godot PCK and ASAR readers record such claims. For a readable member, a directory that declares or omits an MD5 only decides whether REA can check its bytes, so such a member compares unchanged. An encrypted member with an MD5 on only one side stays unknown. Remaining: declared sizes still compare only through observed bytes. * docs(artifacts): check the contract's compared kinds against the table docs/mcp-contracts.md named the format_metadata kinds that compare_artifacts compares in prose, by hand, so a kind added to the declaration table could go unnamed. The integration branch already omits esp-flash-partition, which the ESP flash layout branch adds to the table. The paragraph now names each compared kind by its identifier. A conformance test requires it to name exactly the kinds whose table entry has a declaration, which artifactFormatMetadata.ts exports as COMPARED_FORMAT_METADATA_KINDS, and no kind that comparison ignores. * test(artifacts): share the comparison acceptance MCP fixture Six artifact comparison acceptance tests each copied the same MCP steps: open_binary, inspect_artifact, the inventory_artifact substep Evidence, then compare_artifacts. The coverage test repeated the inspect steps three times and kept its own inventory lookup. tests/fixtures/artifactComparisonMcp.ts now owns these steps on the schema-validating connectLocalToolsMcp client. connectArtifactComparisonMcp() returns call, inspect(path) and compare(left, right). inventoryEvidence() reads the inventory Evidence that an inspection retains, from MCP or CLI output. Each test still writes its own authored input and keeps its expectations. The coverage lookup matched evidence.operation, not the substep operation. The inspection schema fixes both to inventory_artifact, so both select the same substep. The plist acceptance test does not change. It compares the inspection Evidence itself across CLI and MCP and passes the comparison Evidence to verify_reconstruction and find_changed_behavior. No assertion or scenario changes. * fix(artifacts): leave .NET bundle member declarations uncompared Upstream main compares no .NET bundle member facts. Comparing `file_type` changed `compare_artifacts` output beyond this branch's fixes, so the kind stays uncompared until that is decided. A changed manifest still shows as a content change of the host. * fix(artifacts): leave HEX region declarations uncompared The comparison table compared a region's start_address and length. A region's path names its start and end addresses, and its length follows from them, so two regions that pair at one path always declare the same facts: the entry could never find a change. It only made a region with equal bytes unknown when the other side recorded no metadata, such as a directory of extracted regions. The kind is not compared, as on main. The contract paragraph also stops naming .NET bundle members, which 3e51a7760 left uncompared. * test(artifacts): compare node-less records of any compared kind The record rule follows the comparison table, not PyInstaller, but only PyInstaller records exercised it. An ESP segment record shows both outcomes: an equal declaration compares unchanged and a moved one changed in metadata, without the availability that the fallback for unread content would add. * test(artifacts): compare a node-less record's recorded ASAR storage The record rule compares ASAR storage beside the declaration and `encrypted`, but no test recorded it on a metadata record. A changed value is changed in metadata, and a value that only one side records is unknown in availability, as for any occurrence. * test(artifacts): compare UnityFS member declarations through MCP The table compares a UnityFS member's declared source_path and raw flags, but no comparison test covered the kind, so dropping either field from the projection left every test passing. Bundles with equal member bytes now show both: other flags, or a stored path in decomposed form, compare changed in metadata, and reordered members, whose index and stream offset are layout, compare unchanged. * test(artifacts): use the shared comparison fixture for UnityFS The UnityFS comparison test arrived from the bottom branch with its own copy of the open_binary, inspect_artifact and compare_artifacts steps, which tests/fixtures/artifactComparisonMcp.ts owns here. It now uses connectArtifactComparisonMcp() like the other comparison acceptance tests. No assertion changes. * test(artifacts): find the contract's compared kinds by marker The docs conformance test found its paragraph by an exact English phrase, so rewording "also compares" failed it while the named kinds stayed right. The paragraph now sits between BEGIN and END markers, as the allocation report's sections do, and the test reads the kinds between them. It still requires exactly the kinds whose table entry has a declaration. --- docs/godot-pck.md | 27 ++ docs/hex-records.md | 5 +- docs/mcp-contracts.md | 43 ++ docs/python-packaged-applications.md | 19 +- docs/unityfs.md | 4 +- src/artifacts/hostExecutableMarking.test.ts | 28 +- src/artifacts/hostExecutableMarking.ts | 20 +- src/domain/artifactComparison.test.ts | 429 ++++++++++++++++-- src/domain/artifactComparison.ts | 301 ++++++------ src/domain/artifactFormatMetadata.ts | 277 ++++++++--- .../artifactComparisonCoverage.test.ts | 95 ++-- .../artifactComparisonAsarUnpacked.test.ts | 33 +- .../artifactComparisonEspImage.test.ts | 23 +- .../artifactComparisonGodotPck.test.ts | 147 ++++++ .../artifactComparisonNodeSea.test.ts | 35 +- .../artifactComparisonPyInstaller.test.ts | 119 +++++ .../artifactComparisonUnityFs.test.ts | 89 ++++ .../acceptance/artifactComparisonZip.test.ts | 132 ++++++ .../artifactComparisonDocs.test.ts | 31 ++ tests/fixtures/artifactComparisonMcp.ts | 37 ++ 20 files changed, 1505 insertions(+), 389 deletions(-) create mode 100644 tests/acceptance/artifactComparisonGodotPck.test.ts create mode 100644 tests/acceptance/artifactComparisonPyInstaller.test.ts create mode 100644 tests/acceptance/artifactComparisonUnityFs.test.ts create mode 100644 tests/acceptance/artifactComparisonZip.test.ts create mode 100644 tests/conformance/artifactComparisonDocs.test.ts create mode 100644 tests/fixtures/artifactComparisonMcp.ts diff --git a/docs/godot-pck.md b/docs/godot-pck.md index 19b6f1d938..748fcf0d2d 100644 --- a/docs/godot-pck.md +++ b/docs/godot-pck.md @@ -25,6 +25,33 @@ by default; `--integrity-policy record-and-continue` retains the observed bytes and an explicit untrusted contradiction. An all-zero MD5 is an unknown claim, not a verified digest. +Artifact comparison pairs members by `logical_path`, the stored path without +`res://`, and compares their bytes and occurrence facts. It does not compare +`godot-pck-entry` metadata. Godot 4.3 +[stores `res://` paths](https://github.com/godotengine/godot/blob/77dcf97d82cbfe4e4615475fa52ca03da645dbd8/editor/export/editor_export_platform.cpp#L228) +and [keys members by them](https://github.com/godotengine/godot/blob/77dcf97d82cbfe4e4615475fa52ca03da645dbd8/core/io/file_access_pack.cpp#L50-L52). +Godot 4.4.1 +[stores paths without the prefix](https://github.com/godotengine/godot/blob/49a5bc7b616bd04689a2c89e89bda41f50241464/editor/export/editor_export_platform.cpp#L257-L260) +and [removes it before keying](https://github.com/godotengine/godot/blob/49a5bc7b616bd04689a2c89e89bda41f50241464/core/io/file_access_pack.cpp#L48-L50). +Each version finds `res://scenes/main.gd` in its own export, but comparing the +stored `source_path` would mark every member changed between such exports. +`logical_path` names the member Godot's lookup finds, except in two cases. +REA normalizes Unicode to NFC, so a member renamed only between Unicode forms +compares unchanged, although Godot treats it as another path. Godot reads `\` +as `/` and REA keeps it literal, so a member spelled with backslashes compares +as removed and added. Directory order and offsets are layout. The encryption +bit, the only member flag REA admits, is the occurrence's `encrypted` field. A +member whose encryption changes compares `unknown` in `metadata` and +`availability`: the change is known, but REA does not read encrypted bytes, +so whether the script changed is unknown. Godot's exporter +[stores the MD5 of a member's original bytes](https://github.com/godotengine/godot/blob/a13da4feb8d8aefc283c3763d33a2f170a18d541/editor/export/editor_export_platform.cpp#L465-L468), +as the 4.3 and 4.4.1 exporters do, so the declared MD5 does not depend on the +encryption IV. Encrypted members whose declared MD5s differ therefore also +compare `unknown` in `metadata` and `availability`: the MD5s declare +different scripts, but REA cannot verify them, and equal MD5s add nothing for +the same reason. An encrypted member with an MD5 on only one side stays +`unknown`. + ## CLI Start with a compact summary and keep the complete Evidence bundle: diff --git a/docs/hex-records.md b/docs/hex-records.md index 86af9665c1..2cffe3b7a3 100644 --- a/docs/hex-records.md +++ b/docs/hex-records.md @@ -36,7 +36,10 @@ regions and are never filled. Region `format_metadata` (`hex-record-region`) records `start_address`, `end_address`, `length`, `record_count`, and `first_line`/`last_line`: the -1-based line span of the contributing data records. +1-based line span of the contributing data records. `compare_artifacts` +compares none of these fields: a region's name encodes its start and end +addresses, its length follows from them, and record and line counts are +layout. Root `format_metadata` (`hex-records`) records: diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index b4bf29bce2..8943ec4b2f 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -1564,6 +1564,49 @@ support for omitted historical values. known and differ. If only one value is known, storage comparison is unknown; when both are omitted, storage is not compared. + + +`compare_artifacts` also compares the declarations in the `format_metadata` +of UnityFS members (`unityfs-entry`), PyInstaller entries +(`pyinstaller-entry`), Node SEA members (`node-sea-entry`), +ESP image segments (`esp-image-segment`), DfuSe elements (`dfuse-element`), +ESP flash partitions (`esp-flash-partition`) and FIT images (`fit-image`); +each format's guide names the compared fields. A changed declaration is a +metadata change, including when content is unavailable. A declaration that +only one side records, or that the other side records as another kind, is +unknown and adds a limitation. A field that either side could not read is +unknown, and the other fields still compare. Other kinds are not compared: +their fields record layout, come from the bytes of the artifact or its +container or from caller options that identify the target, or repeat +occurrence fields, except Godot PCK source paths, which +[Godot PCK archives](godot-pck.md) explains. + + + +Every occurrence records `entry_kind`, `executable` and `encrypted`. A +different entry kind, `encrypted` value or occurrence limitation text is a +`metadata` difference, and a different `executable` value is an `executable` +difference. A symlink's own permission bits never mark it `executable`, +because running a symlink runs its target. When either side's content is +unavailable, `compare_artifacts` lists the differences that it can still +observe with `availability`, and the path compares `unknown` unless a +declaration changed. REA does not read symlink targets or encrypted bytes, +so a non-executable file that becomes a symlink compares `unknown` in +`metadata` and `availability`, and so does a member that becomes encrypted. +Declared sizes do not count without content: REA compares observed sizes. +For a node-less metadata record with a compared declaration, only that +declaration, these fields and any recorded ASAR storage decide the +comparison, so a record that becomes encrypted compares `changed` in +`metadata`. Such a record adds `availability` only when a declaration is +unknown. REA does not compare the record's unread payload or its +limitations, which describe the observation. + +When either side has no content, `compare_artifacts` also compares the +occurrences' `declared_integrity` claims, which Godot PCK and ASAR readers +record. Claims of one algorithm that differ add `metadata`. REA cannot verify +them, so they do not make the path `changed`, and equal claims add nothing. +With content on both sides, the observed bytes decide. + POSIX letter-colon names such as `a:b` remain valid logical names; Windows extraction refuses names its filesystem cannot materialize. diff --git a/docs/python-packaged-applications.md b/docs/python-packaged-applications.md index 53c0d8c353..d4d91f51bf 100644 --- a/docs/python-packaged-applications.md +++ b/docs/python-packaged-applications.md @@ -77,14 +77,17 @@ table index; original names remain available in the supplied artifact. Path failures use bounded escaped previews and report the original length; accepted paths and retained source names are not shortened. -`compare_artifacts` compares these node-less records as declarations using the -container, container name, source name and type code. Equal option or dependency -declarations can therefore compare unchanged without implying that an external -dependency's bytes were found. For materialized PyInstaller members, the same -fields participate in the metadata delta independently of content identity; -archive offsets and table positions remain location facts, not declaration -identity. Missing declaration facts and unavailable non-metadata content remain -unknown. +`compare_artifacts` compares node-less records, namely runtime options, +dependencies and namespace packages, as declarations using the container, +container name, source name and type code, and whether the record is +encrypted; their limitations are not compared. Equal declarations can +therefore compare unchanged without implying that an external dependency's +bytes were found or that unread legacy namespace payloads match. For +materialized PyInstaller members, the same fields participate in the metadata +delta independently of content identity, including when the member's bytes +are unavailable; archive offsets and table positions remain location facts, +not declaration identity. Missing declaration facts remain unknown, as does +unavailable non-metadata content whose declaration is unchanged. Request `detail: "summary"` for a compact inspection, then query retained Evidence with `inspect_analysis_view`. An inventory view can select diff --git a/docs/unityfs.md b/docs/unityfs.md index 963a6b218d..e9564508c9 100644 --- a/docs/unityfs.md +++ b/docs/unityfs.md @@ -10,7 +10,9 @@ strings, declared size, dataflags, blocks-and-directory table bounds, the producer's 16-byte data hash as declared, and the resolved data-area offset. Members (`CAB-*` SerializedFiles, `.resS` resource streams, shared assets) retain declared source paths, virtual stream offsets, sizes and raw node flags. -The directory flag identifies directories. Virtual offsets are not physical +The directory flag identifies directories. `compare_artifacts` compares each +member's declared `source_path` and raw `flags`; its index and offset are +layout. Virtual offsets are not physical carrier locations: member `source_location` is null. Member reads only examine their covering blocks. Uncompressed ranges stream directly; compressed ranges reuse at most one completed decoded block. Independent 64 KiB output chunks diff --git a/src/artifacts/hostExecutableMarking.test.ts b/src/artifacts/hostExecutableMarking.test.ts index 9f97186d24..ad1b39858c 100644 --- a/src/artifacts/hostExecutableMarking.test.ts +++ b/src/artifacts/hostExecutableMarking.test.ts @@ -2,8 +2,21 @@ import { describe, expect, it } from "vitest"; import { hostExecutableMarking } from "./hostExecutableMarking.js"; -const file = (mode: number) => ({ mode, isFile: () => true }); -const directory = (mode: number) => ({ mode, isFile: () => false }); +const file = (mode: number) => ({ + mode, + isFile: () => true, + isSymbolicLink: () => false, +}); +const directory = (mode: number) => ({ + mode, + isFile: () => false, + isSymbolicLink: () => false, +}); +const symlink = (mode: number) => ({ + mode, + isFile: () => false, + isSymbolicLink: () => true, +}); describe("hostExecutableMarking", () => { it("marks Windows launchable files by extension because NTFS has no execute bit", () => { @@ -48,4 +61,15 @@ describe("hostExecutableMarking", () => { hostExecutableMarking("/scratch/whoami.exe", file(0o644), "darwin"), ).toBe(false); }); + + it("never marks a symlink, whose target decides what runs", () => { + for (const [path, platform] of [ + ["/scratch/current", "linux"], + ["/scratch/current", "darwin"], + ["C:\\scratch\\tool.exe", "win32"], + ] as const) + expect(hostExecutableMarking(path, symlink(0o120777), platform)).toBe( + false, + ); + }); }); diff --git a/src/artifacts/hostExecutableMarking.ts b/src/artifacts/hostExecutableMarking.ts index 5549d099f6..14413c5965 100644 --- a/src/artifacts/hostExecutableMarking.ts +++ b/src/artifacts/hostExecutableMarking.ts @@ -9,16 +9,20 @@ const windowsLaunchableExtensions = new Set([".bat", ".cmd", ".com", ".exe"]); * * POSIX hosts mark executability with permission bits. Windows has no execute * bit (Node reports file modes from the read-only attribute alone), so it marks - * launchable files by extension instead. + * launchable files by extension instead. A symlink carries no marking on any + * host: running it runs its target, and its own permission bits do not + * control that (Linux reports 0777 for every symlink). */ export const hostExecutableMarking = ( path: string, - metadata: Pick, + metadata: Pick, platform: NodeJS.Platform = process.platform, ): boolean => - platform === "win32" - ? metadata.isFile() && - windowsLaunchableExtensions.has(extname(path).toLowerCase()) - : typeof metadata.mode === "bigint" - ? (metadata.mode & 0o111n) !== 0n - : (metadata.mode & 0o111) !== 0; + metadata.isSymbolicLink() + ? false + : platform === "win32" + ? metadata.isFile() && + windowsLaunchableExtensions.has(extname(path).toLowerCase()) + : typeof metadata.mode === "bigint" + ? (metadata.mode & 0o111n) !== 0n + : (metadata.mode & 0o111) !== 0; diff --git a/src/domain/artifactComparison.test.ts b/src/domain/artifactComparison.test.ts index 233deaf611..27c3e7cc2e 100644 --- a/src/domain/artifactComparison.test.ts +++ b/src/domain/artifactComparison.test.ts @@ -1,40 +1,25 @@ import { expect, it } from "vitest"; import { ARTIFACT_COMPARISON_EXAMPLE } from "../contracts/artifactComparisonExample.js"; -import { compareArtifacts } from "./artifactComparison.js"; +import { + compareArtifacts, + FORMAT_DECLARATION_UNKNOWN_LIMITATION, +} from "./artifactComparison.js"; import { artifactInventoryResultSchema, + type ArtifactInventoryResult, type ArtifactNode, type ArtifactOccurrence, } from "./artifactGraph.js"; -import { artifactGraphDigest, artifactManifestId } from "./artifactIdentity.js"; +import { + artifactGraphDigest, + artifactIdForContent, + artifactManifestId, + occurrenceIdForLocation, +} from "./artifactIdentity.js"; import { createEvidence } from "./evidence.js"; /** Recommit observation variants so comparisons exercise valid Evidence carriers. */ -const packingObservation = ( - unpacked: boolean | undefined, - hashStatus: ArtifactOccurrence["hash_status"] = "verified", - changedContent = false, - occurrenceChanges: Partial = {}, - nodeChanges: Partial = {}, -) => { - const source = changedContent - ? ARTIFACT_COMPARISON_EXAMPLE.right - : ARTIFACT_COMPARISON_EXAMPLE.left; - const inventory = artifactInventoryResultSchema.parse( - source.normalized_result, - ); - inventory.occurrences = inventory.occurrences.map((occurrence) => ({ - ...occurrence, - ...(unpacked === undefined ? {} : { unpacked }), - hash_status: hashStatus, - ...occurrenceChanges, - })); - if (occurrenceChanges.artifact_format !== undefined) - inventory.manifest.root_format = occurrenceChanges.artifact_format; - inventory.nodes = inventory.nodes.map((node) => ({ - ...node, - ...nodeChanges, - })); +const commit = (inventory: ArtifactInventoryResult) => { const graphSha256 = artifactGraphDigest({ nodes: inventory.nodes, occurrences: inventory.occurrences, @@ -52,7 +37,7 @@ const packingObservation = ( sha256: inventory.manifest.root_sha256, format: inventory.manifest.root_format, }, - source.provider, + ARTIFACT_COMPARISON_EXAMPLE.left.provider, { operation: "inventory_artifact", parameters: {}, @@ -63,6 +48,47 @@ const packingObservation = ( ); }; +const observation = ( + facts: Pick & Partial, + changedContent = false, + nodeChanges: Partial = {}, +) => { + const source = changedContent + ? ARTIFACT_COMPARISON_EXAMPLE.right + : ARTIFACT_COMPARISON_EXAMPLE.left; + const inventory = artifactInventoryResultSchema.parse( + source.normalized_result, + ); + inventory.occurrences = inventory.occurrences.map((occurrence) => ({ + ...occurrence, + ...facts, + })); + if (facts.artifact_format !== undefined) + inventory.manifest.root_format = facts.artifact_format; + inventory.nodes = inventory.nodes.map((node) => ({ + ...node, + ...nodeChanges, + })); + return commit(inventory); +}; + +const packingObservation = ( + unpacked: boolean | undefined, + hashStatus: ArtifactOccurrence["hash_status"] = "verified", + changedContent = false, + occurrenceChanges: Partial = {}, + nodeChanges: Partial = {}, +) => + observation( + { + ...(unpacked === undefined ? {} : { unpacked }), + hash_status: hashStatus, + ...occurrenceChanges, + }, + changedContent, + nodeChanges, + ); + it.each([ [{ artifact_kind: "unknown" }, { artifact_kind: "javascript" }, "verified"], [{ artifact_kind: "javascript" }, { artifact_kind: "unknown" }, "verified"], @@ -233,3 +259,350 @@ it("keeps integrity contradictions ahead of packing differences", () => { changes: [{ classification: "contradiction", dimensions: ["integrity"] }], }); }); + +type FormatMetadata = NonNullable; + +const declared = (formatMetadata: FormatMetadata) => + observation({ hash_status: "verified", format_metadata: formatMetadata }); + +/** One metadata record in a container; a content digit gives it a node. */ +const recordObservation = ( + formatMetadata: FormatMetadata | undefined, + record: { + readonly content?: string; + readonly encrypted?: boolean; + readonly executable?: boolean; + readonly limitations?: string[]; + readonly unpacked?: boolean; + } = {}, +) => { + const inventory = artifactInventoryResultSchema.parse( + ARTIFACT_COMPARISON_EXAMPLE.left.normalized_result, + ); + const root = inventory.occurrences[0]; + if (root === undefined) throw new Error("Missing root occurrence"); + const encrypted = record.encrypted ?? false; + const sha256 = record.content?.repeat(64); + const artifactId = sha256 === undefined ? null : artifactIdForContent(sha256); + if (sha256 !== undefined && artifactId !== null) + inventory.nodes.push({ + artifact_id: artifactId, + format: "file", + sha256, + size: 0, + media_type: null, + architecture: null, + content_state: "materialized", + limitations: [], + }); + inventory.occurrences.push({ + occurrence_id: occurrenceIdForLocation({ + rootArtifactId: inventory.manifest.root_artifact_id, + logicalPath: "record", + entryKind: "metadata", + }), + artifact_id: artifactId, + parent_occurrence_id: root.occurrence_id, + logical_path: "record", + entry_kind: "metadata", + artifact_kind: "package-metadata", + artifact_format: "unknown", + declared_size: 0, + compressed_size: null, + executable: record.executable ?? false, + encrypted, + hash_status: + sha256 !== undefined + ? "verified" + : encrypted + ? "unavailable" + : "not-hashed", + source_location: null, + ...(formatMetadata === undefined + ? {} + : { format_metadata: formatMetadata }), + limitations: record.limitations ?? [], + ...(record.unpacked === undefined ? {} : { unpacked: record.unpacked }), + }); + inventory.coverage = { scope: "container-entries", status: "complete" }; + inventory.manifest.node_count = inventory.nodes.length; + inventory.manifest.occurrence_count = inventory.occurrences.length; + return commit(inventory); +}; + +const espImageSegment: FormatMetadata = { + kind: "esp-image-segment", + index: 1, + load_addr: 0x3fc8_8000, + data_len: 4, + header_offset: 24, + classification: "loaded", +}; +const movedSegment: FormatMetadata = { + ...espImageSegment, + load_addr: 0x3fc8_a000, +}; +const runtimeOption: FormatMetadata = { + kind: "pyinstaller-entry", + container: "carchive", + container_name: null, + index: 1, + source_name: "pyi-contents-directory _internal", + type_code: "o", + relative_offset: 0, + directory_record_offset: null, + directory_record_size: null, + compression_flag: 0, + bytecode_header: "not-bytecode", + bytecode_magic: null, + pyz_status: "not-pyz", + pyz_sha256: null, + pyz_python_magic: null, + pyz_python_version: null, + pyz_entry_count: null, + encryption_source: null, +}; + +it("compares an ESP segment declaration without its layout or derived class", () => { + expect( + compareArtifacts( + declared(espImageSegment), + declared({ + ...espImageSegment, + header_offset: 64, + classification: "unknown", + }), + ), + ).toMatchObject({ status: "unchanged", changes: [] }); + expect( + compareArtifacts(declared(espImageSegment), declared(movedSegment)), + ).toMatchObject({ + status: "changed", + changes: [{ classification: "changed", dimensions: ["metadata"] }], + }); +}); + +it("retains a changed declaration when content is unavailable", () => { + expect( + compareArtifacts( + declared(espImageSegment), + observation({ + hash_status: "unavailable", + format_metadata: movedSegment, + }), + ), + ).toMatchObject({ + status: "changed", + changes: [ + { classification: "changed", dimensions: ["metadata", "availability"] }, + ], + }); + // Declared dimensions keep the schema order, ahead of availability. + expect( + compareArtifacts( + declared(espImageSegment), + observation({ + hash_status: "unavailable", + format_metadata: movedSegment, + executable: true, + }), + ).changes, + ).toMatchObject([ + { + classification: "changed", + dimensions: ["executable", "metadata", "availability"], + }, + ]); +}); + +it.each([ + ["one side omits it", undefined], + ["the other side records another kind", runtimeOption], +] as const)("keeps a declaration unknown when %s", (_case, other) => { + const result = compareArtifacts( + declared(espImageSegment), + other === undefined + ? observation({ hash_status: "verified" }) + : declared(other), + ); + expect(result).toMatchObject({ + status: "unknown", + changes: [{ classification: "unknown", dimensions: ["availability"] }], + }); + expect(result.limitations).toEqual([FORMAT_DECLARATION_UNKNOWN_LIMITATION]); +}); + +it("does not compare ESP signature block metadata", () => { + const block: FormatMetadata = { + kind: "esp-image-signature-block", + index: 0, + version: 2, + scheme: "rsa", + sha_version: null, + curve_id: null, + digest_algorithm: "sha256", + image_digest: "a".repeat(64), + image_digest_matches: true, + public_key_sha256: "b".repeat(64), + signature_status: "unverified", + }; + for (const other of [ + declared({ ...block, image_digest_matches: false }), + observation({ hash_status: "verified" }), + ]) + expect(compareArtifacts(declared(block), other)).toMatchObject({ + status: "unchanged", + changes: [], + limitations: [], + }); +}); + +it("does not compare HEX region metadata, which the region's path encodes", () => { + const region: FormatMetadata = { + kind: "hex-record-region", + start_address: 0x0800_0000, + end_address: 0x0800_001f, + length: 32, + record_count: 2, + first_line: 1, + last_line: 2, + }; + // Record and line counts are layout, and equal bytes stay unchanged beside + // an observation without the metadata, such as an extracted region. + for (const other of [ + declared({ ...region, record_count: 4, first_line: 3, last_line: 6 }), + observation({ hash_status: "verified" }), + ]) + expect(compareArtifacts(declared(region), other)).toMatchObject({ + status: "unchanged", + changes: [], + limitations: [], + }); +}); + +it("compares a node-less record of any compared kind by its declaration", () => { + // No reader emits such a record; the rule follows the table, not a format. + expect( + compareArtifacts( + recordObservation(espImageSegment), + recordObservation(espImageSegment), + ), + ).toMatchObject({ status: "unchanged", changes: [] }); + expect( + compareArtifacts( + recordObservation(espImageSegment), + recordObservation(movedSegment), + ).changes, + ).toEqual([ + expect.objectContaining({ + logical_path: "record", + classification: "changed", + dimensions: ["metadata"], + }), + ]); +}); + +it("compares node-less records by declaration and keeps content changes", () => { + expect( + compareArtifacts( + recordObservation(runtimeOption), + recordObservation(runtimeOption), + ), + ).toMatchObject({ status: "unchanged", changes: [] }); + expect( + compareArtifacts( + recordObservation(runtimeOption), + recordObservation({ ...runtimeOption, source_name: "pyi-python-flag O" }), + ).changes, + ).toEqual([ + expect.objectContaining({ + logical_path: "record", + classification: "changed", + dimensions: ["metadata"], + }), + ]); + // Equal declarations do not hide a change in content that a record names. + expect( + compareArtifacts( + recordObservation(runtimeOption, { content: "a" }), + recordObservation(runtimeOption, { content: "b" }), + ).changes, + ).toEqual([ + expect.objectContaining({ + logical_path: "record", + classification: "changed", + dimensions: ["content"], + }), + ]); +}); + +it("compares a node-less record's occurrence fields but not its limitations", () => { + // A PYZ namespace record and the reader's caveat for it. + const namespace: FormatMetadata = { + ...runtimeOption, + container: "pyz", + container_name: "PYZ.pyz", + source_name: "legacy", + type_code: 3, + }; + const caveat = + "Namespace package has no importable bytecode; its declared payload is not materialized."; + const plain = recordObservation(namespace, { limitations: [caveat] }); + // A known encryption change survives a declaration that one side omits. + const result = compareArtifacts( + plain, + recordObservation(undefined, { encrypted: true, limitations: [caveat] }), + ); + expect(result.changes).toEqual([ + expect.objectContaining({ + logical_path: "record", + classification: "changed", + dimensions: ["metadata", "availability"], + }), + ]); + expect(result.limitations).toEqual([FORMAT_DECLARATION_UNKNOWN_LIMITATION]); + // A record's `executable` marking is a declared field like `encrypted`. + expect( + compareArtifacts( + plain, + recordObservation(namespace, { executable: true, limitations: [caveat] }), + ).changes, + ).toEqual([ + expect.objectContaining({ + logical_path: "record", + classification: "changed", + dimensions: ["executable"], + }), + ]); + // Limitations describe the observation, so a caveat alone is no change. + expect(compareArtifacts(plain, recordObservation(namespace))).toMatchObject({ + status: "unchanged", + changes: [], + }); +}); + +it("compares a node-less record's recorded ASAR storage", () => { + // No reader records storage on a metadata record. A recorded value is a + // declared fact like `encrypted`, and an omitted one stays unknown. + expect( + compareArtifacts( + recordObservation(runtimeOption, { unpacked: false }), + recordObservation(runtimeOption, { unpacked: true }), + ).changes, + ).toEqual([ + expect.objectContaining({ + logical_path: "record", + classification: "changed", + dimensions: ["metadata"], + }), + ]); + expect( + compareArtifacts( + recordObservation(runtimeOption, { unpacked: true }), + recordObservation(runtimeOption), + ), + ).toMatchObject({ + status: "unknown", + changes: [{ classification: "unknown", dimensions: ["availability"] }], + }); +}); diff --git a/src/domain/artifactComparison.ts b/src/domain/artifactComparison.ts index 214b68cbe6..9736646f80 100644 --- a/src/domain/artifactComparison.ts +++ b/src/domain/artifactComparison.ts @@ -14,12 +14,8 @@ import { type InventorySet, } from "./artifactInventoryEvidence.js"; import { - dfuSeElementDeclaration, - espFlashPartitionDeclaration, - espImageSegmentDeclaration, - fitImageDeclaration, - nodeSeaEntryDeclaration, - pyInstallerEntryDeclaration, + formatMetadataDeclaration, + type FormatMetadataDeclaration, } from "./artifactFormatMetadata.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; import { canonicalJson } from "./comparisonSemantics.js"; @@ -100,6 +96,10 @@ export type ArtifactComparisonResult = z.infer< >; type ArtifactChange = z.infer; +/** Comparison limitation when a path's format declarations cannot be compared. */ +export const FORMAT_DECLARATION_UNKNOWN_LIMITATION = + "Format metadata declarations are missing on one side or differ in kind for matching paths; omitted metadata may be a legacy or non-container observation and leaves the complete declaration unavailable."; + const comparisonInventory = (input: unknown, side: "left" | "right") => { try { return parseArtifactInventoryEvidence(input); @@ -186,10 +186,8 @@ export const compareArtifacts = ( "ASAR packing metadata is recorded on only one side for matching paths; omitted metadata may be a legacy observation and does not establish packed or unpacked storage.", ] : []), - ...(comparison.hasUnknownNodeSeaMetadata - ? [ - "Node SEA member metadata is recorded on only one side for matching paths; omitted metadata may be a legacy observation and leaves the complete runtime member declaration unavailable.", - ] + ...(comparison.hasUnknownFormatDeclaration + ? [FORMAT_DECLARATION_UNKNOWN_LIMITATION] : []), ...(comparison.hasAmbiguousDuplicatePaths ? [ @@ -252,7 +250,7 @@ const compareOccurrences = ( readonly changes: ArtifactChange[]; readonly hasAmbiguousDuplicatePaths: boolean; readonly hasUnknownAsarPacking: boolean; - readonly hasUnknownNodeSeaMetadata: boolean; + readonly hasUnknownFormatDeclaration: boolean; } => { const leftNodes = new Map(left.nodes.map((item) => [item.artifact_id, item])); const rightNodes = new Map( @@ -266,7 +264,7 @@ const compareOccurrences = ( const output: ArtifactChange[] = []; let hasAmbiguousDuplicatePaths = false; let hasUnknownAsarPacking = false; - let hasUnknownNodeSeaMetadata = false; + let hasUnknownFormatDeclaration = false; for (const path of paths) { const leftMatches = leftByPath.get(path) ?? []; const rightMatches = rightByPath.get(path) ?? []; @@ -289,7 +287,8 @@ const compareOccurrences = ( const metadata = occurrenceMetadata({ leftOccurrence, rightOccurrence }); if (leftOccurrence !== undefined && rightOccurrence !== undefined) { if (metadata.asarPacking === "unknown") hasUnknownAsarPacking = true; - if (metadata.nodeSea === "unknown") hasUnknownNodeSeaMetadata = true; + if (metadata.formatDeclaration === "unknown") + hasUnknownFormatDeclaration = true; } const change = classifyPath({ metadata, @@ -310,7 +309,7 @@ const compareOccurrences = ( changes: output, hasAmbiguousDuplicatePaths, hasUnknownAsarPacking, - hasUnknownNodeSeaMetadata, + hasUnknownFormatDeclaration, }; }; @@ -407,7 +406,7 @@ const classifyPath = (input: ClassifyPathInput): ArtifactChange | null => { if (integrity !== null) return integrity; const presence = classifyPresence(input, base); if (presence !== null) return presence; - const declaration = classifyPyInstallerDeclaration(input, base); + const declaration = classifyDeclarationRecord(input, base); if (declaration.handled) return declaration.change; const availability = classifyAvailability(input, base); if (availability !== null) { @@ -417,7 +416,8 @@ const classifyPath = (input: ClassifyPathInput): ArtifactChange | null => { if (known.format) dimensions.push("format"); if (known.executable) dimensions.push("executable"); if (known.relations) dimensions.push("relations"); - if (known.metadata) dimensions.push("metadata"); + if (known.metadata || declaredDigestsDiffer(input)) + dimensions.push("metadata"); dimensions.push("availability"); return { ...base, @@ -430,7 +430,7 @@ const classifyPath = (input: ClassifyPathInput): ArtifactChange | null => { const dimensions = changedDimensions(input); const occurrence = occurrenceDifferences(input); if ( - Object.values(input.metadata).includes("unknown") || + declarationUnknown(input.metadata) || occurrence.kindUnknown || occurrence.formatUnknown || differsWithUnknown(input.leftNode?.format, input.rightNode?.format) || @@ -464,24 +464,31 @@ type OccurrencePair = Pick< type MetadataComparison = "unchanged" | "changed" | "unknown"; interface OccurrenceMetadata { + /** ASAR storage is an occurrence field, not format metadata. */ readonly asarPacking: MetadataComparison; - readonly nodeSea: MetadataComparison; - readonly espImageSegment: MetadataComparison; - readonly espFlashPartition: MetadataComparison; - readonly fitImage: MetadataComparison; - readonly fitImageAvailability: MetadataComparison; - readonly dfuSeElement: MetadataComparison; - readonly pyInstaller: MetadataComparison; + /** Every occurrence records `encrypted`, so a pair's comparison is known. */ + readonly encryption: Exclude; + /** Absent when neither occurrence records a compared declaration. */ + readonly formatDeclaration: MetadataComparison | "absent"; + /** Unknown when either side could not read a compared field. */ + readonly formatDeclarationAvailability: Exclude< + MetadataComparison, + "changed" + >; } /** A declaration that differs even when content is unavailable. */ const declarationChanged = (metadata: OccurrenceMetadata): boolean => metadata.asarPacking === "changed" || - metadata.nodeSea === "changed" || - metadata.espImageSegment === "changed" || - metadata.espFlashPartition === "changed" || - metadata.fitImage === "changed" || - metadata.dfuSeElement === "changed"; + metadata.formatDeclaration === "changed"; + +/** A declared fact that differs: a declaration or the encryption. */ +const declaredFactChanged = (metadata: OccurrenceMetadata): boolean => + Object.values(metadata).includes("changed"); + +/** A declared fact that one side omits, records as another kind or cannot read. */ +const declarationUnknown = (metadata: OccurrenceMetadata): boolean => + Object.values(metadata).includes("unknown"); /** Unknown classifications and absent node metadata cannot establish a difference. */ const differsWithUnknown = ( @@ -536,174 +543,134 @@ const occurrenceDifferences = (input: { JSON.stringify(input.leftRelations) !== JSON.stringify(input.rightRelations), metadata: - declarationChanged(input.metadata) || - input.metadata.pyInstaller === "changed" || + declaredFactChanged(input.metadata) || input.leftOccurrence?.entry_kind !== input.rightOccurrence?.entry_kind || - input.leftOccurrence?.encrypted !== input.rightOccurrence?.encrypted || JSON.stringify(input.leftOccurrence?.limitations) !== JSON.stringify(input.rightOccurrence?.limitations), }); -const compareOptionalMetadata = ( - left: T | undefined, - right: T | undefined, - declaration: (value: T) => unknown, -): MetadataComparison => { - if (left === undefined || right === undefined) - return left === right ? "unchanged" : "unknown"; - return canonicalJson(declaration(left)) === canonicalJson(declaration(right)) +const compareAsarPacking = ( + left: boolean | undefined, + right: boolean | undefined, +): MetadataComparison => + left === right ? "unchanged" - : "changed"; + : left === undefined || right === undefined + ? "unknown" + : "changed"; + +/** + * Declarations compare within one kind. A declaration that only one side + * records, including one whose other side records another kind, is unknown. + * A field that either side could not read is unknown, and the other fields + * still compare. + */ +const compareFormatDeclarations = ( + left: FormatMetadataDeclaration | undefined, + right: FormatMetadataDeclaration | undefined, +): Pick< + OccurrenceMetadata, + "formatDeclaration" | "formatDeclarationAvailability" +> => { + if (left === undefined && right === undefined) + return { + formatDeclaration: "absent", + formatDeclarationAvailability: "unchanged", + }; + if (left === undefined || right === undefined || left.kind !== right.kind) + return { + formatDeclaration: "unknown", + formatDeclarationAvailability: "unchanged", + }; + const unreadable = new Set([...left.unreadable, ...right.unreadable]); + const readable = (facts: FormatMetadataDeclaration["facts"]) => + Object.fromEntries( + Object.entries(facts).filter(([field]) => !unreadable.has(field)), + ); + return { + formatDeclaration: + canonicalJson(readable(left.facts)) === + canonicalJson(readable(right.facts)) + ? "unchanged" + : "changed", + formatDeclarationAvailability: Object.keys(left.facts).some((field) => + unreadable.has(field), + ) + ? "unknown" + : "unchanged", + }; }; -/** Interpret each pair's format declarations once, preserving historical omission. */ +/** Interpret each pair's declarations once, preserving historical omission. */ const occurrenceMetadata = (input: OccurrencePair): OccurrenceMetadata => { const left = input.leftOccurrence; const right = input.rightOccurrence; - const fit = compareFitImageDeclarations(input); return { - fitImage: fit.declaration, - fitImageAvailability: fit.availability, - asarPacking: compareOptionalMetadata( - left?.unpacked, - right?.unpacked, - (value) => value, - ), - nodeSea: compareOptionalMetadata( - left?.format_metadata?.kind === "node-sea-entry" - ? left.format_metadata - : undefined, - right?.format_metadata?.kind === "node-sea-entry" - ? right.format_metadata - : undefined, - nodeSeaEntryDeclaration, - ), - espImageSegment: compareOptionalMetadata( - left?.format_metadata?.kind === "esp-image-segment" - ? left.format_metadata - : undefined, - right?.format_metadata?.kind === "esp-image-segment" - ? right.format_metadata - : undefined, - espImageSegmentDeclaration, - ), - dfuSeElement: compareOptionalMetadata( - left?.format_metadata?.kind === "dfuse-element" - ? left.format_metadata - : undefined, - right?.format_metadata?.kind === "dfuse-element" - ? right.format_metadata - : undefined, - dfuSeElementDeclaration, - ), - espFlashPartition: compareOptionalMetadata( - left?.format_metadata?.kind === "esp-flash-partition" - ? left.format_metadata - : undefined, - right?.format_metadata?.kind === "esp-flash-partition" - ? right.format_metadata - : undefined, - espFlashPartitionDeclaration, - ), - pyInstaller: compareOptionalMetadata( - isPyInstallerEntry(left) ? left.format_metadata : undefined, - isPyInstallerEntry(right) ? right.format_metadata : undefined, - pyInstallerEntryDeclaration, + asarPacking: compareAsarPacking(left?.unpacked, right?.unpacked), + encryption: left?.encrypted === right?.encrypted ? "unchanged" : "changed", + ...compareFormatDeclarations( + formatMetadataDeclaration(left?.format_metadata), + formatMetadataDeclaration(right?.format_metadata), ), }; }; -/** Keep known FIT changes even when another declaration is unavailable. */ -const compareFitImageDeclarations = ({ - leftOccurrence: left, - rightOccurrence: right, -}: OccurrencePair): { - readonly declaration: MetadataComparison; - readonly availability: MetadataComparison; -} => { - const lm = - left?.format_metadata?.kind === "fit-image" - ? left.format_metadata - : undefined; - const rm = - right?.format_metadata?.kind === "fit-image" - ? right.format_metadata - : undefined; - if (lm === undefined || rm === undefined) { - const result = compareOptionalMetadata(lm, rm, fitImageDeclaration); - return { - declaration: result, - availability: result === "unknown" ? "unknown" : "unchanged", - }; - } - const ld = fitImageDeclaration(lm); - const rd = fitImageDeclaration(rm); - const keys = Object.keys(ld) as (keyof typeof ld)[]; - let changed = false; - let unavailable = false; - for (const key of keys) { - if ( - lm.unreadable_fields.some((field) => field === key) || - rm.unreadable_fields.some((field) => field === key) - ) { - unavailable = true; - continue; - } - if (ld[key] !== rd[key]) changed = true; - } - return { - declaration: changed ? "changed" : "unchanged", - availability: unavailable ? "unknown" : "unchanged", - }; -}; - -const isPyInstallerEntry = ( - occurrence: ArtifactOccurrence | undefined, -): occurrence is ArtifactOccurrence & { - readonly format_metadata: Extract< - NonNullable, - { readonly kind: "pyinstaller-entry" } - >; -} => occurrence?.format_metadata?.kind === "pyinstaller-entry"; - -const classifyPyInstallerDeclaration = ( +/** + * A node-less metadata record has no content node, so its declared facts are + * all that comparison checks; an unread payload is not compared. Neither are + * its limitations, which readers derive from those facts. + */ +const classifyDeclarationRecord = ( input: ClassifyPathInput, base: ReturnType, ): { readonly handled: boolean; readonly change: ArtifactChange | null } => { - const leftIsDeclaration = input.leftOccurrence?.entry_kind === "metadata"; - const rightIsDeclaration = input.rightOccurrence?.entry_kind === "metadata"; - if (!leftIsDeclaration && !rightIsDeclaration) - return { handled: false, change: null }; - // Metadata is not inherently a PyInstaller declaration. Only apply this - // format-specific comparison when a producer explicitly identified one. + // Metadata is not inherently a declaration. Only apply this comparison + // when a producer explicitly recorded a compared one. A record that names + // content keeps the content comparison. if ( - !isPyInstallerEntry(input.leftOccurrence) && - !isPyInstallerEntry(input.rightOccurrence) + input.leftOccurrence?.entry_kind !== "metadata" || + input.rightOccurrence?.entry_kind !== "metadata" || + input.leftOccurrence.artifact_id !== null || + input.rightOccurrence.artifact_id !== null || + input.metadata.formatDeclaration === "absent" ) return { handled: false, change: null }; - if (!leftIsDeclaration || !rightIsDeclaration) - return { handled: false, change: null }; - if ( - !isPyInstallerEntry(input.leftOccurrence) || - !isPyInstallerEntry(input.rightOccurrence) - ) + const dimensions: ArtifactChange["dimensions"] = []; + if (input.leftOccurrence.executable !== input.rightOccurrence.executable) + dimensions.push("executable"); + if (declaredFactChanged(input.metadata)) dimensions.push("metadata"); + const unknown = declarationUnknown(input.metadata); + if (dimensions.length === 0) return { handled: true, - change: { - ...base, - classification: "unknown", - dimensions: ["availability"], - }, + change: unknown + ? { ...base, classification: "unknown", dimensions: ["availability"] } + : null, }; - const changed = input.metadata.pyInstaller === "changed"; + if (unknown) dimensions.push("availability"); return { handled: true, - change: changed - ? { ...base, classification: "changed", dimensions: ["metadata"] } - : null, + change: { ...base, classification: "changed", dimensions }, }; }; +/** + * Without observed content on both sides, the containers' declared digests + * are the remaining content evidence. Claims of one algorithm that differ + * are a metadata difference, but they are unverified, so they do not make + * the path changed; equal claims add nothing. Observed bytes replace both. + */ +const declaredDigestsDiffer = (input: OccurrencePair): boolean => { + const left = input.leftOccurrence?.declared_integrity; + const right = input.rightOccurrence?.declared_integrity; + return ( + left !== undefined && + right !== undefined && + left.algorithm === right.algorithm && + left.digest !== right.digest + ); +}; + const changedDimensions = (input: { readonly metadata: OccurrenceMetadata; readonly leftNode: ArtifactNode | undefined; diff --git a/src/domain/artifactFormatMetadata.ts b/src/domain/artifactFormatMetadata.ts index d6f8e95479..a53a6c0524 100644 --- a/src/domain/artifactFormatMetadata.ts +++ b/src/domain/artifactFormatMetadata.ts @@ -235,14 +235,6 @@ export const dfuSeElementMetadataSchema = z.strictObject({ length: offset, }); -/** The device memory an element targets, which its path and bytes do not show. */ -export const dfuSeElementDeclaration = ( - metadata: z.infer, -) => ({ - alternate_setting: metadata.alternate_setting, - target_name: metadata.target_name, -}); - /** * Intel HEX or Motorola S-record facts derived from every record. Start * addresses are declared values, not proven entry points. @@ -359,23 +351,6 @@ export const fitImageMetadataSchema = z.strictObject({ signature_algorithms: z.array(z.string()), }); export type FitImageMetadata = z.infer; -/** Payload interpretation and sharing declarations, independent of FIT placement. */ -export const fitImageDeclaration = (metadata: FitImageMetadata) => ({ - type: metadata.type, - arch: metadata.arch, - os: metadata.os, - compression: metadata.compression, - load: metadata.load, - entry: metadata.entry, - image_data: metadata.image_data, - cipher_present: metadata.cipher_present, - cipher_algo: metadata.cipher_algo, -}); - -/** Runtime member identity, independent of blob layout and directory order. */ -export const nodeSeaEntryDeclaration = ( - metadata: z.infer, -) => ({ role: metadata.role, source_path: metadata.source_path }); export const uint8 = z.number().int().min(0).max(0xff); const uint16 = z.number().int().min(0).max(0xffff); @@ -509,14 +484,6 @@ export type EspImageSegmentMetadata = z.infer< typeof espImageSegmentMetadataSchema >; -/** - * What a segment header declares: where its bytes load and how many there - * are. Header offsets are layout, and the classification is derived. - */ -export const espImageSegmentDeclaration = ( - metadata: EspImageSegmentMetadata, -) => ({ load_addr: metadata.load_addr, data_len: metadata.data_len }); - /** One Secure Boot v2 block with its 0xE7 magic and a matching CRC32. */ export const espImageSignatureBlockMetadataSchema = z.strictObject({ kind: z.literal("esp-image-signature-block"), @@ -576,27 +543,6 @@ export const espFlashPartitionMetadataSchema = z.strictObject({ written_end: uint32.nullable(), written_bytes: uint32.nullable(), }); -export type EspFlashPartitionMetadata = z.infer< - typeof espFlashPartitionMetadataSchema ->; - -/** - * What a partition entry declares; its index is layout, and the written - * fields are observed from its bytes, which content comparison covers. - */ -export const espFlashPartitionDeclaration = ( - metadata: EspFlashPartitionMetadata, -) => ({ - label: metadata.label, - label_hex: metadata.label_hex, - type: metadata.type, - subtype: metadata.subtype, - flash_address: metadata.flash_address, - size: metadata.size, - encrypted: metadata.encrypted, - readonly: metadata.readonly, - unknown_flags: metadata.unknown_flags, -}); /** ESP image facts for a region of a dump; offsets are relative to the region. */ export const espFlashImageFactsSchema = z.strictObject({ @@ -721,16 +667,6 @@ export const espFlashImageMetadataSchema = z.strictObject({ }); export type EspFlashImageMetadata = z.infer; -/** Declaration identity that is stable across archive layout changes. */ -export const pyInstallerEntryDeclaration = ( - metadata: PyInstallerEntryMetadata, -) => ({ - container: metadata.container, - container_name: metadata.container_name, - source_name: metadata.source_name, - type_code: metadata.type_code, -}); - /** * U-Boot legacy uImage header facts. Declared addresses are facts from the * header, not proven entry points; CRCs are verified before inventory. @@ -855,3 +791,216 @@ export const artifactFormatMetadataSchema = z.discriminatedUnion("kind", [ export type ArtifactFormatMetadata = z.infer< typeof artifactFormatMetadataSchema >; + +type FormatMetadataByKind = { + readonly [Metadata in ArtifactFormatMetadata as Metadata["kind"]]: Metadata; +}; +type FormatMetadataKind = keyof FormatMetadataByKind; + +/** + * What each kind declares beyond its own bytes, which artifact comparison + * compares, or why comparison uses none of its facts. Every kind needs an + * entry, so a new kind does not compile without this decision; a new field + * of an existing kind still needs a review of its projection. For a node-less + * metadata record, the projection is all of its format metadata that + * comparison checks. + */ +const declarationsByKind: { + readonly [Kind in FormatMetadataKind]: + | { + readonly declaration: ( + metadata: FormatMetadataByKind[Kind], + ) => FormatMetadataDeclaration["facts"]; + /** Projected fields the producer could not interpret. */ + readonly unreadable?: ( + metadata: FormatMetadataByKind[Kind], + ) => FormatMetadataDeclaration["unreadable"]; + } + | { readonly notCompared: string }; +} = { + "godot-pck": { + notCompared: + "Its fields come from the pack's own bytes, which content comparison covers.", + }, + "godot-pck-entry": { + notCompared: + "index and declared_offset are layout; flags and md5_status repeat occurrence facts; source_path spells Godot's lookup path in a version-dependent way. docs/godot-pck.md gives the evidence.", + }, + unityfs: { + notCompared: + "Its fields come from the bundle's own bytes, which content comparison covers.", + }, + /** What the directory declares for a member: its source path and raw node flags. Index and offset are layout. */ + "unityfs-entry": { + declaration: (metadata) => ({ + source_path: metadata.source_path, + flags: metadata.flags, + }), + }, + pyinstaller: { + notCompared: + "Its fields come from the archive's own bytes, which content comparison covers.", + }, + /** Declaration identity that is stable across archive layout changes. */ + "pyinstaller-entry": { + declaration: (metadata) => ({ + container: metadata.container, + container_name: metadata.container_name, + source_name: metadata.source_name, + type_code: metadata.type_code, + }), + }, + "dotnet-bundle-entry": { + notCompared: + "Its fields come from the host's bundle manifest, whose bytes the host's content comparison covers.", + }, + "bun-standalone": { + notCompared: + "Its fields come from the carrier's own bytes, which content comparison covers.", + }, + "bun-standalone-entry": { + notCompared: + "Its fields come from the carrier's module graph, whose bytes the carrier's content comparison covers.", + }, + "node-sea": { + notCompared: + "Its fields come from the carrier's own bytes, which content comparison covers.", + }, + /** Runtime member identity, independent of blob layout and directory order. */ + "node-sea-entry": { + declaration: (metadata) => ({ + role: metadata.role, + source_path: metadata.source_path, + }), + }, + "hex-records": { + notCompared: + "Its fields come from the file's own records, which content comparison covers.", + }, + "hex-record-region": { + notCompared: + "start_address and end_address restate the region's path, and length follows from them; record and line counts are layout.", + }, + uimage: { + notCompared: + "Its header facts come from the image's own bytes, which content comparison covers.", + }, + "uimage-entry": { + notCompared: + "role and index restate the entry's path; its stored bytes are compared by content.", + }, + "esp-image": { + notCompared: + "Its fields come from the image's own bytes, which content comparison covers.", + }, + /** + * What a segment header declares: where its bytes load and how many there + * are. Header offsets are layout, and the classification is derived. + */ + "esp-image-segment": { + declaration: (metadata) => ({ + load_addr: metadata.load_addr, + data_len: metadata.data_len, + }), + }, + "esp-image-signature-block": { + notCompared: + "Its fields come from the block's own bytes, which content comparison covers, and image_digest_matches follows the image's content.", + }, + uf2: { + notCompared: + "Its counts come from the file's own blocks, which content comparison covers, and family_name_registry names REA's label registry.", + }, + "uf2-region": { + notCompared: + "Its familyID and addresses are in its path, family_name is REA's label for the familyID, and its block counts and positions are layout; content comparison covers its bytes.", + }, + dfuse: { + notCompared: + "Its fields come from the file's own prefix, target headers and suffix, which content comparison covers.", + }, + /** The device memory an element targets, which its path and bytes do not show. */ + "dfuse-element": { + declaration: (metadata) => ({ + alternate_setting: metadata.alternate_setting, + target_name: metadata.target_name, + }), + }, + "esp-flash-image": { + notCompared: + "Its fields come from the dump's bytes and from the caller's flash_offset and partition_table_offset, which are target identity; each partition's range, compared by content, reflects them.", + }, + /** + * What a partition entry declares; its index is layout, and the written + * fields are observed from its bytes, which content comparison covers. + */ + "esp-flash-partition": { + declaration: (metadata) => ({ + label: metadata.label, + label_hex: metadata.label_hex, + type: metadata.type, + subtype: metadata.subtype, + flash_address: metadata.flash_address, + size: metadata.size, + encrypted: metadata.encrypted, + readonly: metadata.readonly, + unknown_flags: metadata.unknown_flags, + }), + }, + fit: { + notCompared: + "Its fields come from the FIT's own bytes, which content comparison covers.", + }, + /** + * Payload interpretation and sharing declarations, independent of FIT + * placement: the image's name, index and data position are layout. + */ + "fit-image": { + declaration: (metadata) => ({ + type: metadata.type, + arch: metadata.arch, + os: metadata.os, + compression: metadata.compression, + load: metadata.load, + entry: metadata.entry, + image_data: metadata.image_data, + cipher_present: metadata.cipher_present, + cipher_algo: metadata.cipher_algo, + }), + unreadable: (metadata) => metadata.unreadable_fields, + }, +}; + +/** Kinds whose declarations artifact comparison compares, in table order. */ +export const COMPARED_FORMAT_METADATA_KINDS = Object.entries( + declarationsByKind, +).flatMap(([kind, entry]) => ("declaration" in entry ? [kind] : [])); + +/** A declaration that artifact comparison compares within its kind. */ +export interface FormatMetadataDeclaration { + readonly kind: FormatMetadataKind; + readonly facts: Readonly>; + /** Facts whose value is unknown, which compare as neither equal nor changed. */ + readonly unreadable: readonly string[]; +} + +// Generic in the kind, so each projection receives its own metadata type. +const declarationOf = ( + kind: Kind, + metadata: FormatMetadataByKind[Kind], +): FormatMetadataDeclaration | undefined => { + const entry = declarationsByKind[kind]; + return "declaration" in entry + ? { + kind, + facts: entry.declaration(metadata), + unreadable: entry.unreadable?.(metadata) ?? [], + } + : undefined; +}; + +/** The compared declaration, or undefined when comparison uses none. */ +export const formatMetadataDeclaration = ( + metadata: ArtifactFormatMetadata | undefined, +): FormatMetadataDeclaration | undefined => + metadata === undefined ? undefined : declarationOf(metadata.kind, metadata); diff --git a/tests/acceptance/applications/artifactComparisonCoverage.test.ts b/tests/acceptance/applications/artifactComparisonCoverage.test.ts index e84138a9a8..8b22fc310c 100644 --- a/tests/acceptance/applications/artifactComparisonCoverage.test.ts +++ b/tests/acceptance/applications/artifactComparisonCoverage.test.ts @@ -1,4 +1,4 @@ -import { chmod, mkdir, writeFile } from "node:fs/promises"; +import { chmod, mkdir, symlink, writeFile } from "node:fs/promises"; import { createServer } from "node:net"; import { join } from "node:path"; import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; @@ -9,7 +9,6 @@ import { artifactInventoryResultSchema, type ArtifactInventoryResult, } from "../../../src/domain/artifactGraph.js"; -import { artifactInspectionResultSchema } from "../../../src/domain/artifactInspection.js"; import { artifactIdForContent } from "../../../src/domain/artifactIdentity.js"; import { createEvidence, @@ -17,6 +16,10 @@ import { type Evidence, } from "../../../src/domain/evidence.js"; +import { + connectArtifactComparisonMcp, + inventoryEvidence, +} from "../../fixtures/artifactComparisonMcp.js"; import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -25,17 +28,6 @@ import { cliTest, type TestCli } from "../../support/cli/cliFixture.js"; const plist = (message: string) => `\nmessage${message}\n`; -const inventoryEvidence = (input: unknown): Evidence => { - const inspection = parseEvidence(input); - const inventory = artifactInspectionResultSchema - .parse(inspection.normalized_result) - .substeps.find( - ({ evidence }) => evidence.operation === "inventory_artifact", - )?.evidence; - if (inventory === undefined) throw new Error("Expected inventory Evidence"); - return parseEvidence(inventory); -}; - const inspectPair = async ( cli: TestCli, leftPath: string, @@ -46,12 +38,8 @@ const inspectPair = async ( }); expect(result.exitCode, `${result.stderr}\n${result.stdout}`).toBe(0); const left = inventoryEvidence(result.json); - const { call } = await connectLocalToolsMcp(); - const opened = await call("open_binary", { path: rightPath }); - expect(opened.isError, JSON.stringify(opened)).not.toBe(true); - const inspected = await call("inspect_artifact", {}); - expect(inspected.isError, JSON.stringify(inspected)).not.toBe(true); - return { left, right: inventoryEvidence(inspected.structuredContent), call }; + const { call, inspect } = await connectArtifactComparisonMcp(); + return { left, right: await inspect(rightPath), call }; }; const assertCoverageControls = async ({ @@ -360,14 +348,7 @@ cliTest( }); expect(leftResult.exitCode, leftResult.stderr).toBe(0); const left = inventoryEvidence(leftResult.json); - const { call } = await connectLocalToolsMcp(); - const inspect = async (path: string) => { - const opened = await call("open_binary", { path }); - expect(opened.isError, JSON.stringify(opened)).not.toBe(true); - const result = await call("inspect_artifact", {}); - expect(result.isError, JSON.stringify(result)).not.toBe(true); - return inventoryEvidence(result.structuredContent); - }; + const { call, inspect } = await connectArtifactComparisonMcp(); const compare = async (left: Evidence, right: Evidence) => { const result = await call("compare_artifacts", { left, right }); expect(result.isError, JSON.stringify(result)).not.toBe(true); @@ -466,12 +447,8 @@ cliTest.skipIf(process.platform === "win32")( }); expect(result.exitCode, result.stderr).toBe(0); const left = inventoryEvidence(result.json); - const { call } = await connectLocalToolsMcp(); - const opened = await call("open_binary", { path: rightPath }); - expect(opened.isError, JSON.stringify(opened)).not.toBe(true); - const inspected = await call("inspect_artifact", {}); - expect(inspected.isError, JSON.stringify(inspected)).not.toBe(true); - const right = inventoryEvidence(inspected.structuredContent); + const { call, inspect } = await connectArtifactComparisonMcp(); + const right = await inspect(rightPath); expect(right.subject?.digest.sha256).toBe(left.subject?.digest.sha256); const comparison = await call("compare_artifacts", { left, right }); expect(comparison.isError, JSON.stringify(comparison)).not.toBe(true); @@ -491,6 +468,58 @@ cliTest.skipIf(process.platform === "win32")( 30_000, ); +cliTest.skipIf(process.platform === "win32")( + "reports a directory file that becomes a symlink without marking the link executable", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-directory-symlink-"); + const leftPath = join(root, "l.app"), + rightPath = join(root, "r.app"); + await Promise.all([mkdir(leftPath), mkdir(rightPath)]); + await Promise.all( + [leftPath, rightPath].flatMap((path) => [ + writeFile( + join(path, "Info.plist"), + "CFBundleExecutablemain", + ), + writeFile(join(path, "main"), machoImage({})), + writeFile(join(path, "target.txt"), "target"), + ]), + ); + // The file holds the bytes the link stores: its target path. + await writeFile(join(leftPath, "current"), "target.txt"); + await chmod(join(leftPath, "current"), 0o644); + await symlink("target.txt", join(rightPath, "current")); + const { left, right, call } = await inspectPair(cli, leftPath, rightPath); + const current = (evidence: Evidence) => + artifactInventoryResultSchema + .parse(evidence.normalized_result) + .occurrences.find(({ logical_path }) => logical_path === "current"); + expect(current(left)).toMatchObject({ + entry_kind: "file", + executable: false, + }); + // The host gives the link permission bits of its own; they mark nothing. + expect(current(right)).toMatchObject({ + entry_kind: "symlink", + executable: false, + }); + const compared = await call("compare_artifacts", { left, right }); + expect(compared.isError, JSON.stringify(compared)).not.toBe(true); + expect( + parseEvidence(compared.structuredContent).normalized_result, + ).toMatchObject({ + changes: expect.arrayContaining([ + expect.objectContaining({ + logical_path: "current", + classification: "unknown", + dimensions: ["metadata", "availability"], + }), + ]), + }); + }, + 30_000, +); + cliTest.skipIf(process.platform === "win32")( "retains traversal uncertainty when a directory omits special filesystem entries", async ({ cli, processes }) => { diff --git a/tests/acceptance/artifactComparisonAsarUnpacked.test.ts b/tests/acceptance/artifactComparisonAsarUnpacked.test.ts index 476632caa9..19992608a7 100644 --- a/tests/acceptance/artifactComparisonAsarUnpacked.test.ts +++ b/tests/acceptance/artifactComparisonAsarUnpacked.test.ts @@ -8,19 +8,13 @@ import { artifactInventoryResultSchema, type ArtifactInventoryResult, } from "../../src/domain/artifactGraph.js"; -import { artifactComparisonResultSchema } from "../../src/domain/artifactComparison.js"; -import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; import { artifactGraphDigest, artifactManifestId, } from "../../src/domain/artifactIdentity.js"; import { parseArtifactInventoryEvidence } from "../../src/domain/artifactInventoryEvidence.js"; -import { - createEvidence, - parseEvidence, - type Evidence, -} from "../../src/domain/evidence.js"; -import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; +import { createEvidence, type Evidence } from "../../src/domain/evidence.js"; +import { connectArtifactComparisonMcp } from "../fixtures/artifactComparisonMcp.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; const historicalInventory = (source: Evidence): Evidence => { @@ -77,28 +71,7 @@ it("compares ASAR companion declarations from public MCP inventories and preserv // This fixture validates advertised SDK-converted input/output schemas and // every structured response against the declared JSON Schema dialect. - const { call } = await connectLocalToolsMcp(); - const inspect = async (path: string): Promise => { - const opened = await call("open_binary", { path }); - expect(opened.isError, JSON.stringify(opened)).not.toBe(true); - const inspected = await call("inspect_artifact", {}); - expect(inspected.isError, JSON.stringify(inspected)).not.toBe(true); - const inspection = artifactInspectionResultSchema.parse( - parseEvidence(inspected.structuredContent).normalized_result, - ); - const inventory = inspection.substeps.find( - ({ operation }) => operation === "inventory_artifact", - )?.evidence; - if (inventory === undefined) throw new Error("Missing inventory Evidence"); - return parseEvidence(inventory); - }; - const compare = async (left: Evidence, right: Evidence) => { - const response = await call("compare_artifacts", { left, right }); - expect(response.isError, JSON.stringify(response)).not.toBe(true); - return artifactComparisonResultSchema.parse( - parseEvidence(response.structuredContent).normalized_result, - ); - }; + const { inspect, compare } = await connectArtifactComparisonMcp(); const packed = await inspect(packedPath); const unpacked = await inspect(unpackedPath); const packedInventory = artifactInventoryResultSchema.parse( diff --git a/tests/acceptance/artifactComparisonEspImage.test.ts b/tests/acceptance/artifactComparisonEspImage.test.ts index bb836b53ea..a6fcfed74e 100644 --- a/tests/acceptance/artifactComparisonEspImage.test.ts +++ b/tests/acceptance/artifactComparisonEspImage.test.ts @@ -16,36 +16,21 @@ import { parseEvidence, type Evidence, } from "../../src/domain/evidence.js"; +import { connectArtifactComparisonMcp } from "../fixtures/artifactComparisonMcp.js"; import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; it("reports an ESP segment whose load address alone changed", async () => { const root = await createTestTempDirectory("rea-esp-comparison-"); - const { call } = await connectLocalToolsMcp(); + const { inspect: inspectFile, compare } = + await connectArtifactComparisonMcp(); const inspect = async ( name: string, segments: typeof ESP32C3_APP_SEGMENTS, ): Promise => { const path = join(root, name); await writeFile(path, espImage({ segments })); - const opened = await call("open_binary", { path }); - expect(opened.isError, JSON.stringify(opened)).not.toBe(true); - const response = await call("inspect_artifact", {}); - expect(response.isError, JSON.stringify(response)).not.toBe(true); - const inventory = artifactInspectionResultSchema - .parse(parseEvidence(response.structuredContent).normalized_result) - .substeps.find( - ({ operation }) => operation === "inventory_artifact", - )?.evidence; - if (inventory === undefined) throw new Error("Missing inventory Evidence"); - return parseEvidence(inventory); - }; - const compare = async (left: Evidence, right: Evidence) => { - const response = await call("compare_artifacts", { left, right }); - expect(response.isError, JSON.stringify(response)).not.toBe(true); - return artifactComparisonResultSchema.parse( - parseEvidence(response.structuredContent).normalized_result, - ); + return inspectFile(path); }; const segment = (evidence: Evidence, path: string) => { const occurrence = artifactInventoryResultSchema diff --git a/tests/acceptance/artifactComparisonGodotPck.test.ts b/tests/acceptance/artifactComparisonGodotPck.test.ts new file mode 100644 index 0000000000..e52156f56f --- /dev/null +++ b/tests/acceptance/artifactComparisonGodotPck.test.ts @@ -0,0 +1,147 @@ +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; + +import { artifactInventoryResultSchema } from "../../src/domain/artifactGraph.js"; +import type { Evidence } from "../../src/domain/evidence.js"; +import { connectArtifactComparisonMcp } from "../fixtures/artifactComparisonMcp.js"; +import { buildGodotPck, type GodotPckMember } from "../fixtures/godotPck.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +/** Inspect one-member packs and compare their inventories through MCP. */ +const connectPackComparison = async () => { + const root = await createTestTempDirectory("rea-pck-comparison-"); + const { inspect: inspectFile, compare } = + await connectArtifactComparisonMcp(); + const inspect = async ( + name: string, + packMember: GodotPckMember, + ): Promise => { + const file = join(root, name); + await writeFile(file, buildGodotPck(2, [packMember])); + return inspectFile(file); + }; + return { inspect, compare }; +}; + +const member = (evidence: Evidence) => { + const occurrence = artifactInventoryResultSchema + .parse(evidence.normalized_result) + .occurrences.find(({ logical_path }) => logical_path === "scenes/main.gd"); + if (occurrence === undefined) throw new Error("Missing scenes/main.gd"); + return occurrence; +}; + +it("pairs a Godot PCK member across stored res:// spellings", async () => { + const { inspect, compare } = await connectPackComparison(); + + // Godot 4.3 exports store res:// paths and 4.4.1 exports do not. Godot + // looks both up by the same simplified path, which logical_path holds. + const prefixed = await inspect("prefixed.pck", { + path: "res://scenes/main.gd", + data: "extends Node\n", + }); + const relative = await inspect("relative.pck", { + path: "scenes/main.gd", + data: "extends Node\n", + }); + expect(member(prefixed).format_metadata).toMatchObject({ + source_path: "res://scenes/main.gd", + }); + expect(member(relative).format_metadata).toMatchObject({ + source_path: "scenes/main.gd", + }); + expect(member(relative).artifact_id).toBe(member(prefixed).artifact_id); + + const result = await compare(prefixed, relative); + expect( + result.changes.filter(({ logical_path }) => logical_path !== "."), + ).toEqual([]); + expect(result.summary).toMatchObject({ unchanged: 1, unknown: 0 }); +}); + +it("reports a Godot PCK member that becomes encrypted", async () => { + const { inspect, compare } = await connectPackComparison(); + const script = { path: "res://scenes/main.gd", data: "extends Node\n" }; + const plain = await inspect("plain.pck", script); + const encrypted = await inspect("encrypted.pck", { ...script, flags: 1 }); + // REA does not decrypt members, so the encrypted member has no content. + expect(member(encrypted)).toMatchObject({ + artifact_id: null, + encrypted: true, + hash_status: "unavailable", + }); + + // The encryption change is known; whether the script changed is not. + const result = await compare(plain, encrypted); + expect( + result.changes.filter(({ logical_path }) => logical_path !== "."), + ).toMatchObject([ + { + logical_path: "scenes/main.gd", + classification: "unknown", + dimensions: ["metadata", "availability"], + left_occurrence_id: member(plain).occurrence_id, + right_occurrence_id: member(encrypted).occurrence_id, + }, + ]); +}); + +it("compares encrypted Godot PCK members by their declared MD5", async () => { + const { inspect, compare } = await connectPackComparison(); + // Godot hashes a member's unencrypted bytes, so every export of one script + // declares the same MD5 whatever its random IV. + const encrypted = (name: string, ciphertext: string, script: string) => + inspect(name, { + path: "res://scenes/main.gd", + data: ciphertext, + flags: 1, + md5: createHash("md5").update(script).digest(), + }); + const first = await encrypted("first.pck", "ciphertext A", "extends Node\n"); + const again = await encrypted("again.pck", "ciphertext B", "extends Node\n"); + const edited = await encrypted( + "edited.pck", + "ciphertext C", + "extends Sprite2D\n", + ); + const memberChanges = async (left: Evidence, right: Evidence) => + (await compare(left, right)).changes.filter( + ({ logical_path }) => logical_path !== ".", + ); + + // Equal claims are unverified, so they do not establish equality. + expect(await memberChanges(first, again)).toMatchObject([ + { + logical_path: "scenes/main.gd", + classification: "unknown", + dimensions: ["availability"], + }, + ]); + // An all-zero MD5 is no claim, and one claim alone proves no change. The + // reader notes the all-zero MD5, and that note is the metadata difference. + const unclaimed = await inspect("unclaimed.pck", { + path: "res://scenes/main.gd", + data: "ciphertext D", + flags: 1, + md5: Buffer.alloc(16), + }); + expect(member(unclaimed)).not.toHaveProperty("declared_integrity"); + expect(await memberChanges(first, unclaimed)).toMatchObject([ + { + logical_path: "scenes/main.gd", + classification: "unknown", + dimensions: ["metadata", "availability"], + }, + ]); + // Claims that differ declare different content, but REA cannot verify + // them, so the difference is known and the content change is not. + expect(await memberChanges(first, edited)).toMatchObject([ + { + logical_path: "scenes/main.gd", + classification: "unknown", + dimensions: ["metadata", "availability"], + }, + ]); +}); diff --git a/tests/acceptance/artifactComparisonNodeSea.test.ts b/tests/acceptance/artifactComparisonNodeSea.test.ts index 945e191f6c..6fa8b8d7e1 100644 --- a/tests/acceptance/artifactComparisonNodeSea.test.ts +++ b/tests/acceptance/artifactComparisonNodeSea.test.ts @@ -7,19 +7,14 @@ import { artifactInventoryResultSchema, type ArtifactInventoryResult, } from "../../src/domain/artifactGraph.js"; -import { artifactComparisonResultSchema } from "../../src/domain/artifactComparison.js"; -import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; +import { FORMAT_DECLARATION_UNKNOWN_LIMITATION } from "../../src/domain/artifactComparison.js"; import { artifactGraphDigest, artifactManifestId, } from "../../src/domain/artifactIdentity.js"; import { parseArtifactInventoryEvidence } from "../../src/domain/artifactInventoryEvidence.js"; -import { - createEvidence, - parseEvidence, - type Evidence, -} from "../../src/domain/evidence.js"; -import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; +import { createEvidence, type Evidence } from "../../src/domain/evidence.js"; +import { connectArtifactComparisonMcp } from "../fixtures/artifactComparisonMcp.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; const withoutEntryMetadata = (source: Evidence): Evidence => { @@ -72,32 +67,15 @@ it("compares raw SEA asset keys without interpreting serialization order as meta const nfc = "caf\u00e9.txt"; const nfd = "cafe\u0301.txt"; const body = "identical asset bytes"; - const { call } = await connectLocalToolsMcp(); + const { inspect: inspectFile, compare } = + await connectArtifactComparisonMcp(); const inspect = async ( name: string, assets: readonly { name: string; body: string }[], ): Promise => { const path = join(root, name); await writeFile(path, nodeSeaPe({ fuse: "set", assets })); - const opened = await call("open_binary", { path }); - expect(opened.isError, JSON.stringify(opened)).not.toBe(true); - const response = await call("inspect_artifact", {}); - expect(response.isError, JSON.stringify(response)).not.toBe(true); - const inspection = artifactInspectionResultSchema.parse( - parseEvidence(response.structuredContent).normalized_result, - ); - const inventory = inspection.substeps.find( - ({ operation }) => operation === "inventory_artifact", - )?.evidence; - if (inventory === undefined) throw new Error("Missing inventory Evidence"); - return parseEvidence(inventory); - }; - const compare = async (left: Evidence, right: Evidence) => { - const response = await call("compare_artifacts", { left, right }); - expect(response.isError, JSON.stringify(response)).not.toBe(true); - return artifactComparisonResultSchema.parse( - parseEvidence(response.structuredContent).normalized_result, - ); + return inspectFile(path); }; const child = (evidence: Evidence) => { const occurrence = artifactInventoryResultSchema @@ -175,6 +153,7 @@ it("compares raw SEA asset keys without interpreting serialization order as meta dimensions: ["availability"], evidence_links: [historical.evidence_id, right.evidence_id], }); + expect(unknown.limitations).toContain(FORMAT_DECLARATION_UNKNOWN_LIMITATION); const reversed = await compare(right, historical); expect( reversed.changes.find(({ logical_path }) => logical_path === nfc), diff --git a/tests/acceptance/artifactComparisonPyInstaller.test.ts b/tests/acceptance/artifactComparisonPyInstaller.test.ts new file mode 100644 index 0000000000..f0c0299ff5 --- /dev/null +++ b/tests/acceptance/artifactComparisonPyInstaller.test.ts @@ -0,0 +1,119 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; + +import { + PYTHON_39_MAGIC, + cArchiveExecutable, + pyzArchive, + type CArchiveFixtureMember, +} from "../../src/artifacts/pyinstaller/CArchive.fixture.js"; +import { artifactInventoryResultSchema } from "../../src/domain/artifactGraph.js"; +import type { Evidence } from "../../src/domain/evidence.js"; +import { connectArtifactComparisonMcp } from "../fixtures/artifactComparisonMcp.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +/** Inspect archives and compare their inventories through MCP. */ +const connectArchiveComparison = async () => { + const root = await createTestTempDirectory("rea-pyinstaller-comparison-"); + const { inspect: inspectFile, compare } = + await connectArtifactComparisonMcp(); + const inspect = async ( + name: string, + members: readonly CArchiveFixtureMember[], + ): Promise => { + const path = join(root, name); + await writeFile( + path, + cArchiveExecutable({ layout: "2.1", pythonVersion: 309, members }), + ); + return inspectFile(path); + }; + return { inspect, compare }; +}; + +const occurrence = (evidence: Evidence, path: string) => { + const found = artifactInventoryResultSchema + .parse(evidence.normalized_result) + .occurrences.find(({ logical_path }) => logical_path === path); + if (found === undefined) throw new Error(`Missing ${path}`); + return found; +}; + +it("reports a changed PyInstaller declaration whose member bytes are unavailable", async () => { + const { inspect, compare } = await connectArchiveComparison(); + const link = (name: string) => [ + { name, typeCode: "n", payload: Buffer.from("target"), compress: false }, + ]; + + // A rebuild on another filesystem can store the link name decomposed. + // Normalization pairs the links; their stored names stay distinct. + const nfc = "café-link"; + const left = await inspect("nfc", link(nfc)); + const right = await inspect("nfd", link("café-link")); + expect(occurrence(left, nfc)).toMatchObject({ + hash_status: "not-hashed", + format_metadata: { source_name: nfc }, + }); + expect(occurrence(right, nfc)).toMatchObject({ + hash_status: "not-hashed", + format_metadata: { source_name: "café-link" }, + }); + + const result = await compare(left, right); + expect( + result.changes.find(({ logical_path }) => logical_path === nfc), + ).toMatchObject({ + classification: "changed", + dimensions: ["metadata", "availability"], + left_occurrence_id: occurrence(left, nfc).occurrence_id, + right_occurrence_id: occurrence(right, nfc).occurrence_id, + }); +}); + +it("reports a PYZ namespace record that becomes encrypted", async () => { + const { inspect, compare } = await connectArchiveComparison(); + const pyz = { + name: "PYZ.pyz", + typeCode: "z", + compress: false, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { + name: "ns.branch", + typeCode: 3, + body: Buffer.from("legacy namespace payload"), + legacyNamespacePayload: true, + }, + ], + }), + }; + const namespace = "PYZ.pyz/ns/branch"; + const plain = await inspect("plain", [pyz]); + // The legacy key module also encrypts a nonempty namespace payload. + const encrypted = await inspect("encrypted", [ + { + name: "pyimod00_crypto_key", + typeCode: "m", + payload: Buffer.from("key module"), + compress: false, + }, + pyz, + ]); + expect(occurrence(encrypted, namespace)).toMatchObject({ + entry_kind: "metadata", + artifact_id: null, + encrypted: true, + }); + + const result = await compare(plain, encrypted); + expect( + result.changes.find(({ logical_path }) => logical_path === namespace), + ).toMatchObject({ + classification: "changed", + dimensions: ["metadata"], + left_occurrence_id: occurrence(plain, namespace).occurrence_id, + right_occurrence_id: occurrence(encrypted, namespace).occurrence_id, + }); +}); diff --git a/tests/acceptance/artifactComparisonUnityFs.test.ts b/tests/acceptance/artifactComparisonUnityFs.test.ts new file mode 100644 index 0000000000..8e1fe6f034 --- /dev/null +++ b/tests/acceptance/artifactComparisonUnityFs.test.ts @@ -0,0 +1,89 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; + +import { artifactInventoryResultSchema } from "../../src/domain/artifactGraph.js"; +import type { Evidence } from "../../src/domain/evidence.js"; +import { connectArtifactComparisonMcp } from "../fixtures/artifactComparisonMcp.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { buildUnityFs, type UnityFsMember } from "../fixtures/unityfs.js"; + +it("compares UnityFS member declarations without their layout", async () => { + const root = await createTestTempDirectory("rea-unityfs-comparison-"); + const { inspect: inspectFile, compare } = + await connectArtifactComparisonMcp(); + const inspect = async ( + name: string, + members: readonly UnityFsMember[], + ): Promise => { + const path = join(root, name); + await writeFile(path, buildUnityFs(members)); + return inspectFile(path); + }; + const member = (evidence: Evidence, path: string) => { + const occurrence = artifactInventoryResultSchema + .parse(evidence.normalized_result) + .occurrences.find(({ logical_path }) => logical_path === path); + if (occurrence === undefined) throw new Error(`Missing ${path}`); + return occurrence; + }; + const memberChanges = async (left: Evidence, right: Evidence) => + (await compare(left, right)).changes.filter( + ({ logical_path }) => logical_path !== ".", + ); + + const nfc = "CAB-café"; + const serialized = { path: nfc, data: "serialized file bytes", flags: 4 }; + const resource = { path: "CAB-cafe.resS", data: "resource bytes" }; + const left = await inspect("left.unity3d", [serialized, resource]); + + // The same bytes with other raw node flags. + const flagged = await inspect("flags.unity3d", [ + { ...serialized, flags: 0 }, + resource, + ]); + expect(member(flagged, nfc).artifact_id).toBe(member(left, nfc).artifact_id); + expect(member(left, nfc).format_metadata).toMatchObject({ flags: 4 }); + expect(member(flagged, nfc).format_metadata).toMatchObject({ flags: 0 }); + expect(await memberChanges(left, flagged)).toEqual([ + { + classification: "changed", + logical_path: nfc, + dimensions: ["metadata"], + left_occurrence_id: member(left, nfc).occurrence_id, + right_occurrence_id: member(flagged, nfc).occurrence_id, + left_artifact_id: member(left, nfc).artifact_id, + right_artifact_id: member(flagged, nfc).artifact_id, + evidence_links: [left.evidence_id, flagged.evidence_id], + }, + ]); + + // A path stored decomposed pairs with its composed form, and the stored + // spelling is the member's declared source path. + const decomposed = await inspect("nfd.unity3d", [ + { ...serialized, path: "CAB-café" }, + resource, + ]); + expect(member(decomposed, nfc).format_metadata).toMatchObject({ + source_path: "CAB-café", + }); + expect(await memberChanges(left, decomposed)).toEqual([ + expect.objectContaining({ + classification: "changed", + logical_path: nfc, + dimensions: ["metadata"], + }), + ]); + + // Directory order and stream offsets are layout. + const reordered = await inspect("reordered.unity3d", [resource, serialized]); + expect(member(reordered, nfc).format_metadata).toMatchObject({ + index: 1, + declared_offset: Buffer.byteLength(resource.data), + }); + expect(member(left, nfc).format_metadata).toMatchObject({ + index: 0, + declared_offset: 0, + }); + expect(await memberChanges(left, reordered)).toEqual([]); +}); diff --git a/tests/acceptance/artifactComparisonZip.test.ts b/tests/acceptance/artifactComparisonZip.test.ts new file mode 100644 index 0000000000..efd85a20eb --- /dev/null +++ b/tests/acceptance/artifactComparisonZip.test.ts @@ -0,0 +1,132 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; +import { expect, it } from "vitest"; + +import type { ArtifactComparisonResult } from "../../src/domain/artifactComparison.js"; +import { artifactInventoryResultSchema } from "../../src/domain/artifactGraph.js"; +import type { Evidence } from "../../src/domain/evidence.js"; +import { connectArtifactComparisonMcp } from "../fixtures/artifactComparisonMcp.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +/** A ZIP member with the Unix mode its external attributes declare. */ +interface ZipMember { + readonly name: string; + readonly data: string; + readonly mode: number; + readonly password?: string; +} + +/** Inspect ZIP archives and compare their inventories through MCP. */ +const connectZipComparison = async () => { + const root = await createTestTempDirectory("rea-zip-comparison-"); + const { inspect: inspectFile, compare } = + await connectArtifactComparisonMcp(); + const inspect = async ( + name: string, + members: readonly ZipMember[], + ): Promise => { + const writer = new ZipWriter(new Uint8ArrayWriter(), { + versionMadeBy: 3 << 8, + }); + for (const { name: path, data, mode, password } of members) + await writer.add(path, new TextReader(data), { + externalFileAttributes: (mode << 16) >>> 0, + ...(password === undefined ? {} : { password }), + }); + const file = join(root, name); + await writeFile(file, await writer.close()); + return inspectFile(file); + }; + return { inspect, compare }; +}; + +const member = (evidence: Evidence, path: string) => { + const occurrence = artifactInventoryResultSchema + .parse(evidence.normalized_result) + .occurrences.find(({ logical_path }) => logical_path === path); + if (occurrence === undefined) throw new Error(`Missing ${path}`); + return occurrence; +}; + +/** The archive root differs whenever members do; compare members only. */ +const memberChanges = (result: ArtifactComparisonResult) => + result.changes.filter(({ logical_path }) => logical_path !== "."); + +it("reports a ZIP member that becomes a symlink", async () => { + const { inspect, compare } = await connectZipComparison(); + // A symlink member stores its target as data, so both members hold the + // same bytes; only the declared entry kind differs. + const file = await inspect("file.zip", [ + { name: "current", data: "target.txt", mode: 0o100644 }, + ]); + const link = await inspect("link.zip", [ + { name: "current", data: "target.txt", mode: 0o120777 }, + ]); + expect(member(file, "current")).toMatchObject({ + entry_kind: "file", + hash_status: "verified", + executable: false, + }); + // REA does not read symlink targets, so the link has no content. + expect(member(link, "current")).toMatchObject({ + entry_kind: "symlink", + artifact_id: null, + hash_status: "not-hashed", + executable: false, + }); + + // The entry kind change is known; whether the content changed is not. + expect(memberChanges(await compare(file, link))).toMatchObject([ + { + logical_path: "current", + classification: "unknown", + dimensions: ["metadata", "availability"], + left_occurrence_id: member(file, "current").occurrence_id, + right_occurrence_id: member(link, "current").occurrence_id, + }, + ]); + // Without a known difference, unread content stays unknown. + expect(memberChanges(await compare(link, link))).toMatchObject([ + { + logical_path: "current", + classification: "unknown", + dimensions: ["availability"], + }, + ]); +}); + +it("reports an executable mode change on an encrypted ZIP member", async () => { + const { inspect, compare } = await connectZipComparison(); + const tool = { name: "tool", data: "#!/bin/sh\n", password: "fixture" }; + const plain = await inspect("plain.zip", [{ ...tool, mode: 0o100644 }]); + const marked = await inspect("marked.zip", [{ ...tool, mode: 0o100755 }]); + // REA does not decrypt members, so neither side has content. + for (const [evidence, executable] of [ + [plain, false], + [marked, true], + ] as const) + expect(member(evidence, "tool")).toMatchObject({ + artifact_id: null, + encrypted: true, + hash_status: "unavailable", + executable, + }); + + // The mode change is known; whether the encrypted bytes changed is not. + expect(memberChanges(await compare(plain, marked))).toMatchObject([ + { + logical_path: "tool", + classification: "unknown", + dimensions: ["executable", "availability"], + }, + ]); + // Without a known difference, the encrypted bytes stay unknown. + expect(memberChanges(await compare(plain, plain))).toMatchObject([ + { + logical_path: "tool", + classification: "unknown", + dimensions: ["availability"], + }, + ]); +}); diff --git a/tests/conformance/artifactComparisonDocs.test.ts b/tests/conformance/artifactComparisonDocs.test.ts new file mode 100644 index 0000000000..cf9a93cf7c --- /dev/null +++ b/tests/conformance/artifactComparisonDocs.test.ts @@ -0,0 +1,31 @@ +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { expect, it } from "vitest"; +import { + artifactFormatMetadataSchema, + COMPARED_FORMAT_METADATA_KINDS, +} from "../../src/domain/artifactFormatMetadata.js"; + +const FORMAT_METADATA_KINDS = new Set( + artifactFormatMetadataSchema.options.map((option) => option.shape.kind.value), +); +// Markers, not the paragraph's wording, delimit the contract's compared kinds. +const BEGIN = ""; +const END = ""; + +it("names exactly the format metadata kinds that artifact comparison compares", async () => { + const contracts = await readFile(resolve("docs/mcp-contracts.md"), "utf8"); + const start = contracts.indexOf(BEGIN); + const end = contracts.indexOf(END); + expect(start).toBeGreaterThanOrEqual(0); + expect(end).toBeGreaterThan(start); + expect(contracts.indexOf(BEGIN, start + 1)).toBe(-1); + expect(contracts.indexOf(END, end + 1)).toBe(-1); + const named = [ + ...contracts.slice(start + BEGIN.length, end).matchAll(/`([^`]+)`/gu), + ] + .map((match) => match[1] ?? "") + .filter((token) => FORMAT_METADATA_KINDS.has(token)); + expect(new Set(named)).toEqual(new Set(COMPARED_FORMAT_METADATA_KINDS)); +}); diff --git a/tests/fixtures/artifactComparisonMcp.ts b/tests/fixtures/artifactComparisonMcp.ts new file mode 100644 index 0000000000..f61f545012 --- /dev/null +++ b/tests/fixtures/artifactComparisonMcp.ts @@ -0,0 +1,37 @@ +import { expect } from "vitest"; + +import { artifactComparisonResultSchema } from "../../src/domain/artifactComparison.js"; +import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; +import { parseEvidence, type Evidence } from "../../src/domain/evidence.js"; +import { connectLocalToolsMcp } from "./localToolsMcp.js"; + +/** Return the inventory Evidence that an artifact inspection retains. */ +export const inventoryEvidence = (inspection: unknown): Evidence => { + const inventory = artifactInspectionResultSchema + .parse(parseEvidence(inspection).normalized_result) + .substeps.find( + ({ operation }) => operation === "inventory_artifact", + )?.evidence; + if (inventory === undefined) throw new Error("Missing inventory Evidence"); + return parseEvidence(inventory); +}; + +/** Inspect artifact files and compare their inventories through MCP. */ +export async function connectArtifactComparisonMcp() { + const { call } = await connectLocalToolsMcp(); + const inspect = async (path: string): Promise => { + const opened = await call("open_binary", { path }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const response = await call("inspect_artifact", {}); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + return inventoryEvidence(response.structuredContent); + }; + const compare = async (left: Evidence, right: Evidence) => { + const response = await call("compare_artifacts", { left, right }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + return artifactComparisonResultSchema.parse( + parseEvidence(response.structuredContent).normalized_result, + ); + }; + return { call, inspect, compare }; +} From b1b6cb64c2c9b0765e0ff6d6f48740140767d5fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=98=9F=E9=97=8C?= <130824939+Xinglan233@users.noreply.github.com> Date: Wed, 4 Nov 2026 23:53:45 +0800 Subject: [PATCH 67/94] fix(analysis-view)!: distinguish unavailable native collections (#8291) --- docs/mcp-contracts.md | 11 ++ .../analysisView/NativeLoadImageViews.test.ts | 7 + .../NativeObservationViews.test.ts | 7 + .../analysisView/analysisViewCoverage.ts | 25 +++ src/domain/analysisView/analysisViewResult.ts | 10 +- .../analysisView/nativeLoadImageView.ts | 12 +- .../analysisView/nativeObservationView.ts | 48 ++++-- .../unavailableNativeCollectionViews.test.ts | 154 ++++++++++++++++++ 8 files changed, 254 insertions(+), 20 deletions(-) create mode 100644 tests/acceptance/tools/unavailableNativeCollectionViews.test.ts diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 8943ec4b2f..ac005e2c8d 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -278,6 +278,17 @@ to `/absolute/view-request.json`, then run `rea inspect-analysis-view /absolute/view-request.json --json`. This also reads pre-summary source records without changing their identity; no live provider is started. +Unavailable or inapplicable native collections keep `items.rows: null` and +report `coverage: {"status":"unavailable","examined":0,"total":null,"next_offset":null,"exhausted":false}`. +This applies to unrecorded `native-load-image` checks/sections and inapplicable +`native-context` result limitations. Their total is unknown; no continuation is +offered. Stop paging that collection and inspect the returned unknowns rather +than treating it as an observed empty set or retrying a null offset. Recorded +empty collections, empty selections and offsets past the end retain numeric +totals and exhausted page coverage. Clients upgrading from the earlier +zero-total representation must handle the new `unavailable` status and nullable +total; historical saved views remain valid. + `procedure_address`, `procedure_callees`, `procedure_callers`, `resolve_containing_procedure` and `get_call_graph` also accept `detail: "summary"` (complete remains the default). The summary includes the first 64 recorded diff --git a/src/application/analysisView/NativeLoadImageViews.test.ts b/src/application/analysisView/NativeLoadImageViews.test.ts index c1616d57c2..2c391694ea 100644 --- a/src/application/analysisView/NativeLoadImageViews.test.ts +++ b/src/application/analysisView/NativeLoadImageViews.test.ts @@ -208,6 +208,13 @@ it.each(["checks", "sections"])( await view(source(), { kind: "native-load-image", collection }), ).toMatchObject({ items: { collection, rows: null }, + coverage: { + status: "unavailable", + examined: 0, + total: null, + next_offset: null, + exhausted: false, + }, unknowns: expect.arrayContaining([ "Null collection counts or rows mean not recorded or not applicable, not observed absence.", ]), diff --git a/src/application/analysisView/NativeObservationViews.test.ts b/src/application/analysisView/NativeObservationViews.test.ts index be9092a4e0..c9c16e99ef 100644 --- a/src/application/analysisView/NativeObservationViews.test.ts +++ b/src/application/analysisView/NativeObservationViews.test.ts @@ -252,6 +252,13 @@ it("keeps absent profile and inapplicable result caveats distinct from observed ).toMatchObject({ source_context: { result_limitation_count: null }, items: { rows: null }, + coverage: { + status: "unavailable", + examined: 0, + total: null, + next_offset: null, + exhausted: false, + }, unknowns: expect.arrayContaining([ expect.stringContaining("not applicable"), ]), diff --git a/src/domain/analysisView/analysisViewCoverage.ts b/src/domain/analysisView/analysisViewCoverage.ts index 392c711095..d5f694c0fd 100644 --- a/src/domain/analysisView/analysisViewCoverage.ts +++ b/src/domain/analysisView/analysisViewCoverage.ts @@ -9,8 +9,33 @@ export const analysisViewCoverageSchema = z.strictObject({ exhausted: z.boolean(), }); +const unavailableViewCoverageSchema = z.strictObject({ + status: z.literal("unavailable"), + examined: z.literal(0), + total: z.null(), + next_offset: z.null(), + exhausted: z.literal(false), +}); + +/** Only views with nullable collections can report unavailable page coverage. */ +export const nullableCollectionViewCoverageSchema = z.union([ + analysisViewCoverageSchema, + unavailableViewCoverageSchema, +]); + export type AnalysisViewCoverage = z.output; +/** Unrecorded or inapplicable collections have no known total or continuation. */ +export const unavailableViewCoverage = (): z.output< + typeof unavailableViewCoverageSchema +> => ({ + status: "unavailable", + examined: 0, + total: null, + next_offset: null, + exhausted: false, +}); + /** Coverage for a summary, facet, or single selected object. */ export const completeWithinViewCoverage = ( examined: number, diff --git a/src/domain/analysisView/analysisViewResult.ts b/src/domain/analysisView/analysisViewResult.ts index c7240d0ce8..d5ea266d4e 100644 --- a/src/domain/analysisView/analysisViewResult.ts +++ b/src/domain/analysisView/analysisViewResult.ts @@ -1,4 +1,7 @@ -import { analysisViewCoverageSchema } from "./analysisViewCoverage.js"; +import { + analysisViewCoverageSchema, + nullableCollectionViewCoverageSchema, +} from "./analysisViewCoverage.js"; import { javaScriptRouteInventorySourceSchema, javaScriptRouteInventorySchema, @@ -447,7 +450,9 @@ const nativeContextViewResultSchema = z.discriminatedUnion("facet", [ nativeContextFor("overview"), nativeContextFor("analysis-profile"), nativeContextFor("limitations"), - nativeContextFor("result-limitations"), + nativeContextFor("result-limitations").extend({ + coverage: nullableCollectionViewCoverageSchema, + }), ]); export const byteEntropySummaryViewResultSchema = z.strictObject({ @@ -1202,6 +1207,7 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ z.strictObject({ ...viewResultBase, kind: z.literal("native-load-image"), + coverage: nullableCollectionViewCoverageSchema, ...nativeViewSourceContext, verification_status: z.enum(["unsupported", "verified", "mismatch"]), parent_operation: z.literal("inspect_native_load_image"), diff --git a/src/domain/analysisView/nativeLoadImageView.ts b/src/domain/analysisView/nativeLoadImageView.ts index eb5b56af88..594d9d688e 100644 --- a/src/domain/analysisView/nativeLoadImageView.ts +++ b/src/domain/analysisView/nativeLoadImageView.ts @@ -23,6 +23,7 @@ import { boundedViewMessages, completeWithinViewCoverage, pageViewCoverage, + unavailableViewCoverage, } from "./analysisViewCoverage.js"; const observation = nativeLoadImageObservationSchema.shape; @@ -290,12 +291,12 @@ export const projectNativeLoadImageView = ( ? report.sections : undefined : observed[view.collection]; - const page = selectRows(selected ?? [], view); + const page = selected === undefined ? undefined : selectRows(selected, view); const symbolUnrecorded = view.collection === "relocations" && (view.fields === undefined || view.fields.some((field: string) => field === "symbol")) && - page.rows.some(({ record }) => !("symbol" in record)); + (page?.rows.some(({ record }) => !("symbol" in record)) ?? false); return ok({ ...shared, unknowns: symbolUnrecorded @@ -309,8 +310,11 @@ export const projectNativeLoadImageView = ( view, items: nativeLoadImageViewItemsSchema.parse({ collection: view.collection, - rows: selected === undefined ? null : page.rows, + rows: page?.rows ?? null, }), - coverage: pageViewCoverage(view.offset, page.rows.length, page.total), + coverage: + page === undefined + ? unavailableViewCoverage() + : pageViewCoverage(view.offset, page.rows.length, page.total), }); }; diff --git a/src/domain/analysisView/nativeObservationView.ts b/src/domain/analysisView/nativeObservationView.ts index 48ef919da2..ee4530adc3 100644 --- a/src/domain/analysisView/nativeObservationView.ts +++ b/src/domain/analysisView/nativeObservationView.ts @@ -20,7 +20,7 @@ import { createAnalysisViewResultReader } from "./analysisViewParent.js"; import { analysisViewInputError } from "./analysisViewErrors.js"; import { pageViewCoverage, - completeWithinViewCoverage, + unavailableViewCoverage, } from "./analysisViewCoverage.js"; import { nativeContextViewRequestSchema, @@ -304,22 +304,42 @@ export const projectNativeObservationView = ( ], }; if (view.kind === "native-context") { - const { total, ...page } = contextRows(context.value, view); - return ok({ + const selected = contextRows(context.value, view); + const shared = { ...common, kind: "native-context" as const, - ...page, parent_operation: source.value.operation, - coverage: - total === null - ? completeWithinViewCoverage(0, 0) - : pageViewCoverage(view.offset, page.items.rows?.length ?? 0, total), - unknowns: - total === null - ? [ - "Result limitations are not applicable to this primitive observation; null does not report an examined empty list.", - ] - : common.unknowns, + }; + if (selected.facet === "result-limitations") { + const { total, ...page } = selected; + return ok({ + ...shared, + ...page, + coverage: + total === null + ? unavailableViewCoverage() + : pageViewCoverage( + view.offset, + page.items.rows?.length ?? 0, + total, + ), + unknowns: + total === null + ? [ + "Result limitations are not applicable to this primitive observation; null does not report an examined empty list.", + ] + : common.unknowns, + }); + } + const { total, ...page } = selected; + return ok({ + ...shared, + ...page, + coverage: pageViewCoverage( + view.offset, + page.items.rows?.length ?? 0, + total, + ), }); } const selection = view.kind === "summary" ? { offset: 0, limit: 64 } : view; diff --git a/tests/acceptance/tools/unavailableNativeCollectionViews.test.ts b/tests/acceptance/tools/unavailableNativeCollectionViews.test.ts new file mode 100644 index 0000000000..fc1a27e2b5 --- /dev/null +++ b/tests/acceptance/tools/unavailableNativeCollectionViews.test.ts @@ -0,0 +1,154 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; + +import { analysisViewResultSchema } from "../../../src/domain/analysisView/analysisView.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { + nativeLoadImageObservationFixture, + nativeRawLoadImageFixture, +} from "../../fixtures/nativeLoadImage.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "distinguishes unavailable native collections from empty selections through CLI and MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-unavailable-native-view-"); + const provider = { + id: "fixture", + name: "Retained native fixture", + version: "1", + }; + const load = (result: unknown) => + createEvidence(undefined, provider, { + operation: "inspect_native_load_image", + parameters: {}, + result, + limitations: [ + "Synthetic retained Evidence; no native provider was run.", + ], + }); + const unsupported = load({ + status: "unsupported", + reason: "Independent ELF load verification was not recorded.", + observations: { + ...nativeLoadImageObservationFixture(), + executable_format: "Executable and Linking Format (ELF)", + }, + limitations: [], + }); + const raw = load(nativeRawLoadImageFixture()); + const primitive = createEvidence(undefined, provider, { + operation: "procedure_callees", + parameters: {}, + result: [], + }); + const graph = createEvidence(undefined, provider, { + operation: "get_call_graph", + parameters: {}, + result: { graph: {}, limitations: [] }, + }); + const unavailable = { + status: "unavailable", + examined: 0, + total: null, + next_offset: null, + exhausted: false, + }; + const empty = (total: number) => ({ + status: "empty", + examined: 0, + total, + next_offset: null, + exhausted: true, + }); + const cases = [ + { + label: "checks-unrecorded", + evidence: unsupported, + view: { kind: "native-load-image", collection: "checks" }, + rows: null, + coverage: unavailable, + }, + { + label: "sections-unrecorded-despite-empty-selection", + evidence: unsupported, + view: { + kind: "native-load-image", + collection: "sections", + offset: 5, + indices: [], + }, + rows: null, + coverage: unavailable, + }, + { + label: "checks-recorded-empty", + evidence: raw, + view: { kind: "native-load-image", collection: "checks" }, + rows: [], + coverage: empty(0), + }, + { + label: "mappings-empty-selection", + evidence: unsupported, + view: { + kind: "native-load-image", + collection: "mappings", + indices: [], + }, + rows: [], + coverage: empty(0), + }, + { + label: "mappings-past-end", + evidence: unsupported, + view: { kind: "native-load-image", collection: "mappings", offset: 5 }, + rows: [], + coverage: empty(1), + }, + { + label: "primitive-result-limitations-inapplicable", + evidence: primitive, + view: { kind: "native-context", facet: "result-limitations" }, + rows: null, + coverage: unavailable, + }, + { + label: "graph-result-limitations-recorded-empty", + evidence: graph, + view: { kind: "native-context", facet: "result-limitations" }, + rows: [], + coverage: empty(0), + }, + ]; + const call = await cli.sharedMcpCall!(); + const run = cli.sharedCliRun!.bind(cli); + for (const test of cases) { + const request = { + source: { kind: "inline", evidence: test.evidence }, + view: test.view, + }; + const path = join(root, `${test.label}.json`); + await writeFile(path, JSON.stringify(request)); + const output = await run({ + arguments: ["inspect-analysis-view", path, "--json"], + }); + expect(output.exitCode, output.stderr || output.stdout).toBe(0); + const page = analysisViewResultSchema.parse( + parseEvidence(output.json).normalized_result, + ); + const response = await call("inspect_analysis_view", request); + expect(response.isError).not.toBe(true); + expect( + parseEvidence(response.structuredContent).normalized_result, + ).toEqual(page); + expect(page, test.label).toMatchObject({ + parent_evidence_id: test.evidence.evidence_id, + items: { rows: test.rows }, + coverage: test.coverage, + }); + } + }, +); From 30a11af92362ff34d716eefd34658ce49b3beca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A4=80=ED=98=B8?= <101695482+chordpli@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:54:29 +0900 Subject: [PATCH 68/94] fix(browser)!: keep the Chromium sandbox for REA-launched browsers (#8738) * fix(browser)!: keep the Chromium sandbox for REA-launched browsers Playwright appends --no-sandbox unless a launch passes chromiumSandbox: true. Neither the browser scenario launch nor the native module resolver passed it, so every REA-launched Chromium ran without its sandbox and REA_BROWSER_NO_SANDBOX had no effect. Both launch sites now take the sandbox decision from ChromiumSandbox.ts: off on Android or when REA_BROWSER_NO_SANDBOX=true, on otherwise. When Chromium reports that it cannot start its sandbox, the launch failure's remediation names the opt-out. Fixes #8722 BREAKING CHANGE: hosts where Chromium cannot create its sandbox, such as root users and Linux containers without user namespaces or a setuid sandbox, now fail to launch until REA_BROWSER_NO_SANDBOX=true is set. * fix(browser): name the sandbox opt-out on any sandboxed launch failure Chromium reports an unusable sandbox through several unrelated startup messages, including a misconfigured setuid helper and a refused namespace move, and Playwright recognizes only some of them. Matching individual messages missed real failures, so a sandboxed launch failure now always names REA_BROWSER_NO_SANDBOX=true as a conditional remediation and the retained startup diagnostics show whether the sandbox was the cause. * fix(browser): keep default recovery for resolver failures after launch The module resolver's outer catch also covers context, navigation and evaluation failures after the browser launched. Attaching the sandbox remediation there replaced their default recovery guidance. Attach the shared launch remediation only when a sandboxed launch itself fails. * docs(browser): note the Android sandbox exception for module traces --- docs/browser-scenario-contract.md | 5 +- docs/website-module-trace.md | 6 +- src/browser/ChromiumSandbox.ts | 32 ++++++++++ src/browser/PlaywrightScenarioBrowser.test.ts | 48 +++++++++++++- src/browser/PlaywrightScenarioBrowser.ts | 14 +++-- .../modules/NativeModuleResolver.test.ts | 62 ++++++++++++++++++- src/browser/modules/NativeModuleResolver.ts | 40 +++++++++--- 7 files changed, 185 insertions(+), 22 deletions(-) create mode 100644 src/browser/ChromiumSandbox.ts diff --git a/docs/browser-scenario-contract.md b/docs/browser-scenario-contract.md index 2c9ed3f35f..614ecff971 100644 --- a/docs/browser-scenario-contract.md +++ b/docs/browser-scenario-contract.md @@ -9,7 +9,10 @@ liveness deadlines. `INPUT_JSON` may be inline JSON or a JSON file path. The minimal request contains only browser launch/connect selection, `start_url`, and at least one `actions` entry. Launch `headless` defaults to `true` and can -be set to `false` where the host supports a visible browser. Navigations follow +be set to `false` where the host supports a visible browser. Launched Chromium +keeps its sandbox; on hosts that cannot provide one, such as root in a +container, set `REA_BROWSER_NO_SANDBOX=true` for the REA process. Termux on +Android always launches without it. Navigations follow the destinations named by the scenario and the page's ordinary browser requests, and [request steps](#request-steps) send only to the start URL's origin; there is no separate origin allowlist. Request steps use a temporary dispatch diff --git a/docs/website-module-trace.md b/docs/website-module-trace.md index f5e55d3e07..7612bcdfd1 100644 --- a/docs/website-module-trace.md +++ b/docs/website-module-trace.md @@ -86,8 +86,10 @@ and import map have 32 MiB, 16 MiB and 4 MiB input limits. Native resolution has one 20-second deadline; cleanup may extend that deadline. One renderer is requested, and there is no aggregate host memory-limit claim. Cancellation closes only the owned browser; cleanup failure cannot return a successful trace. -`REA_BROWSER_NO_SANDBOX=true` passes Chromium's corresponding launch option when -the host requires it; it does not enable an REA security boundary. +Chromium keeps its sandbox, except under Termux on Android, which always +launches it without one. On hosts that cannot provide one, such as root in a +container, `REA_BROWSER_NO_SANDBOX=true` launches it without the sandbox; the +variable does not enable an REA security boundary. See [testing](testing.md) for the native loading oracle and CLI/MCP real lane. diff --git a/src/browser/ChromiumSandbox.ts b/src/browser/ChromiumSandbox.ts new file mode 100644 index 0000000000..e5949c11ec --- /dev/null +++ b/src/browser/ChromiumSandbox.ts @@ -0,0 +1,32 @@ +/** Host opt-out for Chromium launches where the host cannot provide its sandbox. */ +export const CHROMIUM_NO_SANDBOX_VARIABLE = "REA_BROWSER_NO_SANDBOX"; + +/** + * Decide whether REA-launched Chromium keeps its sandbox. + * + * Playwright disables the Chromium sandbox unless the launch asks for it, so + * every launch passes this decision explicitly. Termux's Chromium has no usable + * sandbox; other hosts opt out only through {@link CHROMIUM_NO_SANDBOX_VARIABLE}. + */ +export const chromiumSandboxEnabled = ( + environment: Readonly>, + platform: NodeJS.Platform = process.platform, +): boolean => + platform !== "android" && + environment[CHROMIUM_NO_SANDBOX_VARIABLE] !== "true"; + +const LAUNCH_REMEDIATION = + "Review the selected executable and startup diagnostics, then correct the launch configuration before retrying."; + +/** + * Remediation for a Chromium launch that failed before the browser connected. + * + * Chromium reports an unusable sandbox through several unrelated messages (no + * namespace support, a misconfigured setuid helper, a refused namespace move, + * a root user), so REA does not classify the startup log. A sandboxed launch + * adds the opt-out; the retained diagnostics show whether it applies. + */ +export const chromiumLaunchRemediation = (sandbox: boolean): string => + sandbox + ? `${LAUNCH_REMEDIATION} If the startup diagnostics show that Chromium could not start its sandbox on this host, enable the host's sandbox support or set ${CHROMIUM_NO_SANDBOX_VARIABLE}=true to launch it without the sandbox.` + : LAUNCH_REMEDIATION; diff --git a/src/browser/PlaywrightScenarioBrowser.test.ts b/src/browser/PlaywrightScenarioBrowser.test.ts index 3af1166b3f..b36b3a7462 100644 --- a/src/browser/PlaywrightScenarioBrowser.test.ts +++ b/src/browser/PlaywrightScenarioBrowser.test.ts @@ -172,7 +172,8 @@ it("keeps Termux execution support when launching Chromium on Android", async () diagnostics: { startup_error: "launch boundary reached" }, }); - expect(launchOptions?.args).toEqual(["--no-sandbox", "--single-process"]); + expect(launchOptions?.chromiumSandbox).toBe(false); + expect(launchOptions?.args).toEqual(["--single-process"]); expect(launchOptions?.env).toEqual({ LANG: "selected-language", LD_PRELOAD: "/data/data/com.termux/files/usr/lib/libtermux-exec.so", @@ -180,6 +181,51 @@ it("keeps Termux execution support when launching Chromium on Android", async () }); }); +const launchScenario = browserScenarioSchema.parse({ + browser: { mode: "launch", executable_path: process.execPath }, + start_url: { url: "https://example.test" }, + actions: [{ step_id: "settle", action: "wait_for_timeout", duration_ms: 1 }], +}); + +it.each([ + { environment: {}, sandbox: true }, + { environment: { REA_BROWSER_NO_SANDBOX: "false" }, sandbox: true }, + { environment: { REA_BROWSER_NO_SANDBOX: "1" }, sandbox: true }, + { environment: { REA_BROWSER_NO_SANDBOX: "true" }, sandbox: false }, +])( + "launches Chromium with sandbox $sandbox for $environment", + async ({ environment, sandbox }) => { + Object.defineProperty(process, "platform", { value: "linux" }); + let launchOptions: Parameters< + typeof import("playwright-core").chromium.launchPersistentContext + >[1]; + + await expect( + openPlaywrightScenarioBrowser(launchScenario, environment, { + launcher: { + connectOverCDP: async () => { + throw new Error("Unexpected connection"); + }, + launchPersistentContext: async (_profile, options) => { + launchOptions = options; + throw new Error("launch boundary reached"); + }, + }, + }), + ).rejects.toMatchObject({ + diagnostics: { startup_error: "launch boundary reached" }, + // Chromium's startup log, not REA, says whether the sandbox caused it. + remediationAction: sandbox + ? expect.stringContaining("REA_BROWSER_NO_SANDBOX=true") + : "Review the selected executable and startup diagnostics, then correct the launch configuration before retrying.", + }); + + expect(launchOptions?.chromiumSandbox).toBe(sandbox); + expect(launchOptions?.args).toEqual([]); + expect(launchOptions?.env).toEqual({}); + }, +); + it("waits for bounded event finalization and owned browser teardown", async () => { vi.useFakeTimers(); try { diff --git a/src/browser/PlaywrightScenarioBrowser.ts b/src/browser/PlaywrightScenarioBrowser.ts index 3df1e7b87b..9b525daffe 100644 --- a/src/browser/PlaywrightScenarioBrowser.ts +++ b/src/browser/PlaywrightScenarioBrowser.ts @@ -1,4 +1,8 @@ import { PLAYWRIGHT_BROWSER_SCENARIO_PROVIDER_IDENTITY } from "./providerIdentities.js"; +import { + chromiumLaunchRemediation, + chromiumSandboxEnabled, +} from "./ChromiumSandbox.js"; import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { @@ -869,6 +873,7 @@ export const openPlaywrightScenarioBrowser = async ( let context: BrowserContext; const executablePath = scenario.browser.executable_path; const headless = scenario.browser.headless; + const chromiumSandbox = chromiumSandboxEnabled(environment); let launchStarted = false; try { await chmod(profilePath, 0o700); @@ -894,13 +899,11 @@ export const openPlaywrightScenarioBrowser = async ( handleSIGINT: false, handleSIGTERM: false, timeout: commandTimeoutMs, + chromiumSandbox, // Termux's Chromium cannot create its renderer process through Android's // executable boundary. Single-process mode keeps the browser and page in // the Termux-launched process so scenario navigation remains available. - args: - process.platform === "android" - ? ["--no-sandbox", "--single-process"] - : [], + args: process.platform === "android" ? ["--single-process"] : [], }), commandTimeoutMs, "launch", @@ -923,8 +926,7 @@ export const openPlaywrightScenarioBrowser = async ( { cause, userMessage: "The selected browser could not start.", - remediationAction: - "Review the selected executable and startup diagnostics, then correct the launch configuration before retrying.", + remediationAction: chromiumLaunchRemediation(chromiumSandbox), diagnostics: { executable_path: executablePath, startup_error: diff --git a/src/browser/modules/NativeModuleResolver.test.ts b/src/browser/modules/NativeModuleResolver.test.ts index 1619e8fc44..1a11a15f9c 100644 --- a/src/browser/modules/NativeModuleResolver.test.ts +++ b/src/browser/modules/NativeModuleResolver.test.ts @@ -1,5 +1,8 @@ -import { describe, expect, it, vi } from "vitest"; -import { NativeModuleResolver } from "./NativeModuleResolver.js"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + NativeModuleResolver, + type ModuleResolverLauncher, +} from "./NativeModuleResolver.js"; import { webModuleResolutionObservationSchema } from "../../domain/webModuleTrace.js"; import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; @@ -218,6 +221,61 @@ describe("optional native resolution acquisition", () => { }); }); +describe("native resolution Chromium sandbox", () => { + const hostPlatform = process.platform; + afterEach(() => { + Object.defineProperty(process, "platform", { value: hostPlatform }); + }); + + it.each([ + { environment: {}, sandbox: true }, + { environment: { REA_BROWSER_NO_SANDBOX: "false" }, sandbox: true }, + { environment: { REA_BROWSER_NO_SANDBOX: "true" }, sandbox: false }, + ])( + "launches Chromium with sandbox $sandbox for $environment", + async ({ environment, sandbox }) => { + Object.defineProperty(process, "platform", { value: "linux" }); + let launchOptions: Parameters[0] | undefined; + const provider = new NativeModuleResolver( + { REA_BROWSER_EXECUTABLE: "/browser", ...environment }, + (options) => { + launchOptions = options; + return Promise.reject(new Error("launch boundary reached")); + }, + ); + const result = await provider.resolve(input); + if (result.ok) throw new Error("expected startup failure"); + expect(launchOptions?.chromiumSandbox).toBe(sandbox); + expect(launchOptions?.args).not.toContain("--no-sandbox"); + expect(result.error).toMatchObject({ + remediationAction: sandbox + ? expect.stringContaining("REA_BROWSER_NO_SANDBOX=true") + : undefined, + }); + }, + ); + + it("keeps the default recovery for engine failures after a sandboxed launch", async () => { + Object.defineProperty(process, "platform", { value: "linux" }); + const provider = new NativeModuleResolver( + { REA_BROWSER_EXECUTABLE: "/browser" }, + () => + Promise.resolve({ + version: () => "fixture", + newContext: () => + Promise.reject(new Error("fixture context failure")), + close: () => Promise.resolve(), + }), + ); + const result = await provider.resolve(input); + if (result.ok) throw new Error("expected engine failure"); + expect(result.error).toMatchObject({ + remediationAction: undefined, + diagnostics: { error_message: "fixture context failure" }, + }); + }); +}); + describe("completed native resolution finalization", () => { it.each([false, true])( "retains the completed batch when final close is cancelled (close fails: %s)", diff --git a/src/browser/modules/NativeModuleResolver.ts b/src/browser/modules/NativeModuleResolver.ts index 3cc45301e4..0e2215f542 100644 --- a/src/browser/modules/NativeModuleResolver.ts +++ b/src/browser/modules/NativeModuleResolver.ts @@ -1,4 +1,8 @@ import { snapshotEnvironment } from "../../process/snapshotEnvironment.js"; +import { + chromiumLaunchRemediation, + chromiumSandboxEnabled, +} from "../ChromiumSandbox.js"; import { isAbsolute } from "node:path"; import { z } from "zod"; import type { @@ -73,6 +77,20 @@ export type ModuleResolverLauncher = ( const launch: ModuleResolverLauncher = async (options) => (await import("playwright-core")).chromium.launch(options); +const moduleResolverFailure = ( + cause: unknown, + executable: string, + remediationAction: string | undefined, +): ProviderAdapterError => + new ProviderAdapterError("native-module-resolver", OPERATION, { + cause, + ...(remediationAction === undefined ? {} : { remediationAction }), + diagnostics: { + executable, + error_message: cause instanceof Error ? cause.message : String(cause), + }, + }); + /** Use native import.meta.resolve in an owned context with locally fulfilled requests. */ export class NativeModuleResolver implements WebModuleResolutionPort { #tail: Promise = Promise.resolve(); @@ -132,6 +150,7 @@ export class NativeModuleResolver implements WebModuleResolutionPort { }, ), ); + const chromiumSandbox = chromiumSandboxEnabled(this.environment); const deadline = new ProviderStartupDeadline(TIMEOUT_MS, options?.signal); let activeCall: | { settled: boolean; interruption?: AnalysisError } @@ -166,6 +185,7 @@ export class NativeModuleResolver implements WebModuleResolutionPort { ); }; let lease: AnalysisCleanupLease | undefined; + let launched = false; let cleanup: Promise | undefined; const close = (): Promise => { if (cleanup === undefined && lease !== undefined) { @@ -197,17 +217,16 @@ export class NativeModuleResolver implements WebModuleResolutionPort { handleSIGINT: false, handleSIGTERM: false, handleSIGHUP: false, + chromiumSandbox, args: [ "--disable-dev-shm-usage", "--disable-background-networking", "--renderer-process-limit=1", "--js-flags=--max-old-space-size=128", - ...(this.environment.REA_BROWSER_NO_SANDBOX === "true" - ? ["--no-sandbox"] - : []), ], }), ); + launched = true; lease = new AnalysisCleanupLease(async () => { if ( deadline.signal.aborted && @@ -239,14 +258,15 @@ export class NativeModuleResolver implements WebModuleResolutionPort { cause instanceof AnalysisCapabilityUnavailableError || cause instanceof BrowserObservationError ? cause - : new ProviderAdapterError("native-module-resolver", OPERATION, { + : moduleResolverFailure( cause, - diagnostics: { - executable, - error_message: - cause instanceof Error ? cause.message : String(cause), - }, - }), + executable, + // Only an unresolved sandboxed launch can be a sandbox failure; + // later engine failures keep the default recovery guidance. + launched || !chromiumSandbox + ? undefined + : chromiumLaunchRemediation(true), + ), ); } finally { deadline.signal.removeEventListener("abort", onAbort); From 25b62db838bca5305ef540ae33980778b0918017 Mon Sep 17 00:00:00 2001 From: ShuaixinHuang <1836656514@qq.com> Date: Thu, 5 Nov 2026 00:07:29 +0800 Subject: [PATCH 69/94] test(firmware): cover S-record high address workflows (#8897) --- tests/acceptance/hexRecordsCli.test.ts | 27 +++++++++++++ tests/boundary/mcp/hexRecordsMcp.test.ts | 50 ++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/tests/acceptance/hexRecordsCli.test.ts b/tests/acceptance/hexRecordsCli.test.ts index f085181eba..3c2169895a 100644 --- a/tests/acceptance/hexRecordsCli.test.ts +++ b/tests/acceptance/hexRecordsCli.test.ts @@ -39,6 +39,32 @@ const cases = [ ), }, }, + { + encoding: "srec", + name: "firmware.s28", + text: "S208FEDCBADEADBEEF2B\nS804FEDCBA67\n", + root: { + variant: "S28", + start_address: 0xfe_dcba, + start_address_record: "S8", + }, + regions: { + "0x00fedcba-0x00fedcbd.bin": Buffer.from("deadbeef", "hex"), + }, + }, + { + encoding: "srec", + name: "firmware.s37", + text: "S309FEDCBA98DEADBEEF92\nS705FEDCBA98CE\n", + root: { + variant: "S37", + start_address: 0xfedc_ba98, + start_address_record: "S7", + }, + regions: { + "0xfedcba98-0xfedcba9b.bin": Buffer.from("deadbeef", "hex"), + }, + }, ] as const; cliTest.for(cases)( @@ -135,6 +161,7 @@ cliTest.for(cases)( expect( Buffer.compare(await readFile(join(result.output_root, path)), bytes), ).toBe(0); + expect((await readFile(source)).equals(Buffer.from(text))).toBe(true); }, ); diff --git a/tests/boundary/mcp/hexRecordsMcp.test.ts b/tests/boundary/mcp/hexRecordsMcp.test.ts index 5d144967e8..388a9369c4 100644 --- a/tests/boundary/mcp/hexRecordsMcp.test.ts +++ b/tests/boundary/mcp/hexRecordsMcp.test.ts @@ -12,6 +12,56 @@ import { analysisViewResultSchema } from "../../../src/domain/analysisView/analy // The compiled stdio fixture checks advertised schemas against their dialect and // validates actual tool responses, avoiding a second source server module graph. +it.each(["full", "compact"] as const)( + "preserves S28 high address bytes through MCP inventory and extraction (%s)", + async (profile) => { + const root = await createTestTempDirectory("rea-s28-mcp-"); + const path = join(root, "firmware.s28"); + const bytes = Buffer.from("S208FEDCBADEADBEEF2B\nS804FEDCBA67\n"); + await writeFile(path, bytes); + const { call } = await connectLocalToolsMcp({ + REA_MCP_INPUT_SCHEMA_PROFILE: profile, + }); + let outputRoot: string | undefined; + try { + const opened = await call("open_binary", { path }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const inspected = await call("inspect_artifact", { detail: "summary" }); + expect(inspected.isError, JSON.stringify(inspected)).not.toBe(true); + const summary = z + .object({ + summary: z.object({ root_format_metadata: z.unknown() }), + }) + .parse(parseEvidence(inspected.structuredContent).normalized_result); + expect(summary.summary.root_format_metadata).toMatchObject({ + kind: "hex-records", + encoding: "srec", + variant: "S28", + lowest_address: 0xfe_dcba, + highest_address: 0xfe_dcbd, + start_address: 0xfe_dcba, + start_address_record: "S8", + }); + const extracted = await call("extract_artifact", {}); + expect(extracted.isError, JSON.stringify(extracted)).not.toBe(true); + const result = artifactExtractionResultSchema.parse( + parseEvidence(extracted.structuredContent).normalized_result, + ); + outputRoot = result.output_root; + expect(result.containment_verified).toBe(true); + expect( + (await readFile(join(outputRoot, "0x00fedcba-0x00fedcbd.bin"))).equals( + Buffer.from("deadbeef", "hex"), + ), + ).toBe(true); + expect((await readFile(path)).equals(bytes)).toBe(true); + } finally { + if (outputRoot !== undefined) + await rm(outputRoot, { recursive: true, force: true }); + } + }, +); + it.each(["full", "compact"] as const)( "exposes S-record facts and region extraction through MCP (%s)", async (profile) => { From 52bca6de410da5ea6022ad9d351f9ba67ad6074f Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:19:50 +0800 Subject: [PATCH 70/94] fix(native): retain value trace source evidence (#8384) * fix(native): retain value trace source evidence * fix(native): build trace error evidence only on failure * test(mcp): expect unowned value trace refusal * test(mcp): avoid freezing trace error wording * test(native): keep value-trace stop coverage with source evidence Main's stop and continuation cases now meet the trace's evidence-owner precondition: direct calls supply a source id, and a completed MCP trace runs in a session that can retain it. --------- Co-authored-by: Cursor Agent --- docs/native-investigation.md | 7 ++ scripts/lib/native-value-e2e.mjs | 32 +++++++ scripts/lib/real-ghidra-boundaries.mjs | 13 +++ scripts/verify-real-kuna.mjs | 24 +++++ src/android/JadxDoctor.ts | 6 +- .../AnalysisContextQueries.test.ts | 2 +- src/application/AnalysisProvider.ts | 2 + src/application/WorkflowAnalysisProfile.ts | 4 +- src/application/WorkflowEvidence.test.ts | 93 +++++++++++++++++++ src/application/WorkflowEvidence.ts | 25 ++++- .../binary/AnalysisSnapshotCache.test.ts | 65 +++++++++++++ src/application/binary/BinarySessionOpen.ts | 2 +- .../native/NativeFunctionMatchingService.ts | 6 +- .../native/NativeValueTrace.test.ts | 24 ++++- src/application/native/NativeValueTrace.ts | 29 ++++-- .../native/NativeValueTraceStops.test.ts | 22 ++++- src/browser/history/NodeCaptureAdapter.ts | 2 +- ....01-lists-the-complete-target-open.test.ts | 14 +-- .../nativeValueTraceCleanupIncomplete.test.ts | 13 ++- .../mcp/nativeValueTraceCompleteness.test.ts | 39 +++++++- .../mcp/nativeValueTraceFailures.test.ts | 14 +++ .../hopperDownloadHttpFailure.test.ts | 2 +- 22 files changed, 403 insertions(+), 37 deletions(-) diff --git a/docs/native-investigation.md b/docs/native-investigation.md index a60428100a..d821c2a81c 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -434,6 +434,13 @@ nodes and edges. Node payloads are bounded to 8 MiB and serialized results to 32 MiB. Pagination returns edges whose source is on the node page and preserves stable IDs for endpoints outside that page. +Procedure, node and edge `evidence_id` values identify the source function +observations used by the trace. In an MCP session, retrieve them from the +Evidence bundle for inspection or export. A replayed source keeps the original +observation's identity; the trace also reports that replay occurred. Older +cached value traces are not replayed under the revised workflow profile, while +their historical Evidence remains available. + The one-shot CLI accepts `--evidence_output ` to save the trace and the transitive source Evidence closure named by its links. The destination is never overwritten. `--detail summary` requires `--evidence_output`; the bundle diff --git a/scripts/lib/native-value-e2e.mjs b/scripts/lib/native-value-e2e.mjs index 959a62b6d8..a5f82569ac 100644 --- a/scripts/lib/native-value-e2e.mjs +++ b/scripts/lib/native-value-e2e.mjs @@ -215,6 +215,19 @@ async function verifyNativeValueAdapters( ); assert.deepEqual(mcpEvidence.normalized_result, mcpGraph); assert.deepEqual(mcpGraph, cliGraph); + const warmReply = await client.callTool( + { + name: "trace_native_values", + arguments: { procedure, max_depth: 2, max_functions: 8, limit: 3 }, + }, + { timeout: 180000 }, + ); + const warmEvidence = parseEvidence( + requireMcpOutput(warmReply, "trace_native_values"), + ); + const warmGraph = nativeValueTraceSchema.parse( + warmEvidence.normalized_result, + ); const analyzed = await client.callTool( { name: "analyze_function", @@ -276,6 +289,25 @@ async function verifyNativeValueAdapters( "MCP omitted structured Evidence bundle", ); const bundle = requireMcpEvidenceBundle(bundleReply); + for (const [evidence, graph] of [ + [mcpEvidence, mcpGraph], + [warmEvidence, warmGraph], + ]) { + for (const source of [ + ...graph.procedures, + ...graph.nodes, + ...graph.edges, + ]) { + assert.ok(evidence.evidence_links.includes(source.evidence_id)); + assert.equal( + bundle.records.find( + (record) => record.evidence_id === source.evidence_id, + )?.operation, + "analyze_function", + `Missing source Evidence ${source.evidence_id}`, + ); + } + } for (const evidence of [ mcpEvidence, functionEvidence, diff --git a/scripts/lib/real-ghidra-boundaries.mjs b/scripts/lib/real-ghidra-boundaries.mjs index e6abb17a26..7a6776f837 100644 --- a/scripts/lib/real-ghidra-boundaries.mjs +++ b/scripts/lib/real-ghidra-boundaries.mjs @@ -504,6 +504,9 @@ export async function verifyGhidraBoundaries( // Prime both inventories before editing so stale names cannot hide in caches. await call("list_names"); await call("list_procedures"); + const traceInput = { procedure: address, max_depth: 0, max_functions: 1 }; + const traceBefore = await evidence("trace_native_values", traceInput); + const sourceBefore = traceBefore.normalized_result.procedures[0].evidence_id; const changes = { procedure: address, name: "dead", @@ -517,6 +520,16 @@ export async function verifyGhidraBoundaries( comment: changes.comment, inline_comment: changes.inline_comment, }); + const traceAfter = await evidence("trace_native_values", traceInput); + const sourceAfter = traceAfter.normalized_result.procedures[0].evidence_id; + assert.notEqual(sourceBefore, sourceAfter); + const sourceProcedure = (evidenceId) => + call("inspect_analysis_view", { + source: { kind: "retained-evidence", evidence_id: evidenceId }, + view: { kind: "native", facet: "procedure", offset: 0, limit: 1 }, + }); + assert.equal((await sourceProcedure(sourceBefore)).item.name, name); + assert.equal((await sourceProcedure(sourceAfter)).item.name, changes.name); const regexPattern = "^(a|aa)*b$"; const regexFailure = await client.callTool( { diff --git a/scripts/verify-real-kuna.mjs b/scripts/verify-real-kuna.mjs index de799e3ddd..9c50ce88c9 100644 --- a/scripts/verify-real-kuna.mjs +++ b/scripts/verify-real-kuna.mjs @@ -511,6 +511,30 @@ const verifyKuna = async ( references, }); } + if (functions.length > 0) { + const trace = await cold.call("trace_native_values", { + procedure: functions[0][1], + max_depth: 0, + max_functions: 1, + }); + const bundle = await cold.call("get_evidence_bundle", { + detail: "complete", + }); + for (const source of [ + ...trace.value.procedures, + ...trace.value.nodes, + ...trace.value.edges, + ]) { + assert.ok(trace.structured.evidence_links.includes(source.evidence_id)); + assert.equal( + bundle.value.records.find( + (record) => record.evidence_id === source.evidence_id, + )?.operation, + "analyze_function", + `Missing source Evidence ${source.evidence_id}`, + ); + } + } } catch (cause) { coldFailure = cause; } finally { diff --git a/src/android/JadxDoctor.ts b/src/android/JadxDoctor.ts index 9667f49920..27074f6ea2 100644 --- a/src/android/JadxDoctor.ts +++ b/src/android/JadxDoctor.ts @@ -1,9 +1,11 @@ -import type { RetainedResourceScope } from "../application/AnalysisProvider.js"; +import type { + RetainedResourceScope, + ProviderAvailability, +} from "../application/AnalysisProvider.js"; import type { DoctorProviderCheck, DoctorProviderInspection, } from "../application/Doctor.js"; -import type { ProviderAvailability } from "../application/AnalysisProvider.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { inspectJadxAvailability, diff --git a/src/application/AnalysisContextQueries.test.ts b/src/application/AnalysisContextQueries.test.ts index 8322fadf43..392a9b1add 100644 --- a/src/application/AnalysisContextQueries.test.ts +++ b/src/application/AnalysisContextQueries.test.ts @@ -8,6 +8,7 @@ import { import { createAnalysisExecution, type AnalysisOperationPort, + type AnalysisExecution, } from "./AnalysisProvider.js"; import { AnalysisCapabilityUnavailableError, @@ -21,7 +22,6 @@ import { primaryAnalysisError, type AnalysisError, } from "../domain/analysisErrorBase.js"; -import type { AnalysisExecution } from "./AnalysisProvider.js"; import type { Result } from "../domain/result.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { err, ok } from "../domain/result.js"; diff --git a/src/application/AnalysisProvider.ts b/src/application/AnalysisProvider.ts index 63a498d810..59cdb245ae 100644 --- a/src/application/AnalysisProvider.ts +++ b/src/application/AnalysisProvider.ts @@ -34,6 +34,8 @@ export interface ExecutionOptions { readonly snapshotContext?: "one-shot-target"; readonly signal?: AbortSignal; readonly progress?: ProgressReporter; + /** The composed workflow will cite this observation and needs its Evidence. */ + readonly retainEvidence?: true; /** * Set when a procedure selector came from provider output, such as a * revisited callee, rather than from the caller. A provider-derived diff --git a/src/application/WorkflowAnalysisProfile.ts b/src/application/WorkflowAnalysisProfile.ts index f06cf5a215..eaf1b18d41 100644 --- a/src/application/WorkflowAnalysisProfile.ts +++ b/src/application/WorkflowAnalysisProfile.ts @@ -21,5 +21,7 @@ export const workflowAnalysisProfile = ( ? { workflow_revision: 4 } : operation === "get_call_graph" ? { workflow_revision: 3 } - : { workflow_revision: 1 }), + : operation === "trace_native_values" + ? { workflow_revision: 2 } + : { workflow_revision: 1 }), }); diff --git a/src/application/WorkflowEvidence.test.ts b/src/application/WorkflowEvidence.test.ts index deed631640..62699309a2 100644 --- a/src/application/WorkflowEvidence.test.ts +++ b/src/application/WorkflowEvidence.test.ts @@ -111,6 +111,99 @@ it.each([false, true])( const provider = { id: "fixture", name: "Fixture", version: "1" }; +it("retains requested live observations and links their original producer sources", async () => { + const upstream = createEvidence(undefined, provider, { + operation: "health", + parameters: {}, + result: "source", + }); + const result = await executeComposedWorkflow( + { + execute: async () => + ok( + createAnalysisExecution("live observation", provider, { + cachePolicy: "live", + sourceEvidence: [upstream], + }), + ), + }, + async (observed) => { + const value = await observed.execute( + "analyze_function", + { procedure: "0x1" }, + { + retainEvidence: true, + }, + ); + return value.ok ? ok({ source: value.value.evidenceId ?? null }) : value; + }, + ); + if (!result.ok) throw result.error; + const source = result.sourceEvidence?.find( + (record) => record.operation === "analyze_function", + ); + expect(source).toBeDefined(); + expect(source?.evidence_links).toEqual([upstream.evidence_id]); + expect(result.value.result).toEqual({ source: source?.evidence_id }); + expect(result.value.evidenceLinks).toContain(source?.evidence_id); + expect(result.value.cachePolicy).toBe("live"); +}); + +it("keeps an existing session-issued source identity on replay", async () => { + const original = createEvidence(undefined, provider, { + operation: "analyze_function", + parameters: { procedure: "0x1" }, + result: "original", + }); + const result = await executeComposedWorkflow( + { + execute: async () => + ok({ + ...createAnalysisExecution("original", provider, { + limitations: ["replayed"], + }), + evidenceId: original.evidence_id, + }), + }, + async (observed) => { + const value = await observed.execute( + "analyze_function", + { procedure: "0x1" }, + { + retainEvidence: true, + }, + ); + return value.ok ? ok({ source: value.value.evidenceId ?? null }) : value; + }, + ); + if (!result.ok) throw result.error; + expect(result.value.result).toEqual({ source: original.evidence_id }); + expect(result.value.evidenceLinks).toEqual([original.evidence_id]); + expect(result.sourceEvidence).toBeUndefined(); +}); + +it("retains a completed requested source if the composed workflow is cancelled", async () => { + const result = await executeComposedWorkflow( + { + execute: async () => ok(createAnalysisExecution("observed", provider)), + }, + async (observed) => { + await observed.execute( + "analyze_function", + { procedure: "0x1" }, + { + retainEvidence: true, + }, + ); + return err(new AnalysisCancelledError("trace_native_values")); + }, + ); + expect(result.ok).toBe(false); + expect(result.sourceEvidence?.map((record) => record.operation)).toEqual([ + "analyze_function", + ]); +}); + it("keeps Evidence identity stable when dependent branches finish in opposite orders", async () => { const run = async (order: readonly number[]) => { const gate = () => { diff --git a/src/application/WorkflowEvidence.ts b/src/application/WorkflowEvidence.ts index 2623fa1f4a..dd3c952279 100644 --- a/src/application/WorkflowEvidence.ts +++ b/src/application/WorkflowEvidence.ts @@ -1,6 +1,7 @@ import { AnalysisWorkScope } from "./AnalysisWorkScope.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; import { jsonStringByteLength } from "../domain/jsonStringByteLength.js"; +import { createAnalysisExecutionEvidence } from "../domain/analysisObservation.js"; import { REA_WORKFLOW_PROVIDER } from "../domain/investigationProviders.js"; import { workflowAnalysisProfile } from "./WorkflowAnalysisProfile.js"; import type { @@ -101,12 +102,10 @@ export const executeComposedWorkflow = async ( const observed: AnalysisOperationPort = { resourceScope: analysis.resourceScope, execute: async (operation, parameters, options) => { - const execution = await analysis.execute(operation, parameters, { + let execution = await analysis.execute(operation, parameters, { ...options, workScope, }); - if (execution.ok && execution.value.evidenceId !== undefined) - evidenceLinks.add(execution.value.evidenceId); if (execution.ok && execution.value.sourceEvidence !== undefined) { const recorded = recordEvidenceSources( (source) => sourceLedger.record(source), @@ -114,6 +113,26 @@ export const executeComposedWorkflow = async ( ); if (!recorded.ok) return recorded; } + if ( + execution.ok && + options?.retainEvidence === true && + execution.value.evidenceId === undefined + ) { + const source = createAnalysisExecutionEvidence( + operation, + parameters, + execution.value, + "observed", + ); + const recorded = recordEvidenceSources( + (evidence) => sourceLedger.record(evidence), + [source], + ); + if (!recorded.ok) return recorded; + execution = ok({ ...execution.value, evidenceId: source.evidence_id }); + } + if (execution.ok && execution.value.evidenceId !== undefined) + evidenceLinks.add(execution.value.evidenceId); if (execution.ok && execution.value.cachePolicy === "live") cachePolicy = "live"; if (execution.ok) diff --git a/src/application/binary/AnalysisSnapshotCache.test.ts b/src/application/binary/AnalysisSnapshotCache.test.ts index bf048e6f3e..59544d1f04 100644 --- a/src/application/binary/AnalysisSnapshotCache.test.ts +++ b/src/application/binary/AnalysisSnapshotCache.test.ts @@ -1182,6 +1182,71 @@ it("does not replay pre-envelope call graphs under the revised workflow profile" ).toBe(false); }); +it("keeps older value-trace Evidence as history without replaying its source IDs", () => { + const cache = new AnalysisSnapshotCache(); + const target = ANALYSIS_SNAPSHOT_TARGET; + const profile = ANALYSIS_SNAPSHOT_PROFILE; + const operation = "trace_native_values" as const; + const parameters = { procedure: "0x1000" }; + const oldProfile = workflowAnalysisProfile(profile); + const oldResult = { + seed: "0x1000", + target_sha256: target.sha256, + nodes: [], + edges: [], + procedures: [ + { + address: "0x1000", + depth: 0, + evidence_id: `ev_${"a".repeat(64)}`, + provider: profile.provider.id, + analysis_profile_digest: profile.digest, + }, + ], + unknowns: [], + total_nodes: 0, + total_edges: 0, + decompilations: 1, + next_offset: null, + page_truncated: false, + traversal: { status: "complete" }, + truncated: false, + limitations: [], + }; + const execution = { + ...createAnalysisExecution(oldResult, REA_WORKFLOW_PROVIDER, { + subject: target, + }), + analysisProfile: oldProfile, + }; + cache.recordWorkflow({ target, profile, operation, parameters, execution }); + const evidence = createEvidence(target, REA_WORKFLOW_PROVIDER, { + operation, + parameters, + result: oldResult, + confidence: "derived", + analysisProfile: oldProfile, + }); + const exported = cache.export( + target, + profile, + createEvidenceBundle([evidence]), + ); + if (!exported.ok) throw exported.error; + const snapshot = parseAnalysisSnapshot(exported.value); + expect(snapshot.evidence_bundle.records).toContainEqual(evidence); + expect( + snapshotEvidenceForQuery(snapshot, { + target, + bindingProfile: profile, + operation, + parameters, + provider: REA_WORKFLOW_PROVIDER, + evidenceProfile: workflowAnalysisProfile(profile, operation), + }), + ).toBeUndefined(); +}); + it("declines source-bearing snapshot entries while replaying omitted and empty source controls", () => { const cache = new AnalysisSnapshotCache(); const target = ANALYSIS_SNAPSHOT_TARGET; diff --git a/src/application/binary/BinarySessionOpen.ts b/src/application/binary/BinarySessionOpen.ts index 81b4655c4f..32cb1787aa 100644 --- a/src/application/binary/BinarySessionOpen.ts +++ b/src/application/binary/BinarySessionOpen.ts @@ -12,8 +12,8 @@ import { snapshotMatchesTarget, snapshotTarget, type SnapshotBindingMismatch, + type AnalysisSnapshot, } from "../../domain/analysisSnapshot.js"; -import type { AnalysisSnapshot } from "../../domain/analysisSnapshot.js"; import { parseBinaryTarget } from "../BinaryTargetResolver.js"; import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; import { diff --git a/src/application/native/NativeFunctionMatchingService.ts b/src/application/native/NativeFunctionMatchingService.ts index 02e75920cf..b3e67de062 100644 --- a/src/application/native/NativeFunctionMatchingService.ts +++ b/src/application/native/NativeFunctionMatchingService.ts @@ -10,7 +10,10 @@ import { } from "../../contracts/native/functionMatchingContracts.js"; import { createEvidence, type Evidence } from "../../domain/evidence.js"; import { readFunctionInventory } from "../../domain/native/functionInventory.js"; -import { nativeFunctionMatchingSchema } from "../../domain/native/functionMatching.js"; +import { + nativeFunctionMatchingSchema, + NATIVE_FUNCTION_MATCHING_PREDICATE, +} from "../../domain/native/functionMatching.js"; import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; @@ -20,7 +23,6 @@ import { resolveNativeMatchingSources } from "./NativeMatchingSources.js"; import type { ExecutionOptions } from "../AnalysisProvider.js"; import type { NativeMatchingAllowance } from "../../config/nativeMatchingAllowance.js"; import { FUNCTION_COMPARISON_PROVIDER } from "../../domain/comparisonEvidenceIdentity.js"; -import { NATIVE_FUNCTION_MATCHING_PREDICATE } from "../../domain/native/functionMatching.js"; const OPERATION = "match_native_functions"; diff --git a/src/application/native/NativeValueTrace.test.ts b/src/application/native/NativeValueTrace.test.ts index 1bd9fb1c7f..88b4cb0bfb 100644 --- a/src/application/native/NativeValueTrace.test.ts +++ b/src/application/native/NativeValueTrace.test.ts @@ -1,6 +1,7 @@ import { createAnalysisExecutionEvidence } from "../../domain/analysisObservation.js"; import { describe, expect, it } from "vitest"; -import { traceNativeValues } from "./NativeValueTrace.js"; +import { traceNativeValues as traceNativeValuesCore } from "./NativeValueTrace.js"; +import { executeComposedWorkflow } from "../WorkflowEvidence.js"; import { createAnalysisExecution, type AnalysisOperationPort, @@ -26,6 +27,27 @@ import { valueTraceProvider as provider, valueTraceSubject as subject, } from "./NativeValueTrace.fixture.js"; + +const traceNativeValues: typeof traceNativeValuesCore = async ( + source, + parameters, + signal, +) => { + const result = await executeComposedWorkflow(source, (observed) => + traceNativeValuesCore(observed, parameters, signal), + ); + return result.ok ? ok(result.value.result) : result; +}; + +it("does not return a source ID without an Evidence owner", async () => { + const result = await traceNativeValuesCore(valueTraceDiamond().port, { + procedure: "0x1000", + }); + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error._tag).toBe("AnalysisCapabilityUnavailableError"); +}); + const analysis: AnalysisOperationPort = { execute: async (operation, parameters) => { if (operation === "resolve_native_call_targets") diff --git a/src/application/native/NativeValueTrace.ts b/src/application/native/NativeValueTrace.ts index de1e2efb68..c80aaad55a 100644 --- a/src/application/native/NativeValueTrace.ts +++ b/src/application/native/NativeValueTrace.ts @@ -7,6 +7,7 @@ import { import { createAnalysisExecutionEvidence } from "../../domain/analysisObservation.js"; import { AnalysisCancelledError, + AnalysisCapabilityUnavailableError, AnalysisOutputError, } from "../../domain/analysisErrorCore.js"; import { err, ok } from "../../domain/result.js"; @@ -116,6 +117,7 @@ export const traceNativeValues = async ( { ...(signal === undefined ? {} : { signal }), ...(next.depth > 0 ? { selectorOrigin: "provider" as const } : {}), + retainEvidence: true, }, ); decompilations++; @@ -129,12 +131,13 @@ export const traceNativeValues = async ( } continue; } - const evidence = createAnalysisExecutionEvidence( - "analyze_function", - { procedure: next.procedure }, - execution.value, - "observed", - ); + const partialEvidence = () => + createAnalysisExecutionEvidence( + "analyze_function", + { procedure: next.procedure }, + execution.value, + "observed", + ); const parsed = functionDossierSchema.safeParse(execution.value.result); if (!parsed.success || execution.value.subject == null) return err( @@ -142,7 +145,7 @@ export const traceNativeValues = async ( "trace_native_values", "Function dossier lacks validated facts or artifact identity", parsed.success ? undefined : { cause: parsed.error }, - ).retainPartialObservation(evidence), + ).retainPartialObservation(partialEvidence()), ); const dossier = parsed.data; const address = dossier.procedure.address; @@ -155,9 +158,17 @@ export const traceNativeValues = async ( new AnalysisOutputError( "trace_native_values", "Cross-function target digest mismatch", - ).retainPartialObservation(evidence), + ).retainPartialObservation(partialEvidence()), + ); + const evidenceId = execution.value.evidenceId; + if (evidenceId === undefined) + return err( + new AnalysisCapabilityUnavailableError( + "rea", + "trace_native_values", + "The function observation has no Evidence retention owner.", + ).retainPartialObservation(partialEvidence()), ); - const evidenceId = evidence.evidence_id; procedures.push({ address, depth: next.depth, diff --git a/src/application/native/NativeValueTraceStops.test.ts b/src/application/native/NativeValueTraceStops.test.ts index fae841232e..40a1568d63 100644 --- a/src/application/native/NativeValueTraceStops.test.ts +++ b/src/application/native/NativeValueTraceStops.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import type { AnalysisOperationPort } from "../AnalysisProvider.js"; +import { ok } from "../../domain/result.js"; import { AnalysisCancelledError, AnalysisCapabilityUnavailableError, @@ -34,6 +35,21 @@ describe("native value trace child stops", () => { 0, VALUE_TRACE_DIAMOND_READS.indexOf(read) + 1, ); + // The trace cites each completed dossier. These cases cover stop behavior, + // so the fixture only has to satisfy that precondition. + const withSourceEvidence = ( + port: AnalysisOperationPort, + ): AnalysisOperationPort => ({ + execute: async (operation, parameters, options) => { + const execution = await port.execute(operation, parameters, options); + if (!execution.ok || execution.value.evidenceId !== undefined) + return execution; + return ok({ + ...execution.value, + evidenceId: `ev_${"ab".repeat(32)}`, + }); + }, + }); it.each(childReads)( "returns a cancelled %s read without a partial trace", @@ -43,7 +59,7 @@ describe("native value trace child stops", () => { [read]: cancelled, }); - const result = await traceNativeValues(port, { + const result = await traceNativeValues(withSourceEvidence(port), { procedure: "0x1000", max_depth: 1, }); @@ -86,7 +102,7 @@ describe("native value trace child stops", () => { }; const result = await traceNativeValues( - aborting, + withSourceEvidence(aborting), { procedure: "0x1000", max_depth: 1 }, controller.signal, ); @@ -114,7 +130,7 @@ describe("native value trace child stops", () => { "resolve_native_call_targets:0x1008": failure, }); - const result = await traceNativeValues(port, { + const result = await traceNativeValues(withSourceEvidence(port), { procedure: "0x1000", max_depth: 1, }); diff --git a/src/browser/history/NodeCaptureAdapter.ts b/src/browser/history/NodeCaptureAdapter.ts index 785b157220..6b0a7abd74 100644 --- a/src/browser/history/NodeCaptureAdapter.ts +++ b/src/browser/history/NodeCaptureAdapter.ts @@ -10,8 +10,8 @@ import { NODE_CAPTURE_HEAP_ENFORCED, captureDecoderProcessMemoryLimitBytes, } from "./CaptureRelease.js"; -import { QLOG_CAPTURE_PROVIDER_IDENTITY } from "./CaptureProviderIdentities.js"; import { + QLOG_CAPTURE_PROVIDER_IDENTITY, HAR_CAPTURE_PROVIDER_IDENTITY, SAZ_CAPTURE_PROVIDER_IDENTITY, } from "./CaptureProviderIdentities.js"; diff --git a/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts b/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts index 68313a7be4..d1eacd2063 100644 --- a/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts +++ b/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts @@ -440,14 +440,14 @@ describe("enhanced MCP tools", () => { ]), }, }); + // A fixed adapter has no session to retain source Evidence for the trace. expect(results[13]).toMatchObject({ - target_sha256: "a".repeat(64), - decompilations: 1, - unknowns: [ - expect.objectContaining({ - reason: "Provider has no native def-use model", - }), - ], + error: { + code: "capability_unavailable", + details: { + operation: "trace_native_values", + }, + }, }); expect(results[14]).toMatchObject({ counts: { selected: 6, written: 6, failed: 0, excluded: 0 }, diff --git a/tests/boundary/mcp/nativeValueTraceCleanupIncomplete.test.ts b/tests/boundary/mcp/nativeValueTraceCleanupIncomplete.test.ts index 6e28b05750..29e1a695ba 100644 --- a/tests/boundary/mcp/nativeValueTraceCleanupIncomplete.test.ts +++ b/tests/boundary/mcp/nativeValueTraceCleanupIncomplete.test.ts @@ -159,10 +159,15 @@ describe("trace_native_values after a validated seed", () => { "keeps an ordinary failure from $read as a located unknown and continues", async ({ read, stop, ordinaryReads }) => { const operation = read.split(":")[0] ?? ""; - const { port, reads } = failingDiamond({ - [read]: childFailure(operation, false), - }); - const client = await connect(port); + // A completed trace cites source Evidence, so continuation needs a + // session that can retain it. A fixed adapter still refuses that result. + const { port, reads } = failingDiamond( + { + [read]: childFailure(operation, false), + }, + { sessionSubject: true }, + ); + const { client } = await openSession(port); const response = await client.callTool(trace); diff --git a/tests/boundary/mcp/nativeValueTraceCompleteness.test.ts b/tests/boundary/mcp/nativeValueTraceCompleteness.test.ts index 35e6332b2e..04ddd5abfb 100644 --- a/tests/boundary/mcp/nativeValueTraceCompleteness.test.ts +++ b/tests/boundary/mcp/nativeValueTraceCompleteness.test.ts @@ -2,9 +2,17 @@ import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { afterEach, expect, it } from "vitest"; import { valueTraceDiamond } from "../../../src/application/native/NativeValueTrace.fixture.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; +import { evidenceBundleSchema } from "../../../src/domain/evidenceBundle.js"; +import { z } from "zod"; import { nativeValueTraceSchema } from "../../../src/domain/native/nativeValueTrace.js"; import { advertisedMcpToolSchemas } from "../../fixtures/advertisedMcpTool.js"; import { createServer } from "../../../src/server/createServer.js"; +import { ok } from "../../../src/domain/result.js"; +import { + createBinarySessionTargets, + createTestBinarySession, +} from "../../fixtures/binarySession.js"; +import { observed } from "../../fixtures/analysisExecution.js"; import { closeEnhancedToolResources, resources, @@ -15,8 +23,19 @@ afterEach(closeEnhancedToolResources); it.each(["full", "compact"] as const)( "separates page omission from named traversal budgets through the advertised %s MCP contract", async (profile) => { + const [target] = await createBinarySessionTargets(); + const analysis = valueTraceDiamond({ sessionSubject: true }).port; + const session = createTestBinarySession(() => ({ + execute: (operation, parameters, options) => + operation === "health" + ? Promise.resolve(observed(null)) + : analysis.execute(operation, parameters, options), + close: async () => ok(null), + })); + const opened = await session.open(target); + expect(opened.ok).toBe(true); const server = createServer( - { kind: "fixed", analysis: valueTraceDiamond().port }, + { kind: "session", session }, { environment: { REA_MCP_OUTPUT_SCHEMA_PROFILE: profile } }, ); const client = new Client({ name: "value-trace-test", version: "1.0.0" }); @@ -59,6 +78,24 @@ it.each(["full", "compact"] as const)( traversal: { status: "complete" }, truncated: false, }); + const bundleReply = await client.callTool({ + name: "get_evidence_bundle", + arguments: { detail: "complete" }, + }); + expect(bundleReply.isError).not.toBe(true); + const bundle = z + .object({ result: evidenceBundleSchema }) + .parse(bundleReply.structuredContent).result; + for (const source of [ + ...complete.procedures, + ...complete.nodes, + ...complete.edges, + ]) + expect( + bundle.records.some( + (record) => record.evidence_id === source.evidence_id, + ), + ).toBe(true); const pages = []; for (let offset: number | null = 0; offset !== null;) { const page = await trace({ ...request, offset, limit: 4 }); diff --git a/tests/boundary/mcp/nativeValueTraceFailures.test.ts b/tests/boundary/mcp/nativeValueTraceFailures.test.ts index 4d0540411b..c66c48fcb5 100644 --- a/tests/boundary/mcp/nativeValueTraceFailures.test.ts +++ b/tests/boundary/mcp/nativeValueTraceFailures.test.ts @@ -3,10 +3,24 @@ import { AnalysisInputError } from "../../../src/domain/analysisErrorCore.js"; import { NoBinaryOpenError } from "../../../src/domain/configurationErrors.js"; import { err } from "../../../src/domain/result.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { valueTraceDiamond } from "../../../src/application/native/NativeValueTrace.fixture.js"; import { closeEnhancedToolResources, connect } from "./enhancedToolsHarness.js"; afterEach(closeEnhancedToolResources); +it("refuses a completed trace when the adapter has no source Evidence owner", async () => { + const client = await connect(valueTraceDiamond().port); + const result = await client.callTool({ + name: "trace_native_values", + arguments: { procedure: "0x1000" }, + }); + expect(result.isError).toBe(true); + expect(parseMcpToolError(result).error).toMatchObject({ + code: "capability_unavailable", + message: expect.stringContaining("source Evidence"), + }); +}); + it.each([ { failure: new AnalysisInputError("analyze_function", { diff --git a/tests/process-global/hopperDownloadHttpFailure.test.ts b/tests/process-global/hopperDownloadHttpFailure.test.ts index 98cd0c707a..07cd9cca02 100644 --- a/tests/process-global/hopperDownloadHttpFailure.test.ts +++ b/tests/process-global/hopperDownloadHttpFailure.test.ts @@ -95,7 +95,7 @@ const vendor = ( return { fetch }; }; const failed = (status: number, cancel?: () => void) => () => - new Response(new ReadableStream({ ...(cancel ? { cancel } : {}) }), { + new Response(new ReadableStream(cancel ? { cancel } : {}), { status, }); From 63b66fa4981ee7688ac39492bca29ff89d0b120e Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:20:01 +0800 Subject: [PATCH 71/94] fix(search): clarify regex diagnostics and recovery (#8776) * fix(ghidra): identify invalid regex patterns * docs(native): clarify regex dialects in search contracts * fix(ghidra): guide regex syntax recovery * fix(search): guide regex worker recovery * fix(search): preserve regex parser diagnostics * test(search): update regex boundary expectations * refactor(search): remove unused regex error hook --------- Co-authored-by: Cursor Agent --- bridge/ghidra/ReaGhidraBridge.java | 2 +- src/contracts/officialToolContracts.ts | 2 +- src/ghidra/GhidraProviderClient.ts | 20 ++++++ src/ghidra/GhidraSessionError.ts | 1 + src/process/RegexMatcher.ts | 26 ++++---- .../mcp/ghidraRegexDiagnosticsMcp.test.ts | 63 +++++++++++++++++++ tests/boundary/process/regexMatcher.test.ts | 6 +- 7 files changed, 103 insertions(+), 17 deletions(-) create mode 100644 tests/boundary/mcp/ghidraRegexDiagnosticsMcp.test.ts diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index c815e8edf6..de41cf4261 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -4929,7 +4929,7 @@ public final class ReaGhidraBridge extends HeadlessScript { // without retaining its cause. This producer reason is not bad syntax. if (exception.getDescription().equals("Stack overflow during pattern compilation")) throw regexStackExhausted(expression, "compiling"); - throw new RequestFailure("invalid_request", "Invalid regex pattern " + expression + ": " + exception.getDescription() + " at index " + exception.getIndex()); + throw new RequestFailure("invalid_regex_pattern", "Invalid regex pattern " + expression + ": " + exception.getDescription() + " at index " + exception.getIndex()); } catch (StackOverflowError exhausted) { throw regexStackExhausted(expression, "compiling"); diff --git a/src/contracts/officialToolContracts.ts b/src/contracts/officialToolContracts.ts index 9788b866a9..e1da025271 100644 --- a/src/contracts/officialToolContracts.ts +++ b/src/contracts/officialToolContracts.ts @@ -21,7 +21,7 @@ import { HOPPER_MEMORY_TOOL_DEFINITIONS } from "./hopperMemoryContracts.js"; import { nativeSearchInput } from "../domain/native/nativeSearch.js"; const hopperRegexDeadline = - "Hopper regex mode uses ECMAScript Unicode syntax in a cancellable worker. The matching deadline is REA_REGEX_MATCHING_DEADLINE_MS (a positive decimal integer of milliseconds from 1 to 2147483647, default 5000). Unset keeps the default. A deadline failure names the effective milliseconds, source (default or setting), the default, and the maximum, and leaves the API usable. The worker old-generation request is REA_REGEX_WORKER_HEAP_REQUEST_MIB (a decimal integer of mebibytes from 7 to 16384, default 128). Unset keeps the default. A positive Node --max-old-space-size, including NODE_OPTIONS, overrides that request and is not limited by the setting's floor or maximum. A heap failure names the effective request, its source (default or setting), the default, the floor, and the maximum."; + "Regex dialect depends on the provider: Ghidra uses Java Pattern; Kuna, IDA and Hopper use ECMAScript Unicode. See binary_session for selected-provider limits. Hopper regex mode uses a cancellable worker. The matching deadline is REA_REGEX_MATCHING_DEADLINE_MS (a positive decimal integer of milliseconds from 1 to 2147483647, default 5000). Unset keeps the default. A deadline failure names the effective milliseconds, source (default or setting), the default, and the maximum, and leaves the API usable. The worker old-generation request is REA_REGEX_WORKER_HEAP_REQUEST_MIB (a decimal integer of mebibytes from 7 to 16384, default 128). Unset keeps the default. A positive Node --max-old-space-size, including NODE_OPTIONS, overrides that request and is not limited by the setting's floor or maximum. A heap failure names the effective request, its source (default or setting), the default, the floor, and the maximum."; const official = < Name extends keyof typeof officialOutputSchemas, diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index c7b69896ad..7eb4052e7b 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -1427,6 +1427,26 @@ const projectPrimarySessionError = ( }, ); } + if ( + failure.kind === "remote" && + failure.remoteCode === "invalid_regex_pattern" + ) + return new AnalysisInputError( + operation, + { + ...context, + userMessage: failure.message, + remediationAction: + "Correct the pattern using the reported Ghidra regex syntax error, or select mode literal to search for the text literally, then retry.", + }, + [ + { + path: ["pattern"], + reason: "invalid_format", + message: failure.message, + }, + ], + ); if (failure.kind === "remote" && isGhidraInputFailureCode(failure.remoteCode)) return new AnalysisInputError(operation, context, [ { path: [], reason: "invalid_value", message: failure.message }, diff --git a/src/ghidra/GhidraSessionError.ts b/src/ghidra/GhidraSessionError.ts index 8201e12415..eda2c72666 100644 --- a/src/ghidra/GhidraSessionError.ts +++ b/src/ghidra/GhidraSessionError.ts @@ -190,6 +190,7 @@ const createGhidraSessionError = (failure: { /** Bridge input failures share classification with their domain error projection. */ export const isGhidraInputFailureCode = (code: string | undefined): boolean => code === "invalid_request" || + code === "invalid_regex_pattern" || code === "not_found" || code === "procedure_not_found" || code === "file_offset_unmapped" || diff --git a/src/process/RegexMatcher.ts b/src/process/RegexMatcher.ts index 028f2e1172..d33fc4baad 100644 --- a/src/process/RegexMatcher.ts +++ b/src/process/RegexMatcher.ts @@ -552,7 +552,6 @@ export const matchRegexValues = async (input: { readonly caseSensitive: boolean; readonly values: readonly string[]; readonly signal?: AbortSignal | undefined; - readonly invalidPattern?: (reason: string) => AnalysisError; /** Resolved deadline. Omitted means the 5000 ms default, not ambient `process.env`. */ readonly deadline?: RegexMatchingDeadline | undefined; /** Resolved heap request. Omitted means the 128 MiB default, not ambient `process.env`. */ @@ -623,20 +622,21 @@ export const matchRegexValues = async (input: { if (reply.data.status === "invalid_pattern") return finish( err( - input.invalidPattern?.(reply.data.reason) ?? - new AnalysisInputError( - input.operation, + new AnalysisInputError( + input.operation, + { + userMessage: `Invalid regex pattern: ${reply.data.reason}`, + remediationAction: + "Correct the pattern using the reported regex syntax error, or select mode literal to search for the text literally, then retry.", + }, + [ { - userMessage: `Invalid regex pattern: ${reply.data.reason}`, + path: ["pattern"], + reason: "invalid_format", + message: reply.data.reason, }, - [ - { - path: ["pattern"], - reason: "invalid_format", - message: "Supply a valid regular expression.", - }, - ], - ), + ], + ), ), ); if ( diff --git a/tests/boundary/mcp/ghidraRegexDiagnosticsMcp.test.ts b/tests/boundary/mcp/ghidraRegexDiagnosticsMcp.test.ts new file mode 100644 index 0000000000..54276721f8 --- /dev/null +++ b/tests/boundary/mcp/ghidraRegexDiagnosticsMcp.test.ts @@ -0,0 +1,63 @@ +import { expect, it } from "vitest"; +import { err, ok } from "../../../src/domain/result.js"; +import { GhidraSessionError } from "../../../src/ghidra/GhidraSessionError.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { connectGhidraMcp } from "./ghidraMcpHarness.js"; + +it.each(["compact", "full"] as const)( + "identifies the invalid Ghidra regex field and permits correction (%s)", + async (profile) => { + const message = + "Invalid regex pattern [: Unclosed character class at index 0"; + const harness = await connectGhidraMcp( + "ghidra-regex-diagnostic", + (operation, input) => + Promise.resolve( + input.pattern === "[" + ? err( + new GhidraSessionError( + "remote", + message, + {}, + { remoteCode: "invalid_regex_pattern" }, + ), + ) + : ok([]), + ), + { environment: { REA_MCP_OUTPUT_SCHEMA_PROFILE: profile } }, + ); + try { + for (const operation of [ + "search_strings", + "search_procedures", + ] as const) { + const rejected = await harness.mcp.callTool({ + name: operation, + arguments: { pattern: "[", mode: "regex" }, + }); + const error = parseMcpToolError(rejected).error; + expect(error).toMatchObject({ + code: "invalid_request", + message, + remediation: { + action: expect.stringMatching(/pattern.*mode literal/u), + }, + details: { + issues: [{ path: ["pattern"], reason: "invalid_format", message }], + }, + }); + expect(error.recovery).toMatchObject({ + kind: "inspect_input_contract", + operation, + }); + const corrected = await harness.mcp.callTool({ + name: operation, + arguments: { pattern: "valid", mode: "regex" }, + }); + expect(corrected.isError).not.toBe(true); + } + } finally { + await harness.close(); + } + }, +); diff --git a/tests/boundary/process/regexMatcher.test.ts b/tests/boundary/process/regexMatcher.test.ts index b6b8c4c94b..75f4f65f12 100644 --- a/tests/boundary/process/regexMatcher.test.ts +++ b/tests/boundary/process/regexMatcher.test.ts @@ -583,6 +583,7 @@ it("names the rejected pattern field for a default invalid-pattern refusal", asy expect(projectAnalysisError(matched.error)).toMatchObject({ code: "invalid_request", category: "invalid_input", + remediation: { action: expect.stringMatching(/pattern.*mode literal/u) }, recovery: { kind: "inspect_input_contract", operation: "search_procedures", @@ -593,7 +594,7 @@ it("names the rejected pattern field for a default invalid-pattern refusal", asy { path: ["pattern"], reason: "invalid_format", - message: "Supply a valid regular expression.", + message: expect.stringContaining("Invalid regular expression"), }, ], }, @@ -620,6 +621,7 @@ it("keeps Hopper's result projection and invalid-pattern contract with the share code: "invalid_request", category: "invalid_input", message: expect.stringContaining("Invalid regex pattern"), + remediation: { action: expect.stringMatching(/pattern.*mode literal/u) }, recovery: { kind: "inspect_input_contract", operation: "search_strings" }, details: { operation: "search_strings", @@ -627,7 +629,7 @@ it("keeps Hopper's result projection and invalid-pattern contract with the share { path: ["pattern"], reason: "invalid_format", - message: "Supply a valid regular expression.", + message: expect.stringContaining("Invalid regular expression"), }, ], }, From 5990ff0324de38042ea95ecdf5c369b295aa4bbd Mon Sep 17 00:00:00 2001 From: blankym Date: Thu, 5 Nov 2026 00:52:20 +0800 Subject: [PATCH 72/94] feat(artifacts): read WeChat mini-program wxapkg containers (#8148) * feat(artifacts): read WeChat mini-program wxapkg containers Add a read-only reader for plaintext wxapkg packages: the 0xBE/0xED 18-byte big-endian header, the bounded name/offset/size member table, and verbatim member bytes at absolute carrier offsets. - Members retain declared names, absolute offsets and sizes, so source_location reports real carrier positions. - V1MMWX PC-client packages are detected as encrypted coverage with null member facts; extraction refuses with a typed error and no decryption is attempted. - Bounded table and directory budgets are admitted before allocation; malformed marks, truncated tables, invalid UTF-8 names, member extents inside the header or beyond the file, and counts that cannot fit their table are typed format or limit failures. - Member reads stream in bounded 64 KiB chunks with carrier freshness checks; the reader joins outstanding streams before close and bounds active streams and repeated member I/O. Refs #8121. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(artifacts): bound wxapkg members to the declared data region Normalize the two indexLength conventions before the table bounds check and the table read. Member ranges must then lie in that declared data region, which has to fit in the file. Record wxapkg format metadata as not compared: header facts repeat the package bytes, and entry source paths keep a producer root prefix. * fix(artifacts): choose the wxapkg index after member ranges fit Four trailing bytes made the rows-only layout end on the file and dropped the count-inclusive layout. Keep every admitted layout until the parsed rows and member ranges select one. --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Cursor Agent --- .../references/native-and-artifacts.md | 7 + docs/cli.md | 11 + docs/index.md | 26 +- docs/wxapkg.md | 84 ++++ src/application/BinaryTargetResolver.ts | 6 + .../binary/TargetFormatVocabularies.test.ts | 1 + src/artifacts/ArtifactReader.ts | 1 + .../WxapkgArtifactReader.boundaries.test.ts | 310 ++++++++++++ src/artifacts/WxapkgArtifactReader.test.ts | 468 ++++++++++++++++++ src/artifacts/WxapkgArtifactReader.ts | 311 ++++++++++++ src/artifacts/WxapkgStructures.ts | 214 ++++++++ src/artifacts/artifactFormats.test.ts | 1 + src/artifacts/artifactFormats.ts | 11 + .../extraction/ArtifactExtraction.ts | 9 + .../inventory/ArtifactGraphConstruction.ts | 9 +- src/domain/analysisSnapshot.ts | 1 + .../analysisView/artifactInventoryView.ts | 1 + src/domain/artifactFormatMetadata.ts | 28 ++ src/domain/artifactGraph.ts | 2 + src/domain/binaryTargetTypes.ts | 1 + src/domain/evidenceSubject.ts | 1 + src/domain/liveBinaryTargetSummary.ts | 1 + src/domain/wxapkgFormat.ts | 198 ++++++++ tests/boundary/mcp/wxapkgMcp.test.ts | 87 ++++ tests/fixtures/wxapkg.ts | 114 +++++ 25 files changed, 1889 insertions(+), 14 deletions(-) create mode 100644 docs/wxapkg.md create mode 100644 src/artifacts/WxapkgArtifactReader.boundaries.test.ts create mode 100644 src/artifacts/WxapkgArtifactReader.test.ts create mode 100644 src/artifacts/WxapkgArtifactReader.ts create mode 100644 src/artifacts/WxapkgStructures.ts create mode 100644 src/domain/wxapkgFormat.ts create mode 100644 tests/boundary/mcp/wxapkgMcp.test.ts create mode 100644 tests/fixtures/wxapkg.ts diff --git a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md index 35576998f9..5e854285d2 100644 --- a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -708,6 +708,13 @@ cache one decoded block and support retained filtered/paged views. No decryption or SerializedFile object decoding is provided. See [UnityFS](https://github.com/morluto/rea/blob/main/docs/unityfs.md). +WeChat mini-program packages (`.wxapkg` files with `0xBE`/`0xED` marks) list +members with declared names and absolute carrier offsets; member bytes are +stored verbatim. `V1MMWX` PC-client packages report `encrypted` directory +status with unknown coverage and are refused on extraction; no decryption is +attempted. See +[wxapkg](https://github.com/morluto/rea/blob/main/docs/wxapkg.md). + U-Boot legacy uImages use the same inspection/extraction workflows. Prefer a retained summary and selected inventory views for header facts or component IDs. Header addresses/CPU/OS are declarations; payloads remain as stored. diff --git a/docs/cli.md b/docs/cli.md index 28af2b71e4..3380b0e18f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -281,6 +281,17 @@ Format versions 6 through 8 retain header, block and directory facts with none/LZ4/LZ4HC block decompression. See [UnityFS](unityfs.md) for encryption and LZMA coverage boundaries. +Inspect a WeChat mini-program package without WeChat or an unpacker: + +```sh +rea inspect-artifact app.wxapkg --detail summary --evidence_output evidence.json --json +rea extract-artifact app.wxapkg --json +``` + +Plaintext packages retain header, table and member facts with verbatim member +bytes. `V1MMWX` packages are detected as encrypted coverage and refused on +extraction. See [wxapkg](wxapkg.md) for layout and coverage boundaries. + Inspect an ESP32-family app or bootloader image without esptool: ```sh diff --git a/docs/index.md b/docs/index.md index bd9cbc36a5..7cc44bdc1d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -47,19 +47,19 @@ For runtime instrumentation, see the [Frida guide](./frida-instrumentation.md). ## Choose a guide -| Task | Guides | -| --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Identify a desktop installation | [Runtime candidates, evidence and follow-up files](desktop-applications.md) | -| Inspect native functions, references, or types | [Native binaries](native-investigation.md), [Go metadata](go-binaries.md), [Rust dependencies](rust-binaries.md), [CPython pyc files](python-pyc.md), [PE certificates](pe-certificates.md), [PE imports](pe-imports.md), [PE TLS/load config](pe-load-config.md), [ELF notes](elf-notes.md), [Linux kernel modules](kernel-modules.md), [Apple IPS](apple-ips.md), [Erlang BEAM](erlang-beam.md), [offline ELF layout](binary-diagnostics.md), [recorded crashes](recorded-crashes.md) | -| Select a native provider or import format | [Ghidra semantics](ghidra-provider.md), [Windows](windows-ghidra-p0.md), [DOS](ghidra-dos.md), [raw bytes](raw-binary.md), [GameCube DOL](ghidra-dol.md), [MIPS](ghidra-mips.md), [PSP](ghidra-psp.md), [NativeAOT](ghidra-nativeaot.md), [IDA](ida-provider.md), [Kuna](kuna-provider.md), [JEB](jeb-analysis.md) | -| Analyze JavaScript or Electron artifacts | [Static analysis](javascript-artifact-reconstruction.md), [tracing and comparisons](javascript-application-workflows.md), [application graph](javascript-application-graph.md), [source recovery](javascript-recovery.md) | -| Compare static JavaScript with runtime observations | [Runtime attribution](javascript-runtime-observation.md), [reconciliation](javascript-runtime-reconciliation.md), [Electron observation](electron-observation.md), [Electron fuses](electron-fuses.md) | -| Inspect assemblies, packages, or firmware | [Godot PCK](godot-pck.md), [UnityFS](unityfs.md), [U-Boot uImage](uimage.md), [.NET](managed-code-analysis.md), [Python packages](python-packaged-applications.md), [Android](android-analysis.md), [APK signatures](apk-signature.md), [Apktool resources](apktool-resource-analysis.md), [ADB acquisition](adb-device-analysis.md), [Apple](apple-application-analysis.md), [Harmony](harmony-analysis.md), [firmware](firmware-analysis.md), [ESP images](esp-image.md), [ESP flash images](esp-flash-image.md), [WASM](wasm-artifacts.md), [EVM](evm-bytecode.md) | -| Observe or compare a browser scenario | [CDP observation](browser-observation.md), [scenarios](browser-scenario-contract.md), [runtime evidence](web-runtime.md), [historical captures](web-network-captures.md) | -| Serve recorded behavior as an agent environment | [Agent environments](agent-environments.md) | -| Trace captured website source | [Source locations](web-source-location.md), [module traces](website-module-trace.md), [script export](website-script-export.md) | -| Capture and compare a command | [Process capture](process-capture.md) | -| Use CLI/MCP and retain evidence | [CLI](cli.md), [MCP contracts](mcp-contracts.md), [agent prompts](mcp-prompts.md), [obligation ledgers](reconstruction-obligation-ledgers.md) | +| Task | Guides | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Identify a desktop installation | [Runtime candidates, evidence and follow-up files](desktop-applications.md) | +| Inspect native functions, references, or types | [Native binaries](native-investigation.md), [Go metadata](go-binaries.md), [Rust dependencies](rust-binaries.md), [CPython pyc files](python-pyc.md), [PE certificates](pe-certificates.md), [PE imports](pe-imports.md), [PE TLS/load config](pe-load-config.md), [ELF notes](elf-notes.md), [Linux kernel modules](kernel-modules.md), [Apple IPS](apple-ips.md), [Erlang BEAM](erlang-beam.md), [offline ELF layout](binary-diagnostics.md), [recorded crashes](recorded-crashes.md) | +| Select a native provider or import format | [Ghidra semantics](ghidra-provider.md), [Windows](windows-ghidra-p0.md), [DOS](ghidra-dos.md), [raw bytes](raw-binary.md), [GameCube DOL](ghidra-dol.md), [MIPS](ghidra-mips.md), [PSP](ghidra-psp.md), [NativeAOT](ghidra-nativeaot.md), [IDA](ida-provider.md), [Kuna](kuna-provider.md), [JEB](jeb-analysis.md) | +| Analyze JavaScript or Electron artifacts | [Static analysis](javascript-artifact-reconstruction.md), [tracing and comparisons](javascript-application-workflows.md), [application graph](javascript-application-graph.md), [source recovery](javascript-recovery.md) | +| Compare static JavaScript with runtime observations | [Runtime attribution](javascript-runtime-observation.md), [reconciliation](javascript-runtime-reconciliation.md), [Electron observation](electron-observation.md), [Electron fuses](electron-fuses.md) | +| Inspect assemblies, packages, or firmware | [Godot PCK](godot-pck.md), [UnityFS](unityfs.md), [wxapkg](wxapkg.md), [U-Boot uImage](uimage.md), [.NET](managed-code-analysis.md), [Python packages](python-packaged-applications.md), [Android](android-analysis.md), [APK signatures](apk-signature.md), [Apktool resources](apktool-resource-analysis.md), [ADB acquisition](adb-device-analysis.md), [Apple](apple-application-analysis.md), [Harmony](harmony-analysis.md), [firmware](firmware-analysis.md), [ESP images](esp-image.md), [ESP flash images](esp-flash-image.md), [WASM](wasm-artifacts.md), [EVM](evm-bytecode.md) | +| Observe or compare a browser scenario | [CDP observation](browser-observation.md), [scenarios](browser-scenario-contract.md), [runtime evidence](web-runtime.md), [historical captures](web-network-captures.md) | +| Serve recorded behavior as an agent environment | [Agent environments](agent-environments.md) | +| Trace captured website source | [Source locations](web-source-location.md), [module traces](website-module-trace.md), [script export](website-script-export.md) | +| Capture and compare a command | [Process capture](process-capture.md) | +| Use CLI/MCP and retain evidence | [CLI](cli.md), [MCP contracts](mcp-contracts.md), [agent prompts](mcp-prompts.md), [obligation ledgers](reconstruction-obligation-ledgers.md) | For development, use [testing](testing.md), [client verification](client-verification.md), [tool design](tool-design.md), [admission limits and recovery](design/limits-and-recovery.md), diff --git a/docs/wxapkg.md b/docs/wxapkg.md new file mode 100644 index 0000000000..e898350d83 --- /dev/null +++ b/docs/wxapkg.md @@ -0,0 +1,84 @@ +# WeChat mini-program wxapkg archives + +`inspect-artifact` and `extract-artifact` read plaintext WeChat mini-program +package (`.wxapkg`) containers without running WeChat or an unpacker. The +plaintext form starts with a `0xBE` mark and an 18-byte big-endian header: +an unknown `info` field producers leave zero, the member-table byte length, +the member-data byte length, a `0xED` mark and the member count. Producers +either count only the name/offset/size rows in that table length or include +the four-byte member count as well; both declarations are accepted. The table +region then carries one name/offset/size row per member; member bytes are +stored verbatim at absolute file offsets. + +Inspection retains the header facts: `info`, index and data lengths, member +count and directory status. Members retain declared source paths, absolute +carrier offsets and declared sizes, so `source_location` reports real file +positions. Member reads only examine their declared range in bounded 64 KiB +chunks. Carrier freshness is checked when a member is opened and after its +stream completes. Reader close cancels and joins outstanding streams before +releasing its descriptor. + +Encrypted PC-client packages start with the `V1MMWX` magic instead of the +plain header. They keep their format identity but report `encrypted` +directory status with unknown coverage: member identities and bytes remain +unknown and no decryption is attempted, so extraction refuses with a typed +error rather than producing ciphertext. Declared tables or data regions +that exceed the file, member extents outside the declared data region, +truncated rows, invalid UTF-8 names, and counts that cannot fit their table +are malformed-input errors. + +## Resource bounds + +These are resource policies, not format maxima. The encoded member table is +limited to 16 MiB and its buffer, parsed rows, retained entries and escaped +path projections share a conservative 64 MiB admission budget, reserved +before allocation; a declared member name is limited to 64 KiB. A reader +admits four active member streams. The shared archive plaintext-work ceiling +is the larger of 256 MiB and 500 times carrier size: every declared member +extent is charged, including aliases that overlap one stored range. A +separate member-I/O budget allows four times the larger of 1,024 and the sum +of the member count and declared 64 KiB plaintext chunks, bounding repeated +tiny-member reads independently of output bytes. + +## CLI + +```sh +rea inspect-artifact app.wxapkg --detail summary \ + --evidence_output evidence/package.json --json +rea extract-artifact app.wxapkg --json +``` + +Detection is byte-based on the `0xBE`/`0xED` marks or the `V1MMWX` magic, so +`.wxapkg`, `__APP__.wxapkg` and extensionless cache files all classify as +`wxapkg` containers. Compact delivery keeps complete Evidence in the +requested bundle. + +## Verification scope + +Synthetic fixtures cover plaintext headers, multiple members, empty members, +chunked reads, unicode and dotted names, malformed marks, truncated tables, +invalid UTF-8, table and directory-budget admission, member extents inside +the header, the index, or beyond the declared data region, a data region +declared past the file, a one-byte member whose table length includes the +member count, an overstated first row consuming its successor, V1MMWX +packages, cancellation inside member reads, paused iteration, close during +I/O, cleanup retry, stream capacity and carrier freshness. Compiled +full/compact MCP tests cover inventory and extraction. + +Real producer verification used plaintext `0xBE` packages from a local macOS +WeChat mini-program cache (`__APP__.wxapkg` main packages and +`__PLUGINCODE__.wxapkg` plugin packages across three distinct app +directories): every member extracted byte-identical to its declared carrier +range, including `__plugin__/`-namespaced members and the producer's +root-prefixed (`/name`) member convention. A cached `V1MMWX` PC package +reports the encrypted directory state with unknown member facts and refuses +extraction. + +`wxml`/`wxss` source recovery, compiled bundle semantics, subpackage +relationships and runtime behavior are outside this container reader's +verification scope. + +Artifact comparison pairs members by their package-relative paths and +compares their bytes. It does not compare `wxapkg` or `wxapkg-entry` +metadata: header fields repeat the package bytes, `index` is layout, and +`source_path` keeps a producer root prefix that the occurrence path strips. diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.ts index d69504c5d7..68845303df 100644 --- a/src/application/BinaryTargetResolver.ts +++ b/src/application/BinaryTargetResolver.ts @@ -85,6 +85,10 @@ import { hasDiskImageSignature } from "../domain/apple/diskImageFormat.js"; import { hasGodotPckHeader } from "../domain/godotPckFormat.js"; import { hasUImageMagic } from "../domain/uimageFormat.js"; import { hasUnityFsHeader } from "../domain/unityfsFormat.js"; +import { + hasWxapkgEncryptedMagic, + hasWxapkgHeader, +} from "../domain/wxapkgFormat.js"; import { detectHexRecordEncoding } from "../domain/firmware/hexRecords.js"; import { hasUf2Block } from "../domain/firmware/uf2.js"; import { @@ -777,6 +781,7 @@ const detectArtifactFormat = async ( if (hasZipSignature(magic)) return zipRootPackageFormat(lower); if (hasGodotPckHeader(magic)) return "godot-pck"; if (hasUnityFsHeader(magic)) return "unityfs"; + if (hasWxapkgEncryptedMagic(magic) || hasWxapkgHeader(magic)) return "wxapkg"; if (hasUImageMagic(magic)) return "uimage"; const hexRecords = detectHexRecordEncoding(magic); if (hexRecords !== undefined) return hexRecords; @@ -883,6 +888,7 @@ const isArchiveFormat = ( "ines", "godot-pck", "unityfs", + "wxapkg", "uimage", "dmg", "pkg", diff --git a/src/application/binary/TargetFormatVocabularies.test.ts b/src/application/binary/TargetFormatVocabularies.test.ts index 8aef2e212f..299093c9d1 100644 --- a/src/application/binary/TargetFormatVocabularies.test.ts +++ b/src/application/binary/TargetFormatVocabularies.test.ts @@ -120,6 +120,7 @@ const TARGETS: TargetFixtures = { ines: { ...archive, format: "ines" }, "godot-pck": { ...archive, format: "godot-pck" }, unityfs: { ...archive, format: "unityfs" }, + wxapkg: { ...archive, format: "wxapkg" }, uimage: { ...archive, format: "uimage" }, uf2: { ...archive, format: "uf2" }, dfuse: { ...archive, format: "dfuse" }, diff --git a/src/artifacts/ArtifactReader.ts b/src/artifacts/ArtifactReader.ts index 8709121b3d..ae1621a577 100644 --- a/src/artifacts/ArtifactReader.ts +++ b/src/artifacts/ArtifactReader.ts @@ -199,6 +199,7 @@ export interface ArtifactReader { | "ines" | "godot-pck" | "unityfs" + | "wxapkg" | "uimage" | "pyinstaller" | "dotnet-bundle" diff --git a/src/artifacts/WxapkgArtifactReader.boundaries.test.ts b/src/artifacts/WxapkgArtifactReader.boundaries.test.ts new file mode 100644 index 0000000000..5d8b69eb9c --- /dev/null +++ b/src/artifacts/WxapkgArtifactReader.boundaries.test.ts @@ -0,0 +1,310 @@ +import { once } from "node:events"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { buffer, text } from "node:stream/consumers"; +import { setImmediate } from "node:timers/promises"; +import { describe, expect, it, vi } from "vitest"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { createDeferred } from "../../tests/fixtures/binarySession.js"; +import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js"; +import { buildWxapkg } from "../../tests/fixtures/wxapkg.js"; +import { openRegularFile } from "../filesystem/RegularFile.js"; +import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { WxapkgArtifactReader } from "./WxapkgArtifactReader.js"; + +const fixture = async (bytes: Buffer): Promise => { + const path = join( + await createTestTempDirectory("rea-wxapkg-boundary-"), + "package.wxapkg", + ); + await writeFile(path, bytes); + return path; +}; +const entries = async (reader: WxapkgArtifactReader) => { + const result = []; + for await (const entry of reader.entries()) result.push(entry); + return result; +}; + +describe("wxapkg allocation and malformed-input boundaries", () => { + it("admits the encoded table budget before reading it", async () => { + const bytes = buildWxapkg([{ path: "a", data: "x" }]); + bytes.writeUInt32BE(17 * 1024 * 1024, 5); + const file = Buffer.alloc(18 + 17 * 1024 * 1024); + bytes.copy(file); + const path = await fixture(file); + const admitted = await openRegularFile(path, { symlinks: "reject" }); + const reader = new WxapkgArtifactReader(path, admitted); + const reads = vi.spyOn(admitted.handle, "read"); + try { + await expect(entries(reader)).rejects.toMatchObject({ + reason: "limit", + boundary: "capacity", + }); + expect(reads).toHaveBeenCalledTimes(1); // bounded header only + } finally { + await reader.close(); + reads.mockRestore(); + await admitted.handle.close(); + } + }); + + it("reserves row storage before constructing a large member array", async () => { + // 140k minimal rows fit the encoded table budget but exceed the retained + // directory representation budget mid-parse. + const fileCount = 140_000; + const tableBytes = fileCount * 13; + const file = Buffer.alloc(18 + tableBytes); + file[0] = 0xbe; + file.writeUInt32BE(0, 1); + file.writeUInt32BE(tableBytes, 5); + file.writeUInt32BE(0, 9); + file[13] = 0xed; + file.writeUInt32BE(fileCount, 14); + for (let index = 0; index < fileCount; index++) { + const row = 18 + index * 13; + file.writeUInt32BE(1, row); + file[row + 4] = 0x61; + // Zero-size members at the data start stay admissible so parsing + // reaches the representation budget. + file.writeUInt32BE(18 + tableBytes, row + 5); + file.writeUInt32BE(0, row + 9); + } + const reader = new WxapkgArtifactReader(await fixture(file)); + try { + await expect(entries(reader)).rejects.toMatchObject({ + reason: "limit", + boundary: "capacity", + }); + } finally { + await reader.close(); + } + }); + + it("charges overlapping readable members, not only their unique range", async () => { + const size = 1024 * 1024; + const members = Array.from({ length: 600 }, (_, index) => ({ + path: `alias-${String(index)}`, + data: index === 0 ? Buffer.alloc(size, 0x61) : Buffer.alloc(0), + size, + })); + const reader = new WxapkgArtifactReader( + await fixture(buildWxapkg(members, { aliasOffsets: true })), + ); + try { + await expect(entries(reader)).rejects.toMatchObject({ reason: "limit" }); + } finally { + await reader.close(); + } + }); + + it("types a short later row after an overstated first name consumes its bytes", async () => { + const bytes = buildWxapkg([ + { path: "a".repeat(40), data: "x" }, + { path: "b", data: "y" }, + ]); + // The first row's declared name consumes most of the second row. The + // minimum total table-size check alone cannot establish each later row. + bytes.writeUInt32BE(50, 18); + // Keep the first member admissible at the position the parser reads after + // the inflated name, so the failure must come from the short later row. + const dataStart = 18 + bytes.readUInt32BE(5); + bytes.writeUInt32BE(dataStart, 18 + 4 + 50); + bytes.writeUInt32BE(1, 18 + 4 + 50 + 4); + const reader = new WxapkgArtifactReader(await fixture(bytes)); + try { + await expect(entries(reader)).rejects.toMatchObject({ + reason: "format", + message: expect.stringContaining("truncated"), + }); + } finally { + await reader.close(); + } + }); +}); + +describe("wxapkg stream ownership", () => { + it("retains descriptor ownership after cleanup fails and permits retry", async () => { + const reader = new WxapkgArtifactReader( + await fixture(buildWxapkg([{ path: "x", data: "bytes" }])), + ); + const [entry] = await entries(reader); + const stream = await reader.open(entry!); + const failure = new Error("descriptor close denied"); + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockRejectedValueOnce(failure); + try { + await expect(reader.close()).rejects.toBe(failure); + expect(stream.closed).toBe(true); + await expect(reader.open(entry!)).rejects.toMatchObject({ + reason: "unavailable", + }); + await reader.close(); + expect(close).toHaveBeenCalledTimes(2); + } finally { + close.mockRestore(); + await reader.close(); + } + }); + + it("joins an outstanding positional read before close completes", async () => { + const path = await fixture(buildWxapkg([{ path: "x", data: "bytes" }])); + const admitted = await openRegularFile(path, { symlinks: "reject" }); + const reader = new WxapkgArtifactReader(path, admitted); + const reached = createDeferred(), + release = createDeferred(); + const nativeRead = admitted.handle.read.bind(admitted.handle); + const [entry] = await entries(reader); + const reads = vi.spyOn(admitted.handle, "read").mockImplementation((async ( + ...args: unknown[] + ) => { + reached.resolve(); + await release.promise; + return Reflect.apply(nativeRead, admitted.handle, args); + }) as typeof admitted.handle.read); + try { + const stream = await reader.open(entry!); + stream.read(0); + await awaitAcknowledgement( + reached.promise, + once(stream, "close"), + "positional member read", + ); + let closed = false; + const closing = reader.close().then(() => { + closed = true; + }); + await setImmediate(); + expect(closed).toBe(false); + release.resolve(); + await closing; + expect(stream.closed).toBe(true); + } finally { + release.resolve(); + await reader.close(); + reads.mockRestore(); + await admitted.handle.close(); + } + }); + + it("preserves cancellation inside a member read", async () => { + const data = Buffer.alloc(256 * 1024, 0x61); + const path = await fixture(buildWxapkg([{ path: "x", data }])); + const admitted = await openRegularFile(path, { symlinks: "reject" }); + const reader = new WxapkgArtifactReader(path, admitted); + const controller = new AbortController(); + const reason = new Error("cancel during member read"); + const nativeRead = admitted.handle.read.bind(admitted.handle); + const [entry] = await entries(reader); + const reads = vi.spyOn(admitted.handle, "read").mockImplementation((async ( + ...args: unknown[] + ) => { + const result = await Reflect.apply(nativeRead, admitted.handle, args); + globalThis.setImmediate(() => { + controller.abort(reason); + }); + return result; + }) as typeof admitted.handle.read); + try { + await expect( + buffer(await reader.open(entry!, controller.signal)), + ).rejects.toMatchObject({ reason: "cancelled", cause: reason }); + } finally { + await reader.close(); + reads.mockRestore(); + await admitted.handle.close(); + } + }); +}); + +describe("wxapkg stream capacity and freshness", () => { + it("bounds active streams and releases capacity after abandonment", async () => { + const reader = new WxapkgArtifactReader( + await fixture(buildWxapkg([{ path: "x", data: "bytes" }])), + ); + try { + const [entry] = await entries(reader); + const streams = await Promise.all( + Array.from({ length: 4 }, () => reader.open(entry!)), + ); + await expect(reader.open(entry!)).rejects.toMatchObject({ + reason: "limit", + boundary: "capacity", + }); + const closed = new Promise((resolve) => { + streams[0]!.once("close", resolve); + }); + streams[0]!.destroy(); + await closed; + expect(await text(await reader.open(entry!))).toBe("bytes"); + } finally { + await reader.close(); + } + }); + + it("bounds repeated member I/O operations, including empty members", async () => { + const reader = new WxapkgArtifactReader( + await fixture( + buildWxapkg([ + { path: "x", data: "b" }, + { path: "empty.wxss", data: "" }, + ]), + ), + ); + try { + const [entry, emptyEntry] = await entries(reader); + await expect( + (async () => { + for (let index = 0; index < 5000; index++) + await buffer(await reader.open(entry!)); + })(), + ).rejects.toMatchObject({ reason: "limit", boundary: "capacity" }); + await expect( + (async () => { + for (let index = 0; index < 5000; index++) + await buffer(await reader.open(emptyEntry!)); + })(), + ).rejects.toMatchObject({ reason: "limit", boundary: "capacity" }); + } finally { + await reader.close(); + } + }); + + it("validates freshness before serving member bytes", async () => { + const bytes = buildWxapkg([{ path: "x", data: "before" }]); + const path = await fixture(bytes); + const reader = new WxapkgArtifactReader(path); + try { + const [entry] = await entries(reader); + expect(await text(await reader.open(entry!))).toBe("before"); + await writeFile(path, Buffer.concat([bytes, Buffer.from([0])])); + await expect(reader.open(entry!)).rejects.toMatchObject({ + reason: "integrity", + }); + } finally { + await reader.close(); + } + }); + + it("does not turn a paused inventory into complete coverage when closed", async () => { + const reader = new WxapkgArtifactReader( + await fixture( + buildWxapkg([ + { path: "x", data: "bytes" }, + { path: "y", data: "bytes" }, + ]), + ), + ); + const iterator = reader.entries()[Symbol.asyncIterator](); + try { + expect((await iterator.next()).done).toBe(false); + await reader.close(); + await expect(iterator.next()).rejects.toMatchObject({ + reason: "unavailable", + }); + } finally { + await reader.close(); + } + }); +}); diff --git a/src/artifacts/WxapkgArtifactReader.test.ts b/src/artifacts/WxapkgArtifactReader.test.ts new file mode 100644 index 0000000000..51205da61a --- /dev/null +++ b/src/artifacts/WxapkgArtifactReader.test.ts @@ -0,0 +1,468 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { text } from "node:stream/consumers"; + +import { describe, expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { + buildWxapkg, + wxapkgEncryptedBytes, + type WxapkgMember, +} from "../../tests/fixtures/wxapkg.js"; + +import type { + ArtifactEntry, + ArtifactTraversalOptions, +} from "./ArtifactReader.js"; +import { hasWxapkgHeader } from "../domain/wxapkgFormat.js"; +import { WxapkgArtifactReader } from "./WxapkgArtifactReader.js"; + +const writePackage = async (bytes: Buffer): Promise => { + const path = join( + await createTestTempDirectory("rea-wxapkg-"), + "package.wxapkg", + ); + await writeFile(path, bytes); + return path; +}; + +const list = async ( + reader: WxapkgArtifactReader, + options?: ArtifactTraversalOptions, +): Promise => { + const entries: ArtifactEntry[] = []; + for await (const entry of reader.entries(undefined, options)) + entries.push(entry); + return entries; +}; + +const withReader = async ( + bytes: Buffer, + use: (reader: WxapkgArtifactReader) => Promise, +): Promise => { + const reader = new WxapkgArtifactReader(await writePackage(bytes)); + try { + return await use(reader); + } finally { + await reader.close(); + } +}; + +const members: readonly WxapkgMember[] = [ + { path: "app-config.json", data: "{}" }, + { path: "pages/index/index.js", data: "module.exports={};" }, + { path: "empty.wxss", data: "" }, +]; + +describe("WxapkgArtifactReader", () => { + it("lists members and reads their declared bytes", async () => { + await withReader(buildWxapkg(members), async (reader) => { + const entries = await list(reader); + expect(entries.map(({ path }) => path)).toEqual([ + "app-config.json", + "pages/index/index.js", + "empty.wxss", + ]); + expect(await text(await reader.open(entries[0]!))).toBe("{}"); + expect(await text(await reader.open(entries[1]!))).toBe( + "module.exports={};", + ); + expect(await text(await reader.open(entries[2]!))).toBe(""); + }); + }); + + it("preserves member provenance on entries and metadata", async () => { + await withReader(buildWxapkg(members), async (reader) => { + const entries = await list(reader); + const entry = entries[1]!; + expect(entry).toMatchObject({ + path: "pages/index/index.js", + sourcePath: "pages/index/index.js", + kind: "file", + declaredSize: 18, + compressedSize: null, + encrypted: false, + declaredIntegrity: null, + unpacked: false, + adapterKey: "wxapkg-entry:1", + formatMetadata: { + kind: "wxapkg-entry", + index: 1, + source_path: "pages/index/index.js", + }, + }); + // Member offsets are absolute carrier positions after the table. + expect(entry.byteOffset).toBeGreaterThanOrEqual(18); + expect(reader.formatMetadata()).toMatchObject({ + kind: "wxapkg", + file_count: 3, + directory_status: "available", + }); + }); + }); + + it("reads a member larger than the chunk size", async () => { + const data = Buffer.alloc(200 * 1024, 0x61); + await withReader( + buildWxapkg([{ path: "big.bin", data }]), + async (reader) => { + const [entry] = await list(reader); + const chunks: Buffer[] = []; + for await (const chunk of await reader.open(entry!)) + chunks.push(chunk as Buffer); + expect(Buffer.concat(chunks).equals(data)).toBe(true); + }, + ); + }); + + it("accepts consecutive dots and unicode names", async () => { + await withReader( + buildWxapkg([ + { path: "data..bin", data: "a" }, + { path: "页面/配置.json", data: "b" }, + ]), + async (reader) => { + const entries = await list(reader); + expect(entries[0]?.path).toBe("data..bin"); + expect(entries[1]?.path).toBe("页面/配置.json"); + expect(await text(await reader.open(entries[1]!))).toBe("b"); + }, + ); + }); + + it("translates producer root prefixes while keeping declared names", async () => { + await withReader( + buildWxapkg([ + { path: "/app-config.json", data: "{}" }, + { path: "/pages/index/index.js", data: "x=1;" }, + ]), + async (reader) => { + const entries = await list(reader); + expect(entries.map((entry) => entry.path)).toEqual([ + "app-config.json", + "pages/index/index.js", + ]); + expect(entries[1]).toMatchObject({ + sourcePath: "/pages/index/index.js", + formatMetadata: { source_path: "/pages/index/index.js" }, + }); + expect(await text(await reader.open(entries[0]!))).toBe("{}"); + }, + ); + }); + + it("exposes no members for a V1MMWX-encrypted package", async () => { + await withReader(wxapkgEncryptedBytes(), async (reader) => { + const coverage: { + status?: string; + limitations?: readonly string[]; + } = {}; + const entries = await list(reader, { + onCoverage: (observation) => { + coverage.status = observation.status; + coverage.limitations = observation.limitations; + }, + }); + expect(entries).toEqual([]); + expect(coverage.status).toBe("unknown"); + expect(coverage.limitations?.join(" ")).toContain("V1MMWX"); + expect(reader.formatMetadata()).toMatchObject({ + kind: "wxapkg", + directory_status: "encrypted", + file_count: null, + info: null, + }); + }); + }); +}); + +describe("WxapkgArtifactReader malformed inputs", () => { + it.each([ + ["wrong first mark", { firstMark: 0x00 }], + ["wrong last mark", { lastMark: 0x00 }], + ["file count above the row count", { declaredFileCount: 4 }], + ["index length below the rows", { declaredIndexLength: 10 }], + ["declared name exceeds the row", { nameLength: 1024 }], + ["zero-length declared name", { nameLength: 0 }], + ])("rejects %s as a typed failure", async (_name, options) => { + await withReader(buildWxapkg(members, options), async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ + reason: expect.stringMatching(/^(format|limit)$/), + }); + }); + }); + + it("rejects a member offset into the header region", async () => { + await withReader( + buildWxapkg([{ path: "evil.bin", data: "x", offset: 4 }]), + async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ + reason: "format", + }); + }, + ); + }); + + it("rejects a member extent beyond the file", async () => { + await withReader( + buildWxapkg([{ path: "evil.bin", data: "x", size: 1 << 20 }]), + async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ + reason: "format", + }); + }, + ); + }); + + it("rejects a member offset into the index region", async () => { + // Offset 18 is inside the declared table, not the data region. + await withReader( + buildWxapkg([{ path: "evil.bin", data: "x", offset: 18, size: 1 }]), + async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ reason: "format" }); + }, + ); + }); +}); + +describe("WxapkgArtifactReader data region", () => { + it("rejects a member range that covers unused index bytes", async () => { + const bytes = buildWxapkg([{ path: "a", data: "Z" }], { + tablePadding: 8, + }); + // The row ends at byte 31; the next eight bytes are still the index. + bytes.writeUInt32BE(31, 23); + bytes.writeUInt32BE(1, 27); + await withReader(bytes, async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ reason: "format" }); + }); + }); + + it("reads member bytes that follow unused index bytes", async () => { + await withReader( + buildWxapkg([{ path: "a", data: "Z" }], { tablePadding: 8 }), + async (reader) => { + const [entry] = await list(reader); + expect(entry).toMatchObject({ byteOffset: 39, declaredSize: 1 }); + expect(await text(await reader.open(entry!))).toBe("Z"); + }, + ); + }); + + it("rejects a member extent beyond the declared data region", async () => { + await withReader( + buildWxapkg([{ path: "x", data: "0123456789" }], { + declaredDataLength: 5, + }), + async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ reason: "format" }); + }, + ); + }); + + it("rejects a data region declared past the file", async () => { + await withReader( + buildWxapkg([{ path: "x", data: "ok" }], { + declaredDataLength: 1 << 24, + }), + async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ reason: "format" }); + }, + ); + }); +}); + +describe("WxapkgArtifactReader index length", () => { + it("reads a one-byte member when indexLength counts only the rows", async () => { + const bytes = buildWxapkg([{ path: "a", data: "Z" }]); + expect(bytes).toHaveLength(32); + expect(bytes.readUInt32BE(5)).toBe(13); + await withReader(bytes, async (reader) => { + const [entry] = await list(reader); + expect(entry).toMatchObject({ byteOffset: 31, declaredSize: 1 }); + expect(reader.formatMetadata()).toMatchObject({ + index_length: 13, + data_length: 1, + file_count: 1, + }); + expect(await text(await reader.open(entry!))).toBe("Z"); + }); + }); + + it("reads a one-byte member when indexLength includes the member count", async () => { + const bytes = buildWxapkg([{ path: "a", data: "Z" }], { + indexLengthIncludesFileCount: true, + }); + expect(bytes).toHaveLength(32); + expect(bytes.readUInt32BE(5)).toBe(17); + await withReader(bytes, async (reader) => { + const [entry] = await list(reader); + expect(entry).toMatchObject({ + path: "a", + byteOffset: 31, + declaredSize: 1, + }); + expect(reader.formatMetadata()).toMatchObject({ + index_length: 17, + data_length: 1, + file_count: 1, + }); + expect(await text(await reader.open(entry!))).toBe("Z"); + }); + }); + + it("reads a count-inclusive member when exactly four bytes trail the package", async () => { + const bytes = Buffer.concat([ + buildWxapkg([{ path: "a", data: "Z" }], { + indexLengthIncludesFileCount: true, + }), + Buffer.alloc(4, 0xee), + ]); + expect(bytes).toHaveLength(36); + await withReader(bytes, async (reader) => { + const [entry] = await list(reader); + expect(entry).toMatchObject({ byteOffset: 31, declaredSize: 1 }); + expect(await text(await reader.open(entry!))).toBe("Z"); + }); + }); + + it("rejects a count-inclusive member that covers trailing bytes", async () => { + const bytes = Buffer.concat([ + buildWxapkg([{ path: "a", data: "Z" }], { + indexLengthIncludesFileCount: true, + }), + Buffer.alloc(4, 0xee), + ]); + bytes.writeUInt32BE(5, 27); + await withReader(bytes, async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ reason: "format" }); + }); + }); + + it("keeps a count-inclusive member inside the data region when bytes trail the package", async () => { + const bytes = Buffer.concat([ + buildWxapkg([{ path: "a", data: "Z" }], { + indexLengthIncludesFileCount: true, + }), + Buffer.alloc(8, 0xee), + ]); + await withReader(bytes, async (reader) => { + const [entry] = await list(reader); + expect(entry).toMatchObject({ byteOffset: 31, declaredSize: 1 }); + expect(await text(await reader.open(entry!))).toBe("Z"); + }); + }); + + it("rejects an index byte of a count-inclusive package", async () => { + await withReader( + buildWxapkg([{ path: "a", data: "Z", offset: 18, size: 1 }], { + indexLengthIncludesFileCount: true, + }), + async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ reason: "format" }); + }, + ); + }); +}); + +describe("WxapkgArtifactReader damaged packages", () => { + it("keeps a damaged package's format identity as a typed failure", async () => { + // Marks hold but the declared count cannot fit the declared table. + const bytes = buildWxapkg(members, { + declaredFileCount: 2, + declaredIndexLength: 12, + }); + expect(hasWxapkgHeader(bytes)).toBe(true); + await withReader(bytes, async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ reason: "format" }); + }); + }); + + it("rejects a truncated package before the declared table ends", async () => { + const bytes = buildWxapkg(members); + await withReader(bytes.subarray(0, 30), async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ reason: "format" }); + }); + }); + + it("rejects a non-UTF-8 member name as a typed format failure", async () => { + const bytes = buildWxapkg(members); + bytes[26] = 0xff; // first byte of the first member name + await withReader(bytes, async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ reason: "format" }); + }); + }); + + it("rejects a forged foreign entry with an integrity failure", async () => { + await withReader(buildWxapkg(members), async (reader) => { + const [entry] = await list(reader); + await expect( + reader.open({ ...entry!, adapterKey: "wxapkg-entry:9" }), + ).rejects.toMatchObject({ reason: "integrity" }); + }); + }); +}); + +describe("WxapkgArtifactReader lifecycle", () => { + it("aborts member reads when the caller cancels", async () => { + const data = Buffer.alloc(300 * 1024, 0x62); + await withReader( + buildWxapkg([{ path: "big.bin", data }]), + async (reader) => { + const [entry] = await list(reader); + const controller = new AbortController(); + const stream = await reader.open(entry!, controller.signal); + let delivered = 0; + await expect( + (async () => { + for await (const chunk of stream) { + delivered += (chunk as Buffer).length; + controller.abort(); + } + })(), + ).rejects.toThrow(); + expect(delivered).toBeLessThan(data.length); + }, + ); + }); + + it("survives a destroyed member stream and serves the next entry", async () => { + await withReader(buildWxapkg(members), async (reader) => { + const entries = await list(reader); + const abandoned = await reader.open(entries[0]!); + abandoned.destroy(); + await expect(text(await reader.open(entries[1]!))).resolves.toBe( + "module.exports={};", + ); + }); + }); + + it("fails streams and later reads after close", async () => { + const reader = new WxapkgArtifactReader( + await writePackage(buildWxapkg(members)), + ); + const entries: ArtifactEntry[] = []; + for await (const entry of reader.entries()) entries.push(entry); + const stream = await reader.open(entries[0]!); + const consumed = text(stream).catch((error: unknown) => error); + await reader.close(); + expect(await consumed).toBeInstanceOf(Error); + await expect(reader.open(entries[1]!)).rejects.toMatchObject({ + reason: "unavailable", + }); + }); + + it("bounds concurrent member streams", async () => { + await withReader(buildWxapkg(members), async (reader) => { + const [entry] = await list(reader); + const streams = await Promise.all( + [0, 1, 2, 3].map(() => reader.open(entry!)), + ); + await expect(reader.open(entry!)).rejects.toMatchObject({ + reason: "limit", + }); + for (const stream of streams) stream.destroy(); + }); + }); +}); diff --git a/src/artifacts/WxapkgArtifactReader.ts b/src/artifacts/WxapkgArtifactReader.ts new file mode 100644 index 0000000000..fc7c57a3b1 --- /dev/null +++ b/src/artifacts/WxapkgArtifactReader.ts @@ -0,0 +1,311 @@ +import { artifactExpansionBudgetBytes } from "../domain/artifactExpansionBudget.js"; +import type { FileHandle } from "node:fs/promises"; +import { setImmediate } from "node:timers/promises"; +import { Readable } from "node:stream"; +import { + ArtifactReaderFailure, + artifactFixedLimitFailure, + copyArtifactEntry, + sameArtifactEntry, + type ArtifactEntry, + type ArtifactReader, + type ArtifactTraversalOptions, +} from "./ArtifactReader.js"; +import { ArtifactResourceScope } from "./ArtifactResourceScope.js"; +import { assertArtifactReadActive } from "./ArtifactFileOperation.js"; +import { readArtifactFileBytes } from "./ArtifactFileRange.js"; +import type { + ArtifactFormatMetadata, + WxapkgMetadata, +} from "../domain/artifactFormatMetadata.js"; +import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js"; +import { ArtifactFileSource } from "./ArtifactFileSource.js"; +import { + readWxapkgFileTable, + readWxapkgHeader, + wxapkgFormatFailure, + type WxapkgMember, +} from "./WxapkgStructures.js"; + +const CHUNK_BYTES = 64 * 1024; +/** Bound active member streams; each owns a file-range generator. */ +const MAX_MEMBER_STREAMS = 4; + +type WxapkgEntry = ArtifactEntry & { + readonly declaredSize: number; + readonly formatMetadata: Extract< + ArtifactFormatMetadata, + { kind: "wxapkg-entry" } + >; +}; + +interface WxapkgEntryState { + readonly entry: WxapkgEntry; + readonly member: WxapkgMember; +} + +/** One descriptor, one admitted member table and bounded raw-range member reads. */ +export class WxapkgArtifactReader implements ArtifactReader { + readonly format = "wxapkg" as const; + readonly #scope = new ArtifactResourceScope(); + readonly #entries = new Map(); + readonly #streams = new Map(); + readonly #source: ArtifactFileSource; + #prepared: Promise | undefined; + #header: WxapkgMetadata | undefined; + #remainingReadOperations = 0; + #admittedReadOperations = 0; + #closing = false; + + constructor(path: string, admitted?: StableRegularFileDescriptor) { + this.#source = new ArtifactFileSource(path, "wxapkg package", admitted); + } + + formatMetadata(): WxapkgMetadata | undefined { + return this.#header === undefined ? undefined : { ...this.#header }; + } + + async prepareContainer( + expectedSha256: string, + signal?: AbortSignal, + ): Promise { + await this.#scope.run(async () => { + this.#assertOpen(signal); + await this.#source.bindSha256(expectedSha256, signal); + }); + } + + async *entries( + signal?: AbortSignal, + options?: ArtifactTraversalOptions, + ): AsyncIterable { + await this.#scope.run(async () => { + this.#assertOpen(signal); + this.#prepared ??= this.#prepare(signal); + await this.#prepared; + await this.#source.read(signal); + }); + this.#assertOpen(signal); + if (this.#header?.directory_status === "encrypted") { + options?.onCoverage?.({ + status: "unknown", + limitations: [ + "wxapkg package is V1MMWX-encrypted; member identities and contents remain unknown.", + ], + }); + return; + } + let index = 0; + for (const { entry } of this.#entries.values()) { + if (index++ % 256 === 0) await setImmediate(); + this.#assertOpen(signal); + yield copyArtifactEntry(entry); + } + // A close while paused at yield must not turn a cleared map into completion. + this.#assertOpen(signal); + } + + async open(entry: ArtifactEntry, signal?: AbortSignal): Promise { + return this.#scope.run(async () => { + this.#assertOpen(signal); + const state = this.#entries.get(entry.adapterKey); + if (state === undefined || !sameArtifactEntry(entry, state.entry)) + throw new ArtifactReaderFailure( + "integrity", + "wxapkg entry does not match its owning inventory", + ); + // Every logical open spends the work budget, including empty members + // whose streams perform only the carrier checks below. + this.#reserveMemberRead(); + // Validate carrier freshness before handing out member bytes. + const { handle } = await this.#source.read(signal); + this.#assertOpen(signal); + if (this.#streams.size >= MAX_MEMBER_STREAMS) + throw new ArtifactReaderFailure( + "limit", + "wxapkg reader already owns four active member streams", + { boundary: "capacity" }, + ); + const controller = new AbortController(); + const output = Readable.from( + this.#memberChunks(handle, state, controller.signal), + { + objectMode: false, + highWaterMark: CHUNK_BYTES, + }, + ); + const onAbort = (): void => { + controller.abort(signal?.reason); + output.destroy( + new ArtifactReaderFailure( + "cancelled", + "wxapkg member read cancelled", + { cause: signal?.reason }, + ), + ); + }; + output.on("error", () => undefined); + this.#streams.set(output, controller); + const completion = new Promise((resolve) => { + output.once("close", () => { + signal?.removeEventListener("abort", onAbort); + this.#streams.delete(output); + resolve(); + }); + }); + // Readable.from waits for its async iterator to retire before close. Keep + // that whole lifetime owned, including a suspended positional read. + void this.#scope + .run(() => completion) + .catch((cause: unknown) => { + controller.abort(cause); + output.destroy( + cause instanceof Error + ? cause + : new Error("wxapkg stream ownership failed", { cause }), + ); + }); + signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted === true) onAbort(); + return output; + }); + } + + async close(): Promise { + this.#closing = true; + for (const [stream, controller] of this.#streams) { + const cause = new ArtifactReaderFailure( + "unavailable", + "wxapkg reader closed", + ); + controller.abort(cause); + stream.destroy(cause); + } + await this.#scope.close(); + await this.#source.close(); + this.#entries.clear(); + } + + provenance(): readonly [] { + return []; + } + + #assertOpen(signal?: AbortSignal): void { + assertArtifactReadActive(signal); + if (this.#closing) + throw new ArtifactReaderFailure("unavailable", "wxapkg reader is closed"); + } + + async #prepare(signal?: AbortSignal): Promise { + const { handle, initial } = await this.#source.read(signal); + const header = await readWxapkgHeader(handle, initial.size, signal); + if (header === undefined) + throw wxapkgFormatFailure("no complete wxapkg header"); + this.#header = { + kind: "wxapkg", + info: header.layout.status === "available" ? header.info : null, + index_length: + header.layout.status === "available" ? header.indexLength : null, + data_length: + header.layout.status === "available" ? header.dataLength : null, + file_count: + header.layout.status === "available" ? header.fileCount : null, + directory_status: header.layout.status, + }; + if (header.layout.status !== "available") return; + const { members } = await readWxapkgFileTable( + handle, + header, + initial.size, + signal, + ); + const budget = artifactExpansionBudgetBytes(initial.size); + let declaredBytes = 0n; + for (let index = 0; index < members.length; index++) { + if (index % 256 === 0) await setImmediate(); + this.#assertOpen(signal); + const member = members[index]!; + // Overlapping member ranges are valid; each declared path causes a read. + declaredBytes += BigInt(member.size); + if (declaredBytes > BigInt(budget)) + throw new ArtifactReaderFailure( + "limit", + `wxapkg members declare ${String(declaredBytes)} bytes, exceeding the ${String(budget)}-byte plaintext-work budget`, + { boundary: "memory" }, + ); + const adapterKey = `wxapkg-entry:${String(index)}`; + this.#entries.set(adapterKey, { + entry: { + path: member.path, + sourcePath: member.declaredPath, + kind: "file", + declaredSize: member.size, + compressedSize: null, + executable: false, + encrypted: false, + // Member offsets are absolute carrier positions. + byteOffset: member.offset, + declaredIntegrity: null, + unpacked: false, + limitations: [], + adapterKey, + formatMetadata: { + kind: "wxapkg-entry", + index, + source_path: member.declaredPath, + declared_offset: member.offset, + }, + }, + member, + }); + } + // Header, table and per-chunk member reads; a small-reader floor applies. + this.#admittedReadOperations = + 4 * + Math.max( + 1024, + members.length + Math.ceil(Number(declaredBytes) / CHUNK_BYTES), + ); + this.#remainingReadOperations = this.#admittedReadOperations; + await this.#source.read(signal); + } + + async *#memberChunks( + handle: FileHandle, + state: WxapkgEntryState, + signal: AbortSignal, + ): AsyncGenerator { + const { offset, size } = state.member; + let delivered = 0, + chunks = 0; + for (let position = offset; position < offset + size;) { + if (chunks++ % 64 === 0) await setImmediate(); + this.#assertOpen(signal); + this.#reserveMemberRead(); + const length = Math.min(CHUNK_BYTES, offset + size - position); + const chunk = await readArtifactFileBytes( + handle, + position, + length, + offset + size, + signal, + ); + position += chunk.length; + delivered += chunk.length; + yield chunk; + } + if (delivered !== size) + throw wxapkgFormatFailure("member ended before its declared size"); + await this.#source.read(signal); + } + + #reserveMemberRead(): void { + if (this.#remainingReadOperations <= 0) + throw artifactFixedLimitFailure( + "wxapkg directory-and-chunk I/O work", + this.#admittedReadOperations, + "operations", + ); + this.#remainingReadOperations--; + } +} diff --git a/src/artifacts/WxapkgStructures.ts b/src/artifacts/WxapkgStructures.ts new file mode 100644 index 0000000000..3437ead7a7 --- /dev/null +++ b/src/artifacts/WxapkgStructures.ts @@ -0,0 +1,214 @@ +import type { FileHandle } from "node:fs/promises"; +import { + ArtifactReaderFailure, + artifactFixedLimitFailure, +} from "./ArtifactReader.js"; +import { readArtifactFileBytes } from "./ArtifactFileRange.js"; +import { assertArtifactReadActive } from "./ArtifactFileOperation.js"; +import { setImmediate } from "node:timers/promises"; +import { + hasWxapkgEncryptedMagic, + readWxapkgHeaderFacts, + resolveWxapkgIndexLayouts, + selectWxapkgIndexLayout, + WXAPKG_MIN_ROW_BYTES, + type WxapkgIndexLayout, + type WxapkgIndexLayoutFailure, +} from "../domain/wxapkgFormat.js"; + +/** The member table is a flat name/offset/size row list. */ +const MAX_TABLE_BYTES = 16 * 1024 * 1024; +/** Encoded retained directory representation; paths are charged separately. */ +const DIRECTORY_BUDGET_BYTES = 64 * 1024 * 1024; +/** Member names are relative package paths; larger claims are malformed. */ +const MAX_MEMBER_NAME_BYTES = 64 * 1024; +/** Detecting prefix covers the fixed header, table head and encrypted magic. */ +const HEADER_PREFIX_BYTES = 64; + +const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + +export const wxapkgFormatFailure = (detail: string): ArtifactReaderFailure => + new ArtifactReaderFailure("format", `wxapkg is malformed: ${detail}`); + +/** Reserve buffers, JS rows and escaped name projections before allocating. */ +class WxapkgDirectoryBudget { + #remaining = DIRECTORY_BUDGET_BYTES; + reserve(bytes: number): void { + if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > this.#remaining) + throw artifactFixedLimitFailure( + "wxapkg directory representation", + DIRECTORY_BUDGET_BYTES, + "bytes", + ); + this.#remaining -= bytes; + } +} + +export interface WxapkgHeader { + readonly info: number; + readonly indexLength: number; + readonly dataLength: number; + readonly fileCount: number; + readonly layout: + | { readonly status: "encrypted" } + | { + readonly status: "available"; + readonly indexLayouts: readonly WxapkgIndexLayout[]; + }; +} + +export interface WxapkgMember { + readonly offset: number; + readonly size: number; + /** Package-relative path; declared root prefixes are translated here. */ + readonly path: string; + /** Declared member name as stored in the table. */ + readonly declaredPath: string; +} + +export interface WxapkgFileTable { + readonly members: readonly WxapkgMember[]; +} + +/** Decode header fields once; encrypted variants and absent signatures are distinct. */ +export const readWxapkgHeader = async ( + handle: FileHandle, + size: number, + signal?: AbortSignal, +): Promise => { + const prefix = await readArtifactFileBytes( + handle, + 0, + Math.min(size, HEADER_PREFIX_BYTES), + size, + signal, + ); + if (hasWxapkgEncryptedMagic(prefix)) + return { + info: 0, + indexLength: 0, + dataLength: 0, + fileCount: 0, + layout: { status: "encrypted" }, + }; + const facts = readWxapkgHeaderFacts(prefix); + if (facts === undefined) return undefined; + const indexes = resolveWxapkgIndexLayouts(facts, size); + if (!indexes.ok) throw wxapkgIndexLayoutFailure(indexes.error); + return { + ...facts, + layout: { status: "available", indexLayouts: indexes.value }, + }; +}; + +const wxapkgIndexLayoutFailure = ( + failure: WxapkgIndexLayoutFailure, +): ArtifactReaderFailure => + wxapkgFormatFailure( + failure._tag === "DataRegionExceedsFile" + ? "declared data region exceeds the file" + : "declared member table exceeds the file", + ); + +export const readWxapkgFileTable = async ( + handle: FileHandle, + header: WxapkgHeader, + size: number, + signal?: AbortSignal, +): Promise => { + const layout = header.layout; + if (layout.status !== "available") + throw new ArtifactReaderFailure( + "unavailable", + "wxapkg directory is encrypted", + ); + const indexLayouts = layout.indexLayouts; + let readBytes = 0; + let tableStart: number | undefined; + for (const indexLayout of indexLayouts) { + if (tableStart === undefined) tableStart = indexLayout.tableStart; + if (indexLayout.tableBytes > readBytes) readBytes = indexLayout.tableBytes; + } + if (tableStart === undefined) + throw wxapkgFormatFailure("declared member table exceeds the file"); + if (readBytes > MAX_TABLE_BYTES) + throw artifactFixedLimitFailure( + "wxapkg member table", + MAX_TABLE_BYTES, + "bytes", + ); + const budget = new WxapkgDirectoryBudget(); + budget.reserve(readBytes); + const table = await readArtifactFileBytes( + handle, + tableStart, + readBytes, + size, + signal, + ); + // Loose lower bound already checked in facts; precise rows are checked here. + if (table.length < header.fileCount * WXAPKG_MIN_ROW_BYTES) + throw wxapkgFormatFailure("member table is truncated"); + const members: WxapkgMember[] = []; + let cursor = 0; + for (let index = 0; index < header.fileCount; index++) { + if (index % 256 === 0) await setImmediate(); + assertArtifactReadActive(signal); + if (cursor + 4 > table.length) + throw wxapkgFormatFailure("member row is truncated"); + const nameLength = table.readUInt32BE(cursor); + cursor += 4; + if ( + nameLength === 0 || + nameLength > MAX_MEMBER_NAME_BYTES || + cursor + nameLength + 8 > table.length + ) + throw wxapkgFormatFailure("member row exceeds the table"); + // Row storage plus the reader's owned entries/map; charge before decode. + budget.reserve(512 + nameLength * 24); + let declaredPath: string; + try { + declaredPath = UTF8.decode(table.subarray(cursor, cursor + nameLength)); + } catch (cause: unknown) { + throw new ArtifactReaderFailure( + "format", + "wxapkg member path is not valid UTF-8", + { + cause, + }, + ); + } + cursor += nameLength; + const offset = table.readUInt32BE(cursor); + const sizeMember = table.readUInt32BE(cursor + 4); + cursor += 8; + // Producer names are root-prefixed; translate to package-relative once. + const path = declaredPath.startsWith("/") + ? declaredPath.slice(1) + : declaredPath; + if (path.length === 0) + throw wxapkgFormatFailure("member name has no package-relative path"); + members.push({ offset, size: sizeMember, path, declaredPath }); + } + const candidates = indexLayouts.filter((index) => index.tableBytes >= cursor); + if (candidates.length === 0) + throw wxapkgFormatFailure("member table is truncated"); + // Rows and member ranges choose the convention. EOF is not a signal: four + // trailing bytes can make the other convention end on the carrier. + const index = selectWxapkgIndexLayout(candidates, cursor, members); + if (index === undefined) { + const outside = members.find((member) => { + const extentEnd = member.offset + member.size; + return candidates.every( + (candidate) => + member.offset < candidate.dataStart || + extentEnd > candidate.dataEnd || + extentEnd < member.offset, + ); + }); + throw wxapkgFormatFailure( + `member exceeds the data region: ${outside?.path ?? ""}`, + ); + } + return { members }; +}; diff --git a/src/artifacts/artifactFormats.test.ts b/src/artifacts/artifactFormats.test.ts index da351dc0c8..acbd1fe124 100644 --- a/src/artifacts/artifactFormats.test.ts +++ b/src/artifacts/artifactFormats.test.ts @@ -30,6 +30,7 @@ type FormatsWithReaders = | "ines" | "godot-pck" | "unityfs" + | "wxapkg" | "uimage" | "mach-o-universal" | "directory" diff --git a/src/artifacts/artifactFormats.ts b/src/artifacts/artifactFormats.ts index aeb7dc8bff..92c343e202 100644 --- a/src/artifacts/artifactFormats.ts +++ b/src/artifacts/artifactFormats.ts @@ -40,6 +40,7 @@ import { PyInstallerArtifactReader } from "./PyInstallerArtifactReader.js"; import { Uf2ArtifactReader } from "./Uf2ArtifactReader.js"; import { UImageArtifactReader } from "./UImageArtifactReader.js"; import { UnityFsArtifactReader } from "./UnityFsArtifactReader.js"; +import { WxapkgArtifactReader } from "./WxapkgArtifactReader.js"; import { ZipArtifactReader } from "./ZipArtifactReader.js"; /** How inventory describes a format when no reader was constructed. */ @@ -286,6 +287,16 @@ const NON_ZIP_FORMATS = { createReader: (path, options) => new UnityFsArtifactReader(path, sourceOf(options)), }), + wxapkg: container("wxapkg", { + roleContainer: true, + retainClassifiedSource: true, + extractable: true, + absentInventory: "unavailable", + readerLimitation: + "wxapkg inventory establishes packaged member names and stored bytes; wxml/wxss sources, compiled bundles and runtime behavior are not reconstructed. V1MMWX-encrypted packages are unsupported.", + createReader: (path, options) => + new WxapkgArtifactReader(path, sourceOf(options)), + }), uimage: container("uimage", { roleContainer: true, retainClassifiedSource: true, diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index d51a52c083..16e9745b75 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -215,6 +215,15 @@ export const materializeArtifactInventoryInScope = async ( input.selectedPath ?? input.inputPath, `UnityFS directory is ${containerMetadata.directory_status}; its complete contents cannot be extracted`, ); + if ( + containerMetadata?.kind === "wxapkg" && + containerMetadata.directory_status !== "available" + ) + throw new AnalysisUnsupportedTargetError( + "extract_artifact", + input.selectedPath ?? input.inputPath, + `wxapkg directory is ${containerMetadata.directory_status}; its complete contents cannot be extracted`, + ); const selectedOccurrences = snapshot.occurrences.filter( (occurrence) => (occurrence.entry_kind === "file" || occurrence.entry_kind === "slice") && diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.ts b/src/artifacts/inventory/ArtifactGraphConstruction.ts index b0c77906f8..27561c6772 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.ts @@ -2,6 +2,10 @@ import { artifactFormatDescriptor } from "../artifactFormats.js"; import { hasGodotPckHeader } from "../../domain/godotPckFormat.js"; import { hasUImageMagic } from "../../domain/uimageFormat.js"; import { hasUnityFsHeader } from "../../domain/unityfsFormat.js"; +import { + hasWxapkgEncryptedMagic, + hasWxapkgHeader, +} from "../../domain/wxapkgFormat.js"; import { detectHexRecordEncoding } from "../../domain/firmware/hexRecords.js"; import { recognizeEspImage } from "../../domain/firmware/espImage.js"; import { hasUf2Block } from "../../domain/firmware/uf2.js"; @@ -296,7 +300,8 @@ export const rootOccurrenceFor = ( executable: metadata.executable, encrypted: (metadata.formatMetadata?.kind === "godot-pck" || - metadata.formatMetadata?.kind === "unityfs") && + metadata.formatMetadata?.kind === "unityfs" || + metadata.formatMetadata?.kind === "wxapkg") && metadata.formatMetadata.directory_status === "encrypted", hash_status: "verified", source_location: null, @@ -607,6 +612,8 @@ export const classifyArtifactBytes = ( if (hasZipSignature(prefix)) return "zip"; if (hasGodotPckHeader(prefix)) return "godot-pck"; if (hasUnityFsHeader(prefix)) return "unityfs"; + if (hasWxapkgEncryptedMagic(prefix) || hasWxapkgHeader(prefix)) + return "wxapkg"; if (hasUImageMagic(prefix)) return "uimage"; const hexRecords = detectHexRecordEncoding(prefix); if (hexRecords !== undefined) return hexRecords; diff --git a/src/domain/analysisSnapshot.ts b/src/domain/analysisSnapshot.ts index 7aba9dd72f..913c998ff6 100644 --- a/src/domain/analysisSnapshot.ts +++ b/src/domain/analysisSnapshot.ts @@ -76,6 +76,7 @@ const formatSchema = z.enum([ "ines", "godot-pck", "unityfs", + "wxapkg", "dmg", "pkg", "intel-hex", diff --git a/src/domain/analysisView/artifactInventoryView.ts b/src/domain/analysisView/artifactInventoryView.ts index 50fa587ce1..cd368e68a3 100644 --- a/src/domain/analysisView/artifactInventoryView.ts +++ b/src/domain/analysisView/artifactInventoryView.ts @@ -277,6 +277,7 @@ export const projectArtifactInventoryView = ( view.kind === "inventory" || inventory.manifest.root_format === "godot-pck" || inventory.manifest.root_format === "unityfs" || + inventory.manifest.root_format === "wxapkg" || inventory.manifest.root_format === "pyinstaller" || inventory.manifest.root_format === "node-sea" || inventory.manifest.root_format === "intel-hex" || diff --git a/src/domain/artifactFormatMetadata.ts b/src/domain/artifactFormatMetadata.ts index a53a6c0524..ab07a138ab 100644 --- a/src/domain/artifactFormatMetadata.ts +++ b/src/domain/artifactFormatMetadata.ts @@ -63,6 +63,24 @@ export const unityFsEntryMetadataSchema = z.strictObject({ flags: uint32, }); +/** Parsed source fields owned by the wxapkg adapter, independent of member bytes. */ +export const wxapkgMetadataSchema = z.strictObject({ + kind: z.literal("wxapkg"), + info: uint32.nullable(), + index_length: uint32.nullable(), + data_length: uint32.nullable(), + file_count: uint32.nullable(), + directory_status: z.enum(["available", "encrypted"]), +}); +export type WxapkgMetadata = z.infer; + +export const wxapkgEntryMetadataSchema = z.strictObject({ + kind: z.literal("wxapkg-entry"), + index: uint32, + source_path: z.string(), + declared_offset: offset, +}); + /** Source cookie fields and derived directory counts without Python execution. */ export const pyInstallerMetadataSchema = z.strictObject({ kind: z.literal("pyinstaller"), @@ -765,6 +783,8 @@ export const artifactFormatMetadataSchema = z.discriminatedUnion("kind", [ godotPckEntryMetadataSchema, unityFsMetadataSchema, unityFsEntryMetadataSchema, + wxapkgMetadataSchema, + wxapkgEntryMetadataSchema, pyInstallerMetadataSchema, pyInstallerEntryMetadataSchema, dotnetBundleEntryMetadataSchema, @@ -837,6 +857,14 @@ const declarationsByKind: { flags: metadata.flags, }), }, + wxapkg: { + notCompared: + "Its fields come from the package's own bytes, which content comparison covers.", + }, + "wxapkg-entry": { + notCompared: + "index is layout. source_path keeps the declared name, including a producer root prefix the occurrence path strips.", + }, pyinstaller: { notCompared: "Its fields come from the archive's own bytes, which content comparison covers.", diff --git a/src/domain/artifactGraph.ts b/src/domain/artifactGraph.ts index 9528ba6885..675e7c828e 100644 --- a/src/domain/artifactGraph.ts +++ b/src/domain/artifactGraph.ts @@ -67,6 +67,7 @@ export const ARTIFACT_FORMAT_IDS = [ "ines", "godot-pck", "unityfs", + "wxapkg", "uimage", "javascript-bundle", "source-map", @@ -113,6 +114,7 @@ const artifactContentFormatSchema = artifactFormatSchema.extract([ "zip", "godot-pck", "unityfs", + "wxapkg", "uimage", "intel-hex", "srec", diff --git a/src/domain/binaryTargetTypes.ts b/src/domain/binaryTargetTypes.ts index 743a92c9a2..22e6f3e848 100644 --- a/src/domain/binaryTargetTypes.ts +++ b/src/domain/binaryTargetTypes.ts @@ -293,6 +293,7 @@ export type BinaryTarget = | "ines" | "godot-pck" | "unityfs" + | "wxapkg" | "uimage" | "dmg" | "pkg" diff --git a/src/domain/evidenceSubject.ts b/src/domain/evidenceSubject.ts index 5c0994b32c..d9876742e3 100644 --- a/src/domain/evidenceSubject.ts +++ b/src/domain/evidenceSubject.ts @@ -27,6 +27,7 @@ export const evidenceSubjectFormatSchema = z.enum([ "ines", "godot-pck", "unityfs", + "wxapkg", "uimage", "dmg", "pkg", diff --git a/src/domain/liveBinaryTargetSummary.ts b/src/domain/liveBinaryTargetSummary.ts index ab1efe39f5..45b6d9c0da 100644 --- a/src/domain/liveBinaryTargetSummary.ts +++ b/src/domain/liveBinaryTargetSummary.ts @@ -73,6 +73,7 @@ export const liveBinaryTargetSummaryVariants = [ "ines", "godot-pck", "unityfs", + "wxapkg", "uimage", "dmg", "pkg", diff --git a/src/domain/wxapkgFormat.ts b/src/domain/wxapkgFormat.ts new file mode 100644 index 0000000000..d1cecb0e39 --- /dev/null +++ b/src/domain/wxapkgFormat.ts @@ -0,0 +1,198 @@ +import { err, ok, type Result } from "./result.js"; + +/** + * WeChat mini-program package (wxapkg): an 18-byte big-endian header, a + * name/offset/size member table, then verbatim member data. Members are stored + * uncompressed at absolute file offsets. All multi-byte fields are big-endian. + */ +export const WXAPKG_FIRST_MARK = 0xbe; +export const WXAPKG_LAST_MARK = 0xed; +export const WXAPKG_HEADER_BYTES = 18; + +/** Minimum fixed row fields: name length, offset and size without name bytes. */ +export const WXAPKG_MIN_ROW_BYTES = 12; + +/** + * The member count is stored in the last four header bytes. Some producers + * include those bytes in `indexLength`; others count only the rows that follow. + */ +const WXAPKG_MEMBER_COUNT_BYTES = 4; + +/** + * Encrypted PC-client packages start with this ASCII magic instead of the + * plain header. Their member data is ciphertext; no decryption is supported. + */ +export const WXAPKG_ENCRYPTED_MAGIC = Buffer.from("V1MMWX", "latin1"); + +/** Whether bytes start with the encrypted PC-client magic. */ +export const hasWxapkgEncryptedMagic = (bytes: Uint8Array): boolean => + bytes.length >= WXAPKG_ENCRYPTED_MAGIC.length && + Buffer.from( + bytes.buffer, + bytes.byteOffset, + WXAPKG_ENCRYPTED_MAGIC.length, + ).equals(WXAPKG_ENCRYPTED_MAGIC); + +/** Declared header fields; support and extent admission belong to callers. */ +export interface WxapkgHeaderFacts { + /** Unknown header field after the first mark; producers write zero. */ + readonly info: number; + /** + * Declared index length. This is either the row bytes or those rows plus + * the 4-byte member count; {@link resolveWxapkgIndexLayouts} admits both. + */ + readonly indexLength: number; + /** Declared byte length of the member data region. */ + readonly dataLength: number; + readonly fileCount: number; +} + +/** + * Decode declared fields after the caller admits a prefix covering the fixed + * header. Undefined means the admitted bytes cannot carry a plausible header; + * both mark checks and minimal row accounting must hold. + */ +export const readWxapkgHeaderFacts = ( + bytes: Uint8Array, +): WxapkgHeaderFacts | undefined => { + if ( + bytes.length < WXAPKG_HEADER_BYTES || + bytes[0] !== WXAPKG_FIRST_MARK || + bytes[13] !== WXAPKG_LAST_MARK + ) + return undefined; + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const info = view.getUint32(1); + const indexLength = view.getUint32(5); + const dataLength = view.getUint32(9); + const fileCount = view.getUint32(14); + // Every row carries at least its fixed fields; a table too small for the + // declared count cannot be a wxapkg index. + if (fileCount === 0 || indexLength < fileCount * WXAPKG_MIN_ROW_BYTES) + return undefined; + return { info, indexLength, dataLength, fileCount }; +}; + +/** + * Whether bytes carry the plain wxapkg marks. Recognition is intentionally + * separate from admission: a damaged package keeps its format identity and the + * reader rejects impossible table accounting as a typed format failure. + */ +export const hasWxapkgHeader = (bytes: Uint8Array): boolean => + bytes.length >= WXAPKG_HEADER_BYTES && + bytes[0] === WXAPKG_FIRST_MARK && + bytes[13] === WXAPKG_LAST_MARK; + +/** Row table and declared data region for one index-length convention. */ +export interface WxapkgIndexLayout { + /** Absolute offset of the first name/offset/size row. */ + readonly tableStart: number; + /** Row bytes to read. The member count, when included, stays in the header. */ + readonly tableBytes: number; + /** Inclusive start of the declared member-data region. */ + readonly dataStart: number; + /** Exclusive end of the declared member-data region. */ + readonly dataEnd: number; +} + +export type WxapkgIndexLayoutFailure = + | { readonly _tag: "IndexExceedsFile" } + | { readonly _tag: "DataRegionExceedsFile" }; + +const spanEnd = (start: number, length: number): number | undefined => { + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(length) || + length < 0 + ) + return undefined; + const end = start + length; + return Number.isSafeInteger(end) ? end : undefined; +}; + +const indexLayout = ( + facts: WxapkgHeaderFacts, + fileSize: number, + includedCountBytes: number, +): WxapkgIndexLayout | "index-outside" | "data-outside" | "too-small" => { + if ( + !Number.isSafeInteger(facts.indexLength) || + !Number.isSafeInteger(facts.fileCount) || + facts.fileCount < 0 || + facts.indexLength < includedCountBytes + ) + return "too-small"; + const tableBytes = facts.indexLength - includedCountBytes; + if (tableBytes < facts.fileCount * WXAPKG_MIN_ROW_BYTES) return "too-small"; + const dataStart = spanEnd(WXAPKG_HEADER_BYTES, tableBytes); + if (dataStart === undefined || dataStart > fileSize) return "index-outside"; + const dataEnd = spanEnd(dataStart, facts.dataLength); + if (dataEnd === undefined || dataEnd > fileSize) return "data-outside"; + return { tableStart: WXAPKG_HEADER_BYTES, tableBytes, dataStart, dataEnd }; +}; + +/** + * Admit every index-length convention whose index and declared data region + * both lie inside the carrier. Trailing bytes can make both fit, so callers + * keep every candidate until the parsed rows and member ranges decide. + */ +export const resolveWxapkgIndexLayouts = ( + facts: WxapkgHeaderFacts, + fileSize: number, +): Result => { + if (!Number.isSafeInteger(fileSize) || fileSize < 0) + return err({ _tag: "IndexExceedsFile" }); + const described = [WXAPKG_MEMBER_COUNT_BYTES, 0].map((included) => + indexLayout(facts, fileSize, included), + ); + const admissible = described.filter( + (layout): layout is WxapkgIndexLayout => typeof layout !== "string", + ); + if (admissible.length === 0) + return err({ + _tag: described.some((layout) => layout === "data-outside") + ? "DataRegionExceedsFile" + : "IndexExceedsFile", + }); + return ok(admissible); +}; + +const memberFitsIndex = ( + layout: WxapkgIndexLayout, + member: { readonly offset: number; readonly size: number }, +): boolean => { + const extentEnd = member.offset + member.size; + return ( + member.offset >= layout.dataStart && + extentEnd <= layout.dataEnd && + extentEnd >= member.offset + ); +}; + +/** + * Choose the admitted index that contains the parsed rows and every member + * range. An index that ends on the last parsed row wins a tie. Neither the + * carrier end nor the shorter table is enough on its own: four trailing bytes + * can make the other convention land on EOF. + */ +export const selectWxapkgIndexLayout = ( + layouts: readonly WxapkgIndexLayout[], + consumedRowBytes: number, + members: readonly { readonly offset: number; readonly size: number }[], +): WxapkgIndexLayout | undefined => { + const fitting = layouts.filter( + (layout) => + layout.tableBytes >= consumedRowBytes && + members.every((member) => memberFitsIndex(layout, member)), + ); + const exactRows = fitting.filter( + (layout) => layout.tableBytes === consumedRowBytes, + ); + const pool = exactRows.length > 0 ? exactRows : fitting; + let selected: WxapkgIndexLayout | undefined; + for (const layout of pool) { + if (selected === undefined || layout.tableBytes < selected.tableBytes) + selected = layout; + } + return selected; +}; diff --git a/tests/boundary/mcp/wxapkgMcp.test.ts b/tests/boundary/mcp/wxapkgMcp.test.ts new file mode 100644 index 0000000000..3d1901421a --- /dev/null +++ b/tests/boundary/mcp/wxapkgMcp.test.ts @@ -0,0 +1,87 @@ +import { readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { artifactInspectionResultSchema } from "../../../src/domain/artifactInspection.js"; +import { artifactExtractionResultSchema } from "../../../src/domain/artifactGraph.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { buildWxapkg, wxapkgEncryptedBytes } from "../../fixtures/wxapkg.js"; + +it.each(["full", "compact"] as const)( + "retains wxapkg facts and encrypted coverage through compiled MCP (%s)", + async (profile) => { + const root = await createTestTempDirectory("rea-wxapkg-mcp-"); + const path = join(root, "package.wxapkg"); + const payload = Buffer.from("module.exports={};"); + await writeFile( + path, + buildWxapkg([ + { path: "app-config.json", data: "{}" }, + { path: "pages/index/index.js", data: payload }, + ]), + ); + const { call } = await connectLocalToolsMcp({ + REA_MCP_INPUT_SCHEMA_PROFILE: profile, + }); + let outputRoot: string | undefined; + try { + expect((await call("open_binary", { path })).isError).not.toBe(true); + const reply = await call("inspect_artifact", {}); + expect(reply.isError, JSON.stringify(reply)).not.toBe(true); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(reply.structuredContent).normalized_result, + ); + const observed = inspection.substeps[0]!.evidence.normalized_result; + expect(observed.occurrences).toContainEqual( + expect.objectContaining({ + logical_path: "pages/index/index.js", + hash_status: "verified", + source_location: expect.objectContaining({ + offset: expect.any(Number), + }), + }), + ); + const extracted = await call("extract_artifact", {}); + expect(extracted.isError, JSON.stringify(extracted)).not.toBe(true); + const result = artifactExtractionResultSchema.parse( + parseEvidence(extracted.structuredContent).normalized_result, + ); + outputRoot = result.output_root; + expect( + (await readFile(join(outputRoot, "pages/index/index.js"))).equals( + payload, + ), + ).toBe(true); + await call("close_binary", {}); + // Encrypted PC-client packages remain represented without member guesses. + await writeFile(path, wxapkgEncryptedBytes()); + await call("open_binary", { path }); + const unknownReply = await call("inspect_artifact", {}); + expect(unknownReply.isError, JSON.stringify(unknownReply)).not.toBe(true); + const unknown = artifactInspectionResultSchema.parse( + parseEvidence(unknownReply.structuredContent).normalized_result, + ).substeps[0]!.evidence.normalized_result; + expect(unknown.coverage).toMatchObject({ status: "unknown" }); + expect(unknown.occurrences).toMatchObject([ + { + artifact_format: "wxapkg", + encrypted: true, + format_metadata: { + kind: "wxapkg", + directory_status: "encrypted", + file_count: null, + }, + }, + ]); + const refused = await call("extract_artifact", {}); + expect(refused.isError).toBe(true); + expect(JSON.stringify(refused)).toContain("cannot be extracted"); + await call("close_binary", {}); + } finally { + if (outputRoot !== undefined) + await rm(outputRoot, { recursive: true, force: true }); + } + }, + 60_000, +); diff --git a/tests/fixtures/wxapkg.ts b/tests/fixtures/wxapkg.ts new file mode 100644 index 0000000000..58fa5211d0 --- /dev/null +++ b/tests/fixtures/wxapkg.ts @@ -0,0 +1,114 @@ +/** Synthetic WeChat mini-program package (wxapkg) builders for reader tests. */ + +export interface WxapkgMember { + readonly path: string; + readonly data: string | Buffer; + /** Override the declared member offset (absolute file position). */ + readonly offset?: number; + /** Override the declared member byte size. */ + readonly size?: number; +} + +export interface WxapkgOptions { + readonly firstMark?: number; + readonly lastMark?: number; + readonly info?: number; + readonly declaredFileCount?: number; + readonly declaredIndexLength?: number; + readonly declaredDataLength?: number; + /** + * Count the 4-byte member count inside `indexLength`. Row bytes and member + * offsets stay where they are; only the declared length grows by four. + */ + readonly indexLengthIncludesFileCount?: boolean; + /** Override the encoded row's name-length field for every member. */ + readonly nameLength?: number; + /** Bytes appended to the table region beyond the declared rows. */ + readonly tablePadding?: number; + /** Declare every member after the first at the first member's offset. */ + readonly aliasOffsets?: boolean; + /** Replace the whole package prefix (encrypted variants). */ + readonly magic?: Buffer; + /** Truncate the final package to this many bytes. */ + readonly truncate?: number; +} + +const utf8 = (value: string): Buffer => Buffer.from(value, "utf8"); +const u32 = (value: number): Buffer => { + const out = Buffer.alloc(4); + out.writeUInt32BE(value); + return out; +}; + +/** Encrypted PC-client package prefix, without plaintext members. */ +export const wxapkgEncryptedBytes = (): Buffer => + Buffer.concat([Buffer.from("V1MMWX", "latin1"), Buffer.alloc(64)]); + +/** Plain wxapkg: 18-byte header, name/offset/size rows, then verbatim data. */ +export const buildWxapkg = ( + members: readonly WxapkgMember[], + options: WxapkgOptions = {}, +): Buffer => { + if (options.magic !== undefined) + return Buffer.concat([ + options.magic, + ...members.map(({ data }) => + typeof data === "string" ? utf8(data) : data, + ), + ]); + const names = members.map(({ path }) => utf8(path)); + // Encoded rows carry the real name bytes regardless of any declared override. + const rowBytes = names.reduce((sum, name) => sum + 12 + name.length, 0); + const rowRegion = rowBytes + (options.tablePadding ?? 0); + const indexLength = + options.declaredIndexLength ?? + rowRegion + (options.indexLengthIncludesFileCount === true ? 4 : 0); + // A declared length override repositions the data. Including the member + // count changes only the length field; the rows still begin at byte 18. + const dataStart = + options.declaredIndexLength !== undefined + ? 18 + indexLength + : 18 + rowRegion; + let cursor = dataStart; + const rows: Buffer[] = []; + const payload: Buffer[] = []; + let firstOffset: number | undefined; + members.forEach((member, index) => { + const name = names[index]!; + const data = + typeof member.data === "string" ? utf8(member.data) : member.data; + const aliased = + options.aliasOffsets === true && index > 0 && firstOffset !== undefined; + const offset = aliased ? firstOffset! : (member.offset ?? cursor); + const size = member.size ?? data.length; + firstOffset ??= offset; + rows.push( + u32(options.nameLength ?? name.length), + name, + u32(offset), + u32(size), + ); + if (!aliased) { + payload.push(data); + if (offset === cursor) cursor = offset + size; + } + }); + const dataLength = + options.declaredDataLength ?? Math.max(0, cursor - dataStart); + const header = Buffer.concat([ + Buffer.from([options.firstMark ?? 0xbe]), + u32(options.info ?? 0), + u32(indexLength), + u32(dataLength), + Buffer.from([options.lastMark ?? 0xed]), + u32(options.declaredFileCount ?? members.length), + ]); + const table = Buffer.concat([ + ...rows, + Buffer.alloc(options.tablePadding ?? 0), + ]); + let bytes = Buffer.concat([header, table, ...payload]); + if (options.truncate !== undefined) + bytes = bytes.subarray(0, options.truncate); + return bytes; +}; From 0a99c2a5809fb1db8ca3ed58ece6ce69cbb8afe5 Mon Sep 17 00:00:00 2001 From: Entrpi <133157614+Entrpi@users.noreply.github.com> Date: Thu, 5 Nov 2026 03:54:53 +1100 Subject: [PATCH 73/94] fix(artifacts): give Godot PCK and Node SEA readers the ESP member lifecycle (#8395) * feat(artifacts): inventory and extract ESP32 firmware images ESP32-family app and bootloader images (magic 0xE9) had no reader, so open_binary refused OTA downloads, app partitions and bootloaders. Add the esp-image container format behind inspect_artifact and extract_artifact. A pure decoder reads each byte once and keeps only fixed-size fields. Derived facts cover the chip, from a table cited to ESP-IDF v6.1; segment classification as ESP-IDF's should_map and should_load; checksum and appended SHA-256 results; app and bootloader descriptors; Secure Boot v2 signature blocks, observed and unverified; and undescribed trailing byte ranges. Truncation and undescribed bytes make coverage unknown. ESP8266 images are refused with a specific reason, and ciphertext stays unrecognized. Add the format to every target, Evidence, snapshot, live-summary and artifact vocabulary, document it, and add an optional verify:esp-images lane that compares esptool-built images with esptool image-info and espsecure signature-info-v2. Refs #3615 * test(artifacts): run the ESP image lane on esptool v5.6.0 The lane compared descriptor fields only when esptool image-info printed one, so a descriptor that REA reports and esptool does not print passed without comparison. Compare the descriptor kind in both directions. Record that the lane passed for all 13 images with esptool and espsecure v5.6.0, the version the layout facts cite, as well as with 4.12.0. * refactor(artifacts): define ESP image layout sizes once The decoder and the derived facts each defined the segment-header, appended-digest and flash-sector sizes, the erased-byte scan and the offset formatter. Define them once with the other fixed-layout structures and import them in both places. The signature slot count is the one constant that drives the slot scan. Correct the observedEnd comment: it is the end of the last chunk read. * fix(artifacts): report ESP-IDF's segment length limit by chip ESP-IDF's verify_segment_header rejects a segment of 16 MiB or more unless the chip maps flash through 32-bit cache addresses (SOC_SPI_MEM_SUPPORT_CACHE_32BIT_ADDR_MAP: ESP32-S3, P4, C5, C61, H4 and S31 in v6.1). Record that capability in the chip table and add a limitation for such segments; unknown chips get none. `loaded` repeats should_load. It does not mean the bootloader accepts the segment: verify_load_addresses still rejects one outside the chip's RAM, which REA does not model. Say so in the docs, the skill reference and the inventory limitation. * test(artifacts): check ESP lane tools first and verify extraction The lane started work before checking its tools, so a missing RISC-V target, ld.lld, esptool or espsecure surfaced as a raw spawn or command error. Check each one before building and name the variable to set. Warn when esptool is not v5.6.0, the release the layout facts cite. Assert each segment's class for the ESP32-C3 app, its variants and the bootloader, as the Xtensa image already did. Extract the app and the signed image through the CLI and compare every file with the bytes at esptool's segment offsets or in the appended signature sector; the source's SHA-256 must be unchanged and equal the reported root_sha256. * docs(artifacts): state what the ESP image lane does not establish The Verification section listed the lane's checks but not its gaps: only ESP32-C3 and ESP32 images come from a real producer, no real project build, OTA download or flash dump is inspected, no signature is verified, and the lane does not run in CI. Record the timing of the final lane run on esptool v5.6.0. * fix(artifacts): compare ESP segment declarations Segment entries hash only their data, so compare_artifacts reported a segment that moved to another load address with the same bytes as unchanged; only the image itself changed. Compare each segment's load_addr and data_len as its declaration, as Node SEA and PyInstaller members are compared, and count a changed declaration as a metadata change, also when content is unavailable. Header offsets are layout and the classification is derived, so neither is compared. * test(artifacts): run ESP lane commands as owned process trees The lane ran its tools through execFile. A timeout killed only the direct child, so a linker or assembler that Clang or GCC had started could keep running while the lane removed its workspace. Run every command through createVerifierCommands, check the tools inside withVerifierCleanup, and remove the workspace only after commands.close() has stopped every tree; a failed close keeps the workspace. Only a nonzero exit now counts as a refusal or an expected esptool failure; timeouts and spawn failures propagate. Supervising each command roughly doubles the lane's run time. * test(artifacts): cancel ESP decoding after its first decoded chunk The cancellation test requested cancel or close on setImmediate, which can land while the file is still being opened and checked, so it never showed that decoding had started. It also passed against a reader that ignored both requests while decoding. Acknowledge the decoder's first chunk, then cancel or close, join the operation, and assert that no later chunk reaches the decoder and that the reader ends closed. * docs(artifacts): record which ESP metadata comparison ignores Segments are compared by their declaration. The image's and the signature blocks' format_metadata are not: every field comes from their own bytes, which content comparison already covers, and a block's image_digest_matches follows the image's content. * feat(artifacts): inventory and extract ESP flash layouts A whole-flash dump of an ESP32-family chip was read as its bootloader image, with the partition table, apps and data left as undescribed bytes, or refused when it began with erased flash. Read it as an esp-flash-image container instead. The partition table is decoded as esp_partition_table_verify accepts it and is found at the caller's partition_table_offset, at the default 0x8000, or by scanning 0x1000-aligned sectors for a table whose MD5 verifies, which the inventory reports as derived. The bootloader is located at its chip's ROM bootloader offset. Each partition is an entry with its declaration in flat format_metadata; the bootloader and app partitions carry ESP image facts from the existing streaming decoder; the otadata records are reported with a selection derived without rollback or anti-rollback, and the running partition stays unknown. One unreadable partition does not hide the others, and extraction writes every partition, nvs_keys included, as ordinary data. Flash layouts are detected before single ESP images. The scan runs only when an ESP image sits at a ROM bootloader offset, and is refused when a bootloader at another chip's offset shows that the dump does not start at flash address 0. open_binary gains flash_offset and partition_table_offset, which require format esp-flash-image and are part of the snapshot target identity; the CLI takes them with --target-format esp-flash-image. Comparison treats a partition's label, type, subtype, range and flags as its declaration. * test(artifacts): verify ESP flash layouts with gen_esp32part, merge-bin and otatool Add the optional verify:esp-flash lane. It builds default and relocated partition tables with ESP-IDF's gen_esp32part.py, assembles dumps with esptool merge-bin, including one written from flash 0x1000, and writes OTA records with otatool through flash_file_tool.py, which replaces parttool's esptool calls with reads and writes of the dump file. It compares REA's bootloader and app facts with esptool image-info, every partition with gen_esp32part.py's decoding of the extracted table, and the OTA records and derived slot with otatool read_otadata after each switch. It also checks extracted bytes and source integrity, the scan and caller table offsets, flash_offset on the 0x1000 dump, a corrupted app beside valid partitions and a truncated dump. ESP32 linker scripts place the conformance bootloader and app in ESP32 address ranges. * fix(artifacts): read ESP flash layouts as the bootloader does Read the partition table as bootloader_utility_load_partition_table does: the last factory, otadata and same-subtype OTA declaration wins, and a limitation names repeats. Plan each otadata record within its partition and the dump. An otadata partition smaller than two sectors selects none, the bootloader's INVALID_INDEX; a record the dump lacks leaves the selection unknown while the other record stays reported. The fallbacks follow the bootloader's indexes: ota_0-fallback and factory-fallback name no partition when the table declares none, and the assumptions add GPIO holds and deep-sleep wake. Refuse a dump whose bootloader contradicts its origin before the default table offset is tried, whether flash_offset was omitted or given, and keep the ESP image reader's ESP8266 refusal. A table never lies inside the located bootloader image, so single app images are no longer scanned. An explicit esp-flash-image selection is decided before other container signatures, and the native commands' --target-format no longer takes it. Say when no bootloader confirms the default origin. Exempt the primary bootloader and partition_table entries of ESP-IDF 5.3 from placement limits, report every region inside a longer one, and name partitions by index when labels repeat ignoring case. Keep an undecodable label's bytes in label_hex and compare them. Word undescribed ranges as starting at a byte that is not erased, and treat a short erased region as erased. Add an MCP compare_artifacts test for partition declarations, reader identity and close tests, scan edge cases, the 0x2000 ROM offset and a literal otatool CRC. * test(artifacts): verify ESP primary layout entries and the otadata fallback Build a table with the primary bootloader and partition_table entries that gen_esp32part.py fills from --primary-bootloader-offset, and check that REA reads it with complete coverage, no overlap or misplacement limitation, and a table that agrees with gen_esp32part's decoding. Erase the otadata with otatool after the slot switches and check the records against read_otadata and the factory fallback. * refactor(artifacts): share positional entry lifecycle The ESP image and ESP flash-layout readers carried the same 138 lines of lifecycle code. PositionalArtifactReader now owns it: one preparation under the reader's scope, a carrier recheck before each traversal, refusal of forged entries, range streams that check their length and recheck the carrier when they end, and a close that retires open streams before it releases the carrier. Each reader keeps its location, decoding, entries and metadata, and returns its entries and coverage from prepareEntries(). Entries carry a required byte range, which removes the unreachable null-range checks in open(). Failure reasons, messages, order and cancellation do not change. The Godot PCK reader keeps its own copy. It reports a member that ends early as a format failure, where the ESP readers report integrity, and the helper cannot express that without an option. The Node SEA reader has the same difference. * fix(artifacts): report Godot PCK and Node SEA members that end early as integrity Behavior change: a Godot PCK or Node SEA member stream that delivers fewer bytes than its entry declares now fails with reason "integrity" and the message "