fix(cli)!: refuse lossy text output (#7852)

* fix(cli)!: refuse lossy text output

* fix(cli): retain failure identity on text refusal

* fix(cli): check failed TOON token windows

* test(cli): bound keyed archive case duration

docs/cli.md

@@ -1048,8 +1048,8 @@ Malformed format values and token windows fail with `INVALID_OUTPUT_VALUE` in th
 selected structured format. Token limits and offsets must be nonnegative safe
 integers; zero selects an empty window. These checks run before command execution. `--json` is indented on a terminal and compact when piped or
 redirected, which keeps large results smaller for agents and scripts; pipe
-through `jq .` for indented files. Output selection and formatting do not change
-operation status.
+through `jq .` for indented files. A formatting refusal exits nonzero even when
+the underlying operation succeeded.
 
 `--format md` escapes backslashes, pipes, and inline Markdown syntax in table
 headers and cells so source text is displayed literally, and renders cell line
@@ -1061,6 +1061,12 @@ values instead of replacing them with an object label. Table widths are measured
 iteratively, so an admitted row page is not limited by the JavaScript function
 argument count.
 
+Markdown refuses selected NUL characters and unpaired UTF-16 surrogates because
+renderers or UTF-8 output would substitute them. TOON refuses selected unpaired
+surrogates for the same UTF-8 reason. Use JSON or YAML to retain those values.
+An authored replacement character and valid supplementary characters are not
+treated as malformed.
+
 `--format csv` and `--format tsv` export one selected row collection. Select an
 exact array path with `--filter-output` and provide comma-separated direct
 scalar columns with `--columns`; column order is the output column order:
@@ -1080,7 +1086,8 @@ delimiter, quotes, or line breaks are quoted and embedded quotes are doubled.
 Values are emitted as UTF-8 without Unicode normalization. Missing and `null`
 cells are empty; booleans and finite numbers use their ordinary text forms.
 String cells and headers that begin with spreadsheet formula syntax are refused
-with a structured error; JSON retains those values unchanged.
+with a structured error; unpaired surrogates in selected cells or headers are
+also refused. JSON retains those values unchanged.
 
 Delimited output does not flatten nested Evidence or nested field values.
 It returns a structured error and a nonzero status if the filter does not

package-lock.json

@@ -61,6 +61,7 @@
         "@stryker-mutator/core": "10.0.0",
         "@stryker-mutator/typescript-checker": "10.0.0",
         "@stryker-mutator/vitest-runner": "10.0.0",
+        "@toon-format/toon": "2.3.1",
         "@types/cross-spawn": "6.0.6",
         "@types/node": "26.6.4",
         "@types/semver": "7.7.1",

package.json

@@ -399,6 +399,7 @@
     "@stryker-mutator/core": "10.0.0",
     "@stryker-mutator/typescript-checker": "10.0.0",
     "@stryker-mutator/vitest-runner": "10.0.0",
+    "@toon-format/toon": "2.3.1",
     "@types/cross-spawn": "6.0.6",
     "@types/node": "26.6.4",
     "@types/semver": "7.7.1",

src/cli/cliOutputFormatter.ts

@@ -33,6 +33,58 @@ export interface CliDelimitedOutputOptions {
 
 type DelimitedScalar = string | number | boolean | null | undefined;
 
+/** UTF-8 cannot encode a lone UTF-16 surrogate without replacing it. */
+export const hasUnpairedSurrogate = (value: string): boolean =>
+  /[\ud800-\udfff]/u.test(value);
+
+/** Check selected output, including keys, before using a non-JSON text format. */
+export const containsUnpairedSurrogate = (value: unknown): boolean => {
+  return containsText(value, hasUnpairedSurrogate);
+};
+
+/** Markdown renderers replace NUL as well as UTF-8-invalid code units. */
+export const containsUnrepresentableMarkdown = (value: unknown): boolean =>
+  containsText(
+    value,
+    (text) => text.includes("\u0000") || hasUnpairedSurrogate(text),
+  );
+
+const containsText = (
+  value: unknown,
+  predicate: (text: string) => boolean,
+): boolean => {
+  const pending: Iterator<unknown>[] = [[value][Symbol.iterator]()];
+  const seen = new WeakSet<object>();
+  while (pending.length > 0) {
+    const next = pending.at(-1)!.next();
+    if (next.done === true) {
+      pending.pop();
+      continue;
+    }
+    const current: unknown = next.value;
+    if (typeof current === "string") {
+      if (predicate(current)) return true;
+      continue;
+    }
+    if (typeof current !== "object" || current === null || seen.has(current))
+      continue;
+    seen.add(current);
+    pending.push(outputValues(current));
+  }
+  return false;
+};
+
+function* outputValues(value: object): Generator<unknown> {
+  if (Array.isArray(value)) {
+    yield* value;
+    return;
+  }
+  for (const key of Object.keys(value)) {
+    yield key;
+    yield (value as Record<string, unknown>)[key];
+  }
+}
+
 export interface PreparedDelimitedOutput {
   readonly encodedHeaders: readonly string[];
   readonly rows: readonly (readonly DelimitedScalar[])[];
@@ -97,6 +149,14 @@ export const prepareDelimitedOutput = (
 
   const encodedHeaders: string[] = [];
   for (const field of fields) {
+    if (hasUnpairedSurrogate(field))
+      return err(
+        delimitedOutputInputError(
+          ["--columns"],
+          "CSV/TSV cannot preserve unpaired UTF-16 surrogates in UTF-8 output; use JSON.",
+          operation,
+        ),
+      );
     if (isSpreadsheetFormula(field))
       return err(
         delimitedOutputInputError(
@@ -135,6 +195,14 @@ export const prepareDelimitedOutput = (
             operation,
           ),
         );
+      if (typeof value === "string" && hasUnpairedSurrogate(value))
+        return err(
+          delimitedOutputInputError(
+            ["--columns", field],
+            "CSV/TSV cannot preserve unpaired UTF-16 surrogates in UTF-8 output; use JSON.",
+            operation,
+          ),
+        );
       preparedRow.push(value);
     }
     preparedRows.push(preparedRow);
@@ -250,6 +318,11 @@ const escapeInlineSyntax = (
 
 const escapeTableCell = (value: string): string =>
   value
+    .replace(
+      /\u0000|[\ud800-\udfff]/gu,
+      (character) =>
+        `\\u${character.charCodeAt(0).toString(16).padStart(4, "0")}`,
+    )
     .replace(/&/g, "&amp;")
     .replace(/</g, "&lt;")
     .replace(/>/g, "&gt;")

src/cli/cliResultOutput.ts

@@ -29,6 +29,9 @@ import {
   formatCliOutput,
   prepareDelimitedOutput,
   delimitedOutputInputError,
+  containsUnpairedSurrogate,
+  containsUnrepresentableMarkdown,
+  hasUnpairedSurrogate,
 } from "./cliOutputFormatter.js";
 import {
   cliEvidencePublicationReceipt,
@@ -477,6 +480,62 @@ export const createCliResultOutput = (
       return true;
     }
     const selected = selection.value;
+    const refuseUnrepresentableText = async (): Promise<true> => {
+      error = projectAnalysisError(
+        new AnalysisInputError(
+          metadata.command,
+          {
+            userMessage:
+              format === "md"
+                ? "Markdown cannot preserve NUL or unpaired UTF-16 surrogates in rendered output."
+                : "TOON cannot preserve unpaired UTF-16 surrogates in UTF-8 output.",
+            remediationAction:
+              "Use --format json or --format yaml to retain the exact code units.",
+            ...(completed?.originalFailure === undefined
+              ? {}
+              : {
+                  diagnostics: { original_failure: completed.originalFailure },
+                }),
+          },
+          [
+            {
+              path: ["--format"],
+              reason: "invalid_value",
+              message: "The selected format cannot preserve this value.",
+            },
+          ],
+        ),
+      );
+      const document = options.fullOutput
+        ? {
+            ok: false,
+            error,
+            meta: { command: metadata.command, duration: metadata.duration },
+          }
+        : error;
+      handled = true;
+      try {
+        await writeOutputText(
+          destination,
+          `${formatCliOutput(document, format)}\n`,
+        );
+      } catch (cause: unknown) {
+        failed = true;
+        throw cause;
+      }
+      return true;
+    };
+    if (
+      (format === "toon" || format === "md") &&
+      !options.tokenCount &&
+      (!options.tokenWindow ||
+        format === "md" ||
+        operationFailure !== undefined) &&
+      (format === "md"
+        ? containsUnrepresentableMarkdown(selected)
+        : containsUnpairedSurrogate(selected))
+    )
+      return refuseUnrepresentableText();
     const filtered =
       (format === "json" || format === "jsonl") &&
       ownsFilteredOutput &&
@@ -649,6 +708,8 @@ export const createCliResultOutput = (
             format,
           )
         : sliced;
+      if (format === "toon" && hasUnpairedSurrogate(outputDocument))
+        return refuseUnrepresentableText();
       handled = true;
       try {
         await writeOutputText(

tests/acceptance/applications/keyedArchiveTextUnicode.test.ts

@@ -0,0 +1,145 @@
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { decode } from "@toon-format/toon";
+import { buildBinary } from "plist";
+import { parseFragment, type DefaultTreeAdapterTypes } from "parse5";
+import { createMarkdownRenderer, disposeMdItInstance } from "vitepress";
+import { afterAll, expect } from "vitest";
+import { parse as parseYaml } from "yaml";
+
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+import { cliTest } from "../../support/cli/cliFixture.js";
+
+afterAll(() => disposeMdItInstance());
+
+const renderedText = (node: DefaultTreeAdapterTypes.Node): string =>
+  node.nodeName === "#text" && "value" in node
+    ? node.value
+    : "childNodes" in node
+      ? node.childNodes.map(renderedText).join("")
+      : "";
+
+cliTest.for([
+  ["high", "probe:\ud800:end"],
+  ["low", "probe:\udc00:end"],
+  ["pair", "probe:😀:end"],
+  ["replacement", "probe:�:end"],
+  ["nul", "probe:\u0000:end"],
+  ["literal", "probe:\\u0000:end"],
+] as const)(
+  "preserves binary keyed-archive code-unit identity across output formats (%s)",
+  async ([label, value], { cli }) => {
+    const root = await createTestTempDirectory("rea-keyed-text-unicode-");
+    const renderer = await createMarkdownRenderer(root, { linkify: false });
+    const path = join(root, `${label}.bplist`);
+    await writeFile(
+      path,
+      buildBinary({
+        $version: 100000,
+        $archiver: "NSKeyedArchiver",
+        $top: { root: { UID: 1 } },
+        $objects: ["$null", value],
+      }),
+    );
+    const run = (args: readonly string[]) =>
+      cli.run({
+        arguments: ["inspect-keyed-archive", path, ...args],
+      });
+    for (const format of ["json", "yaml"] as const) {
+      const output = await run(["--format", format]);
+      expect(output.exitCode, output.stderr).toBe(0);
+      const document =
+        format === "json"
+          ? JSON.parse(output.stdout)
+          : parseYaml(output.stdout);
+      expect(document.normalized_result.objects[1].value).toBe(value);
+    }
+    for (const args of [[], ["--format", "toon"]] as const) {
+      const output = await run(args);
+      const document = decode(output.stdout) as {
+        code?: string;
+        normalized_result?: { objects: { value?: string }[] };
+      };
+      if (label === "high" || label === "low") {
+        expect(output.exitCode).toBe(1);
+        expect(document.code).toBe("invalid_request");
+        expect(output.stdout).toContain("--format json");
+        expect(output.stdout).not.toContain("�");
+      } else {
+        expect(output.exitCode, output.stderr).toBe(0);
+        expect(document.normalized_result?.objects[1]?.value).toBe(value);
+      }
+    }
+    const markdown = await run(["--format", "md"]);
+    if (label === "high" || label === "low" || label === "nul") {
+      expect(markdown.exitCode).toBe(1);
+      expect(markdown.stdout).toContain("--format json");
+      expect(markdown.stdout).not.toContain("�");
+      return;
+    }
+    expect(markdown.exitCode, markdown.stderr).toBe(0);
+    const rendered = renderedText(
+      parseFragment(renderer.render(markdown.stdout)),
+    );
+    expect(rendered).toContain(value);
+  },
+);
+
+cliTest(
+  "keeps NUL-bearing class names distinct in rendered Markdown",
+  async ({ cli }) => {
+    const root = await createTestTempDirectory("rea-keyed-class-unicode-");
+    const renderer = await createMarkdownRenderer(root, { linkify: false });
+    for (const [label, name, expected] of [
+      ["nul", "My\u0000Class", ""],
+      ["replacement", "My�Class", "My�Class"],
+      ["literal", "My\\u0000Class", "My\\u0000Class"],
+    ] as const) {
+      const path = join(root, `${label}.bplist`);
+      await writeFile(
+        path,
+        buildBinary({
+          $version: 100000,
+          $archiver: "NSKeyedArchiver",
+          $top: { root: { UID: 1 } },
+          $objects: [
+            "$null",
+            { $class: { UID: 2 } },
+            { $classname: name, $classes: [name, "NSObject"] },
+            name,
+            [name, "NSObject"],
+            "NSObject",
+          ],
+        }),
+      );
+      const json = await cli.run({
+        arguments: ["inspect-keyed-archive", path, "--json"],
+      });
+      expect(json.exitCode, json.stderr).toBe(0);
+      expect(
+        JSON.parse(json.stdout).normalized_result.objects[1].class_name,
+      ).toBe(name);
+      const toon = await cli.run({
+        arguments: ["inspect-keyed-archive", path, "--format", "toon"],
+      });
+      expect(toon.exitCode, toon.stderr).toBe(0);
+      const toonDocument = decode(toon.stdout) as {
+        normalized_result: { objects: { class_name?: string }[] };
+      };
+      expect(toonDocument.normalized_result.objects[1]?.class_name).toBe(name);
+      const markdown = await cli.run({
+        arguments: ["inspect-keyed-archive", path, "--format", "md"],
+      });
+      if (label === "nul") {
+        expect(markdown.exitCode).toBe(1);
+        expect(markdown.stdout).toContain("--format json");
+        continue;
+      }
+      expect(markdown.exitCode, markdown.stderr).toBe(0);
+      const rendered = renderedText(
+        parseFragment(renderer.render(markdown.stdout)),
+      );
+      expect(rendered).toContain(expected);
+    }
+  },
+);

tests/boundary/cli/textUnicodeOutput.test.ts

@@ -0,0 +1,192 @@
+import { Writable } from "node:stream";
+import { expect, it } from "vitest";
+import { decode } from "@toon-format/toon";
+import { createCliResultOutput } from "../../../src/cli/cliResultOutput.js";
+import { parseCliOutputArguments } from "../../../src/cliOutput.js";
+import { logCliCommand } from "../../../src/cliLogging.js";
+import { silentLogger } from "../../../src/logger.js";
+import { AnalysisInputError } from "../../../src/domain/analysisErrorCore.js";
+import {
+  formatCliOutput,
+  prepareDelimitedOutput,
+} from "../../../src/cli/cliOutputFormatter.js";
+
+it("refuses TOON values or keys that UTF-8 cannot preserve", async () => {
+  for (const value of [{ name: "a\ud800b" }, { ["a\udc00b"]: "value" }]) {
+    const chunks: Buffer[] = [];
+    const output = createCliResultOutput(
+      parseCliOutputArguments(["--format", "toon"]),
+      new Writable({
+        write(chunk: Buffer, _encoding, done) {
+          chunks.push(chunk);
+          done();
+        },
+      }),
+    );
+    if (output === undefined) throw new Error("invalid output settings");
+    expect(
+      await output.write(value, {
+        command: "fixture",
+        duration: "1ms",
+        format: "toon",
+      }),
+    ).toBe(true);
+    expect(output.error).toMatchObject({ code: "invalid_request" });
+    expect(decode(Buffer.concat(chunks).toString("utf8"))).toMatchObject({
+      code: "invalid_request",
+      remediation: { action: expect.stringContaining("--format json") },
+    });
+  }
+});
+
+it.each([false, true])(
+  "retains a classified operation failure when TOON cannot encode its diagnostic (token window: %s)",
+  async (tokenWindow) => {
+    const previousExitCode = process.exitCode;
+    try {
+      const value = await logCliCommand(silentLogger, "fixture", async () => {
+        throw new AnalysisInputError("fixture", { userMessage: "bad \ud800" });
+      });
+      const output = createCliResultOutput(
+        parseCliOutputArguments([
+          "--format",
+          "toon",
+          "--full-output",
+          ...(tokenWindow ? ["--token-limit", "1000"] : []),
+        ]),
+        new Writable({
+          write(_chunk, _encoding, done) {
+            done();
+          },
+        }),
+      );
+      if (output === undefined) throw new Error("invalid output settings");
+      expect(
+        await output.write(value, {
+          command: "fixture",
+          duration: "1ms",
+          format: "toon",
+        }),
+      ).toBe(true);
+      expect(output.error).toMatchObject({
+        code: "invalid_request",
+        details: {
+          diagnostics: {
+            original_failure: {
+              code: "invalid_request",
+              category: "invalid_input",
+              retryable: false,
+            },
+          },
+        },
+      });
+    } finally {
+      process.exitCode = previousExitCode;
+    }
+  },
+);
+
+it("keeps valid supplementary characters and an authored replacement character", async () => {
+  const output = createCliResultOutput(
+    parseCliOutputArguments(["--format", "toon"]),
+    new Writable({
+      write(_chunk, _encoding, done) {
+        done();
+      },
+    }),
+  );
+  if (output === undefined) throw new Error("invalid output settings");
+  expect(
+    await output.write(
+      { value: "😀�" },
+      { command: "fixture", duration: "1ms", format: "toon" },
+    ),
+  ).toBe(false);
+});
+
+it("only refuses malformed code units in selected TOON output", async () => {
+  const chunks: Buffer[] = [];
+  const output = createCliResultOutput(
+    parseCliOutputArguments(["--format", "toon", "--filter-output", "safe"]),
+    new Writable({
+      write(chunk: Buffer, _encoding, done) {
+        chunks.push(chunk);
+        done();
+      },
+    }),
+  );
+  if (output === undefined) throw new Error("invalid output settings");
+  expect(
+    await output.write(
+      { safe: "retained", other: "\ud800" },
+      { command: "fixture", duration: "1ms", format: "toon" },
+    ),
+  ).toBe(true);
+  expect(output.error).toBeUndefined();
+  expect(decode(Buffer.concat(chunks).toString("utf8"))).toBe("retained");
+});
+
+it("refuses malformed code units retained in a TOON token window", async () => {
+  const chunks: Buffer[] = [];
+  const output = createCliResultOutput(
+    parseCliOutputArguments(["--format", "toon", "--token-limit", "1000"]),
+    new Writable({
+      write(chunk: Buffer, _encoding, done) {
+        chunks.push(chunk);
+        done();
+      },
+    }),
+  );
+  if (output === undefined) throw new Error("invalid output settings");
+  expect(
+    await output.write(
+      { value: "\ud800" },
+      { command: "fixture", duration: "1ms", format: "toon" },
+    ),
+  ).toBe(true);
+  expect(output.error).toMatchObject({ code: "invalid_request" });
+});
+
+it("shows NUL visibly in Markdown", () => {
+  const markdown = formatCliOutput({ rows: [{ value: "a\u0000b�😀" }] }, "md");
+  expect(markdown).toContain("a\\\\u0000b�😀");
+  expect(markdown).not.toContain("\u0000");
+});
+
+it.each(["\ud800", "\u0000"])(
+  "refuses unrepresentable Markdown code units without substituting them",
+  async (value) => {
+    const chunks: Buffer[] = [];
+    const output = createCliResultOutput(
+      parseCliOutputArguments(["--format", "md"]),
+      new Writable({
+        write(chunk: Buffer, _encoding, done) {
+          chunks.push(chunk);
+          done();
+        },
+      }),
+    );
+    if (output === undefined) throw new Error("invalid output settings");
+    expect(
+      await output.write(
+        { value },
+        { command: "fixture", duration: "1ms", format: "md" },
+      ),
+    ).toBe(true);
+    expect(output.error).toMatchObject({ code: "invalid_request" });
+    expect(Buffer.concat(chunks).toString("utf8")).not.toContain("�");
+  },
+);
+
+it("refuses unrepresentable delimited cells and column names before output", () => {
+  for (const [rows, fields] of [
+    [[{ x: "\ud800" }], ["x"]],
+    [[{ ["\udc00"]: "x" }], ["\udc00"]],
+  ] as const) {
+    const result = prepareDelimitedOutput(rows, "csv", {
+      fields,
+      metadata: {},
+    });
+    expect(result.ok).toBe(false);
+  }
+});