From fcfd416dcdeee0d825e1b7fe6966cc8ff6fe1926 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 4 Nov 2026 09:25:25 -0800 Subject: [PATCH] fix(cli): keep inline-image transmits out of the replayed scrollback image_lines put the whole kitty APC transmit (70 KB-1 MB of base64) inside the first placeholder row, and the iTerm2 OSC 1337 overlay (ESC7 ESC[{rows-1}A ESC8) inside the last reserved row, and those rows went into _OUTPUT_HISTORY. On a resize replay that keeps a row only in part, _ansi_drop_cells walked the transmit char by char and cut it into visible base64; a cropped iTerm2 region kept the cursor-up and drew the image over the replayed text above it. prompt_toolkit's ANSI parser also walked the full transmit on every _cprint, _line_rows and replay (35-800 ms on the UI loop), and every resize re-sent it. _cprint takes a `raw` escape now, written verbatim as one zero-width fragment after the row and never recorded; image_rows hands the transmit there and history holds only the placeholder or blank rows. kitty keeps the image by id, so replayed placeholders still show it. An iTerm2 image is pixels over its reserved rows, so a replay repaints those rows blank: the picture is lost from the redrawn viewport (scrollback keeps it) instead of overpainting text. The overlay now also steps past the row indent, as the placeholders do. _ANSI_SEQUENCE_RE matches prompt_toolkit's \001...\002 brackets whole, so a cropped placeholder row keeps the exact truecolor that names its image instead of a split bracket the parser quantizes. --- hermes_cli/cli_inline_images.py | 45 +++++++++------ hermes_cli/cli_render.py | 35 ++++++++---- tests/hermes_cli/test_cli_inline_images.py | 64 +++++++++++++++++----- 3 files changed, 101 insertions(+), 43 deletions(-) diff --git a/hermes_cli/cli_inline_images.py b/hermes_cli/cli_inline_images.py index 8ee868788e..fa076e8565 100644 --- a/hermes_cli/cli_inline_images.py +++ b/hermes_cli/cli_inline_images.py @@ -1,15 +1,22 @@ """Inline images in the classic CLI scrollback: the user's attachments and image tool results. -Images print as ordinary text rows (kitty Unicode placeholders, or half-blocks) so the scrollback -replay and its row accounting keep working. The kitty escapes ride in prompt_toolkit's zero-width -``\\001…\\002`` brackets: its ANSI printer would otherwise quantize the truecolor that carries the -image id to the output's color depth (256 by default), and strip the APC transmit outright. -Half-block colors go through that quantization on purpose, like the rest of the scrollback. +Images print as ordinary text rows (kitty Unicode placeholders, half-blocks, or blank rows an +iTerm2 image is drawn over) so the scrollback replay and its row accounting keep working. The +transmit itself goes out once, raw, beside its row (``_cprint(raw=...)``) and never enters the +output history: a resize replay repaints the rows alone, so it can't crop a 1 MB escape into +text or re-send it. kitty keeps the image by id, so replayed placeholders still show it. An +iTerm2 image is pixels drawn over its reserved rows; a replay repaints those rows blank, so the +picture is lost in the redrawn viewport (scrollback above keeps it) rather than painted over +whatever text the replay put above a partly kept image. + +The kitty id color rides in prompt_toolkit's zero-width ``\\001…\\002`` brackets: its ANSI +printer would otherwise quantize the truecolor that carries the image id to the output's color +depth (256 by default). Half-block colors go through that quantization on purpose, like the rest +of the scrollback. """ from __future__ import annotations -import shutil import sys from collections.abc import Iterable @@ -35,35 +42,37 @@ def _half_block(top, bottom) -> str: return f"\x1b[38;2;{tr};{tg};{tb}m\x1b[48;2;{br};{bg};{bb}m▀" -def image_lines(image: InlineImage) -> list[str]: - """*image* as scrollback lines; the first one (kitty) or the last one (iTerm2) carries the transmit.""" +def image_rows(image: InlineImage) -> list[tuple[str, str]]: + """*image* as ``(row, raw)`` pairs: ``row`` is history-safe scrollback text, ``raw`` the + escape written verbatim right after it (the transmit: first kitty row, last iTerm2 row).""" if image.mode == "kitty": r, g, b = bytes.fromhex(image.color[1:]) color, reset = _zero_width(f"\x1b[38;2;{r};{g};{b}m"), _zero_width("\x1b[39m") - lines = [_INDENT + color + row + reset for row in image.placeholder] - lines[0] = _zero_width(image.transmit) + lines[0] - return lines + rows = [(_INDENT + color + row + reset, "") for row in image.placeholder] + rows[0] = (rows[0][0], image.transmit) + return rows if image.mode == "iterm": # Reserve the rows as empty lines (text written over an image cell erases it), then from - # the last one jump back to the first, draw, and restore the cursor where it was. + # the start of the last one jump to the first, past the indent, draw, and restore. up = f"\x1b[{image.rows - 1}A" if image.rows > 1 else "" - return [""] * (image.rows - 1) + [_INDENT + _zero_width(f"\x1b7{up}{image.transmit}\x1b8")] - return [_INDENT + "".join(_half_block(t, b) for t, b in row) + "\x1b[0m" for row in image.cells] + draw = f"\x1b7{up}\x1b[{len(_INDENT)}C{image.transmit}\x1b8" + return [("", "")] * (image.rows - 1) + [("", draw)] + return [(_INDENT + "".join(_half_block(t, b) for t, b in row) + "\x1b[0m", "") for row in image.cells] def print_inline_images(cli, paths: Iterable) -> None: """Print each image in *paths* under the current scrollback line, per ``display.inline_images``.""" - from cli import _cprint + from cli import _cprint, _terminal_columns stdout = sys.__stdout__ if stdout is None or not stdout.isatty(): return mode = resolve_inline_mode(getattr(cli, "_inline_images_mode", "auto"), overlays=True) if mode == "off": return - max_cols = min(_MAX_COLS, shutil.get_terminal_size().columns - len(_INDENT) - 1) + max_cols = min(_MAX_COLS, _terminal_columns() - len(_INDENT) - 1) for path in paths: image = render_inline_image(path, mode=mode, max_cols=max_cols, max_rows=_MAX_ROWS) # One print per row, like the diff printer: a whole truecolor image in a single write # overruns the Windows console write and the tail rows are lost. - for line in image_lines(image) if image is not None else (): - _cprint(line) + for row, raw in image_rows(image) if image is not None else (): + _cprint(row, raw=raw) diff --git a/hermes_cli/cli_render.py b/hermes_cli/cli_render.py index f52ab7c294..1a65ba3745 100644 --- a/hermes_cli/cli_render.py +++ b/hermes_cli/cli_render.py @@ -521,7 +521,9 @@ def _record_output_history(text: str, *, force: bool = False) -> None: _cli()._OUTPUT_HISTORY.extend(lines) -_ANSI_SEQUENCE_RE = re.compile(r"\x1b(?:\[[0-?]*[ -/]*[@-~]|\][^\x07\x1b]*(?:\x07|\x1b\\)|[@-Z\\-_])") +# prompt_toolkit's zero-width ``\001…\002`` brackets first: a crop that split one would leak its +# escape to the ANSI parser (which quantizes an inline image's id color) or as text. +_ANSI_SEQUENCE_RE = re.compile(r"\001[^\002]*\002|\x1b(?:\[[0-?]*[ -/]*[@-~]|\][^\x07\x1b]*(?:\x07|\x1b\\)|[@-Z\\-_])") def _ansi_drop_cells(line: str, cells: int) -> str: @@ -638,15 +640,24 @@ def _output_history_rows(limit: int, columns: int, painted: bool): return total -def _pt_print_ansi(text: str) -> None: +def _ansi_with_raw(text: str, raw: str = ""): + """``ANSI(text)``, then *raw* as one zero-width fragment prompt_toolkit writes verbatim.""" + from cli import _PT_ANSI + if not raw: + return _PT_ANSI(text) + from prompt_toolkit.formatted_text import to_formatted_text + return [*to_formatted_text(_PT_ANSI(text)), ("[ZeroWidthEscape]", raw)] + + +def _pt_print_ansi(text: str, raw: str = "") -> None: """``_pt_print(ANSI(text))``, falling back to ``print`` when stdout is not a real console.""" - from cli import _PT_ANSI, _pt_print + from cli import _pt_print try: - _pt_print(_PT_ANSI(text)) + _pt_print(_ansi_with_raw(text, raw)) except Exception: # NoConsoleScreenBufferError (Windows) / OSError when stdout is e.g. a worker log file. with suppress(Exception): - print(text) + print(text + raw) _HELD_PAINTS: list | None = None @@ -733,13 +744,17 @@ def _paint_held(seq: int, paint, app=None) -> bool: return True -def _cprint(text: str): +def _cprint(text: str, *, raw: str = ""): """Print ANSI text through prompt_toolkit's renderer (patch_stdout swallows raw ANSI). From a background thread while an Application runs, a direct print races the input redraw and gets buried, so those are painted on the app's loop via ``call_soon_threadsafe``. + + *raw* is an escape sequence written verbatim right after *text* (an inline image's + transmit): never parsed as ANSI, counted as zero cells, and kept out of the output + history, so a resize replay repaints *text* alone and never crops or re-sends it. """ - from cli import _PT_ANSI, _output_history_recording, _pt_print, _pt_print_ansi, _record_output_history + from cli import _output_history_recording, _pt_print, _pt_print_ansi, _record_output_history recording = _output_history_recording() seq = next(_PAINT_SEQ) @@ -751,8 +766,8 @@ def _cprint(text: str): _record_output_history(text, force=True) paint() return _paint - paint_pt = _painted(lambda: _pt_print(_PT_ANSI(text))) - paint_fallback = _painted(lambda: _pt_print_ansi(text)) + paint_pt = _painted(lambda: _pt_print(_ansi_with_raw(text, raw))) + paint_fallback = _painted(lambda: _pt_print_ansi(text, raw)) try: from prompt_toolkit.application import get_app_or_none, run_in_terminal @@ -793,7 +808,7 @@ def _cprint(text: str): from prompt_toolkit.formatted_text import to_formatted_text from prompt_toolkit.renderer import print_formatted_text as _paint_formatted_text from prompt_toolkit.styles import Style - _paint_formatted_text(app.output, to_formatted_text(_PT_ANSI(text)) + [("", "\n")], Style([])) + _paint_formatted_text(app.output, to_formatted_text(_ansi_with_raw(text, raw)) + [("", "\n")], Style([])) paint_now = _painted(_print_now) def _schedule(): diff --git a/tests/hermes_cli/test_cli_inline_images.py b/tests/hermes_cli/test_cli_inline_images.py index 1111e7de21..e95a938637 100644 --- a/tests/hermes_cli/test_cli_inline_images.py +++ b/tests/hermes_cli/test_cli_inline_images.py @@ -12,7 +12,9 @@ from prompt_toolkit.renderer import print_formatted_text from prompt_toolkit.styles import Style from agent.terminal_image import render_inline_image -from hermes_cli.cli_inline_images import image_lines +import cli as cli_mod +from hermes_cli.cli_inline_images import image_rows, print_inline_images +from hermes_cli.cli_render import _output_tail_fitting PIL = pytest.importorskip("PIL.Image") @@ -35,21 +37,18 @@ def _painted(line: str) -> str: def test_kitty_rows_reach_the_terminal_with_exact_id_color_and_transmit(photo): image = render_inline_image(photo, mode="kitty", max_cols=40, max_rows=10) - lines = image_lines(image) - assert len(lines) == image.rows + rows = image_rows(image) + assert len(rows) == image.rows r, g, b = bytes.fromhex(image.color[1:]) - first = _painted(lines[0]) # The 256-color default would round the color and lose the image id kitty decodes from it. - assert f"\x1b[38;2;{r};{g};{b}m" in first - assert image.transmit in first - assert all(f"\x1b[38;2;{r};{g};{b}m" in _painted(line) for line in lines[1:]) + assert all(f"\x1b[38;2;{r};{g};{b}m" in _painted(row) for row, _raw in rows) + assert [raw for _row, raw in rows] == [image.transmit] + [""] * (image.rows - 1) @pytest.mark.parametrize("mode", ["kitty", "unicode"]) def test_image_rows_measure_as_their_cell_width(photo, mode): - # Scrollback replay sizes rows by measured width: the transmit must count as zero cells. image = render_inline_image(photo, mode=mode, max_cols=40, max_rows=10) - widths = {fragment_list_width(to_formatted_text(ANSI(line))) for line in image_lines(image)} + widths = {fragment_list_width(to_formatted_text(ANSI(row))) for row, _raw in image_rows(image)} assert widths == {image.cols + 2} @@ -58,9 +57,44 @@ def test_iterm_rows_are_reserved_blank_and_the_image_draws_over_them(photo): # ``rows`` blank lines and the last one jumps back up, draws, and restores the cursor. image = render_inline_image(photo, mode="iterm", max_cols=40, max_rows=10) assert image is not None - lines = image_lines(image) - assert len(lines) == image.rows > 1 - assert [fragment_list_width(to_formatted_text(ANSI(line))) for line in lines] == [0] * (image.rows - 1) + [2] - painted = _painted(lines[-1]) - assert f"\x1b7\x1b[{image.rows - 1}A\x1b]1337;File=inline=1;" in painted - assert f"width={image.cols};height={image.rows}" in painted and "\x07\x1b8" in painted + rows = image_rows(image) + assert len(rows) == image.rows > 1 + assert [row for row, _raw in rows] == [""] * image.rows + draw = rows[-1][1] + assert draw.startswith(f"\x1b7\x1b[{image.rows - 1}A\x1b[2C\x1b]1337;File=inline=1;") and draw.endswith("\x07\x1b8") + assert f"width={image.cols};height={image.rows}" in draw + + +@pytest.fixture +def scrollback(monkeypatch): + """The CLI's real scrollback printer into a buffer, with output history on, on a TTY.""" + buf = io.StringIO() + output = Vt100_Output(buf, lambda: Size(rows=40, columns=120), term="xterm-kitty") + monkeypatch.setattr(cli_mod, "_pt_print", lambda text: print_formatted_text(output, text, Style([]))) + monkeypatch.setattr("sys.__stdout__", type("Tty", (), {"isatty": staticmethod(lambda: True)})()) + for key in ("TMUX", "STY"): + monkeypatch.delenv(key, raising=False) + cli_mod._configure_output_history(True, 200) + yield buf + cli_mod._configure_output_history(True, 200) + + +@pytest.mark.parametrize(("mode", "transmit_start"), [("kitty", "\x1b_G"), ("iterm", "\x1b]1337;")]) +def test_transmit_is_written_once_and_kept_out_of_replayed_history(scrollback, photo, mode, transmit_start): + # A resize replay re-prints (and may crop) history rows: a transmit stored there was cut into + # base64 text, re-sent whole on every redraw, or redrawn an image over replayed text. + print_inline_images(type("Cli", (), {"_inline_images_mode": mode})(), [photo]) + assert scrollback.getvalue().count(transmit_start) == 1 + history = list(cli_mod._OUTPUT_HISTORY) + assert history and not any(transmit_start in line or "\x1b7" in line for line in history) + assert not any(transmit_start in line for line in _output_tail_fitting(history, 3, 120)) + + +def test_cropped_kitty_row_keeps_its_exact_id_color(photo): + # A partly kept placeholder row must still paint in the truecolor that names its image. + image = render_inline_image(photo, mode="kitty", max_cols=40, max_rows=10) + row = cli_mod._PaintedLine(image_rows(image)[0][0]) + row.width = 20 # painted at 20 columns: three rows, of which the replay keeps the last + [kept] = _output_tail_fitting([row], 1, 20) + r, g, b = bytes.fromhex(image.color[1:]) + assert f"\x1b[38;2;{r};{g};{b}m" in _painted(kept)