From 20991acc91b966159faa4249f2135b4c252dc6a4 Mon Sep 17 00:00:00 2001 From: Jean-Pierre Martineau Date: Thu, 16 Jul 2026 23:09:16 -0400 Subject: [PATCH] Scroll smear + ragged border: align the width model to a one-cell emoji MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The erase_down workaround stopped the lingering but left the real fault in place, which showed as the leftover artifact: a bordered row with an emoji rendered one cell short, so its right border sat a column in with a blank beside it. Root cause is a three-party disagreement about one glyph. Rich lays out every row (and places the right border), prompt_toolkit diffs against the last frame stepping a *relative* cursor by its own width model, and the terminal draws the result. Rich and pt both measure an emoji as two cells (wcwidth); the terminal draws it in one. That one-cell gap is both symptoms: pt's cursor drifts a column past the terminal and strands stale cells (the smear), and Rich sizes the row one short so the border lands early (the ragged edge). "Drawing the border last" can't reach it — pt diffs per row across all columns, and the terminal has already eaten the cell no matter the compositing order. Fix it at the source: teach both width authorities the terminal's truth. New glyphwidth.align_emoji_cell_width() patches pt's _CHAR_SIZES_CACHE and Rich's get_character_cell_size so an emoji measures one cell — while CJK/fullwidth glyphs (which terminals do draw two wide) stay two, and the width-1 marks and braille are untouched. With all three parties agreeing, ordinary differential painting stays synced (no lingering) and Rich pads rows to a flush border (no notch) — so the erase_down/_has_wide_glyph machinery is gone entirely. Applied at TuiSession construction, before anything renders; gated by MESHTERM_NARROW_EMOJI (default on), the sibling of the MESHTERM_FULL_WIDTH column-reclaim shim. Co-Authored-By: Claude Opus 4.8 --- meshterm/ui/tui/glyphwidth.py | 131 ++++++++++++++++++++++++++++++++++ meshterm/ui/tui/session.py | 54 ++++---------- tests/test_tui.py | 100 +++++++++++++------------- 3 files changed, 194 insertions(+), 91 deletions(-) create mode 100644 meshterm/ui/tui/glyphwidth.py diff --git a/meshterm/ui/tui/glyphwidth.py b/meshterm/ui/tui/glyphwidth.py new file mode 100644 index 0000000..f7ceb6c --- /dev/null +++ b/meshterm/ui/tui/glyphwidth.py @@ -0,0 +1,131 @@ +"""Align the app's glyph-width model with how the terminal actually draws emoji. + +Terminal rendering is a three-party agreement about how many cells each glyph occupies, and +this app has three parties: **Rich** lays out every line (wrapping, padding, and — the part +that bites — the position of a panel's right border), **prompt_toolkit** diffs one frame +against the last and steps a *relative* cursor by its own width model, and the **terminal** +draws the result. Rich and prompt_toolkit both measure an emoji (``👋``, ``🕒``, ``⚡``) as +two cells, via ``wcwidth``. The terminals this app runs on draw it in a *single* cell. + +That one-cell disagreement is the whole bug: + +* prompt_toolkit reserves two cells but the terminal advances one, so from the emoji rightward + the diff's cursor model is a column ahead of the terminal. On the next paint, a cell to the + emoji's right that pt repositions to lands a column early, and the stale cell it meant to + overwrite lingers — the "characters that don't get cleared" smear. +* Rich sizes the row assuming two cells, so a bordered line with one emoji renders one cell + short and its right border sits a column in from the frame edge — a ragged notch with a + blank beside it. + +Both vanish the moment all three parties agree. We can't change the terminal, so we teach the +two width authorities the terminal's truth: an emoji is one cell. Rich then pads the row to a +full-width, flush border, and prompt_toolkit's relative cursor tracks the terminal exactly, so +ordinary differential painting stays synced and nothing lingers — no whole-frame repaint, no +compositing tricks. CJK and fullwidth glyphs, which terminals *do* draw two cells wide, are +left at two; node-type marks and braille are already one cell everywhere and are untouched. + +This is the sibling of :class:`~meshterm.ui.tui.session._WidthExtendedOutput`: both correct a +width assumption the terminal breaks, in the one place the whole pipeline reads it. Gated by +``MESHTERM_NARROW_EMOJI`` (default on); set it to ``0`` on a terminal that genuinely draws +emoji two cells wide, where the alignment would instead *introduce* a one-cell error. +""" + +from __future__ import annotations + +import os + +from wcwidth import wcwidth + +#: Whether to align the width model to a one-cell emoji (see the module docstring). Default on; +#: ``MESHTERM_NARROW_EMOJI=0`` opts out for a terminal that draws emoji two cells wide. +_ALIGN_NARROW_EMOJI = os.environ.get("MESHTERM_NARROW_EMOJI", "1") != "0" + + +def _is_cjk_or_fullwidth(cp: int) -> bool: + """Whether ``cp`` is a codepoint terminals reliably draw *two* cells wide. + + These are the East Asian Wide/Fullwidth blocks — CJK ideographs and their radicals, the + kana and Hangul syllables, and the fullwidth ASCII forms. They share ``wcwidth``'s value of + two with emoji, so width alone can't tell them apart; the split is by block. Everything here + is left at two cells, because the terminal draws it at two — only the *rest* of the width-2 + range (the emoji) is the part the terminal narrows. + """ + return ( + 0x1100 <= cp <= 0x115F # Hangul Jamo + or 0x2E80 <= cp <= 0x9FFF # CJK radicals, Kangxi, kana, CJK Unified Ideographs + or 0xA000 <= cp <= 0xA4CF # Yi syllables + or 0xAC00 <= cp <= 0xD7A3 # Hangul syllables + or 0xF900 <= cp <= 0xFAFF # CJK compatibility ideographs + or 0xFE30 <= cp <= 0xFE4F # CJK compatibility forms + or 0xFF00 <= cp <= 0xFF60 # Fullwidth forms + or 0xFFE0 <= cp <= 0xFFE6 # Fullwidth signs + or 0x20000 <= cp <= 0x3FFFD # CJK Unified Ideographs Extension B and beyond + ) + + +def is_narrow_emoji(char: str) -> bool: + """Whether ``char`` is a glyph ``wcwidth`` calls two cells but the terminal draws in one. + + True for an emoji or pictographic symbol — a single codepoint that ``wcwidth`` measures at + two and that is not in a CJK/fullwidth block. The ``>= 0x1100`` guard skips the ASCII/Latin + bulk before the width lookup; ``len == 1`` keeps this to lone codepoints (multi-codepoint + grapheme clusters — flags, ZWJ sequences — are left to their parts). + """ + if len(char) != 1: + return False + cp = ord(char) + return cp >= 0x1100 and wcwidth(char) == 2 and not _is_cjk_or_fullwidth(cp) + + +#: Set once :func:`align_emoji_cell_width` has patched the width authorities, so repeated calls +#: (one per :class:`~meshterm.ui.tui.session.TuiSession`) patch the global state exactly once. +_applied = False + + +def align_emoji_cell_width() -> None: + """Teach Rich and prompt_toolkit that an emoji is one cell, matching the terminal. + + Idempotent and gated by :data:`_ALIGN_NARROW_EMOJI`. Patches the two width authorities the + render pipeline reads: + + * **prompt_toolkit** — ``prompt_toolkit.utils._CHAR_SIZES_CACHE``, the dict behind + ``get_cwidth`` and therefore every ``Char.width``. We swap in a subclass whose + ``__missing__`` returns one for a narrow emoji and otherwise defers to the original + ``wcwidth`` logic, so the diff's cursor model matches the terminal. + * **Rich** — ``rich.cells.get_character_cell_size``, resolved as a module global by + ``_cell_len`` on every measurement. We wrap it to return one for a narrow emoji, then + clear ``cached_cell_len``'s cache so no pre-patch measurement survives, so Rich lays out + (and borders) each row at the terminal's true width. + + Called at :class:`~meshterm.ui.tui.session.TuiSession` construction, before any frame is + drawn, so no stale two-cell ``Char`` or cached measurement is ever painted. + """ + global _applied + if _applied or not _ALIGN_NARROW_EMOJI: + return + _applied = True + + import prompt_toolkit.utils as ptu + + base_cache_cls = type(ptu._CHAR_SIZES_CACHE) + + class _NarrowEmojiCache(base_cache_cls): # type: ignore[valid-type, misc] + def __missing__(self, string: str) -> int: + if is_narrow_emoji(string): + self[string] = 1 + return 1 + return super().__missing__(string) + + ptu._CHAR_SIZES_CACHE = _NarrowEmojiCache() + + import rich.cells as rc + + _measure = rc.get_character_cell_size + + def _narrow_emoji_cell_size(character: str, unicode_version: str = "auto") -> int: + if is_narrow_emoji(character): + return 1 + return _measure(character, unicode_version) + + rc.get_character_cell_size = _narrow_emoji_cell_size + rc.cached_cell_len.cache_clear() diff --git a/meshterm/ui/tui/session.py b/meshterm/ui/tui/session.py index d190afa..60ac727 100644 --- a/meshterm/ui/tui/session.py +++ b/meshterm/ui/tui/session.py @@ -24,11 +24,11 @@ from prompt_toolkit.key_binding import KeyBindings from prompt_toolkit.keys import Keys from prompt_toolkit.layout import ConditionalContainer, Float, FloatContainer, Layout, Window from prompt_toolkit.layout.controls import FormattedTextControl -from prompt_toolkit.utils import get_cwidth from rich.console import RenderableType from rich.text import Text from . import frame +from .glyphwidth import align_emoji_cell_width from .overlay import BusyOverlay from .progress import TuiProgress from .prompt import ( @@ -91,18 +91,6 @@ _RECLAIM_LAST_COLUMN = os.environ.get("MESHTERM_FULL_WIDTH", "1") != "0" #: real nesting — a tool's list, an item's detail popup, and a confirm over that is only three. _MAX_DIALOG_LAYERS = 8 -def _has_wide_glyph(text: str) -> bool: - """Whether ``text`` holds a glyph prompt_toolkit reserves two cells for. - - A width-2 glyph is where the terminal and the renderer can disagree: an emoji (``👋``) - that the terminal draws in a *single* cell is the common case here — pt reserves two, - the terminal advances one, and from that point the row's cursor model is off. Node-type - marks (``▲●■``) and chart braille are width-1 everywhere, so they never trip this. The - ``>= 0x1100`` guard skips the ASCII/Latin bulk of a frame before the width lookup, which - matters because this runs over the whole composed frame on every repaint. - """ - return any(ord(ch) >= 0x1100 and get_cwidth(ch) == 2 for ch in text) - class _WidthExtendedOutput: """A prompt_toolkit ``Output`` proxy that reports one extra terminal column. @@ -173,6 +161,9 @@ class TuiSession: output: Optional prompt_toolkit output to render to (tests use a dummy); defaults to the real terminal. """ + # Align the width model with how the terminal draws emoji (one cell, not two) before + # anything renders, so Rich borders sit flush and pt's differential cursor stays synced. + align_emoji_cell_width() self._stack: list[Screen] = [] self._header = header or (lambda cols: Text("MeshTerm", style="brand")) self._app: Optional[Application] = None @@ -1006,28 +997,6 @@ class TuiSession: """Whether any dialog floats over the background this frame.""" return bool(self._float_layers()) - def _emit(self, text: str) -> ANSI: - """Wrap a composed frame as prompt_toolkit :class:`ANSI`, forcing a full repaint - when it holds a glyph the terminal may draw narrower than pt reserves for it. - - prompt_toolkit paints differentially: it rewrites only the cells that changed since - the last frame, and it steps the cursor *relative* to its own width model. That is - sound only while every glyph is one cell wide. A width-2 glyph the terminal draws in - a single cell (an emoji in a chat line, a menu icon) leaves the terminal's cursor one - column left of pt's model for the rest of that row; on the next paint, any cell to the - emoji's right that pt repositions to — because the emoji itself didn't change and was - skipped — lands one column off, and the stale cell it should have overwritten lingers. - So when a frame carries such a glyph, drop pt's cached frame (:meth:`_invalidate_last_frame`) - so the next paint is a full ``erase_down`` + redraw: every cell is rewritten - contiguously, letting the terminal's own cursor advance keep the row aligned, and - nothing stale survives. Frames with only width-1 glyphs keep the fast differential - paint. This is checked per frame, so it covers a scroll, a resize, or a timer tick - alike — wherever the glyph is (a full-frame chat, a floating dialog, the overlay). - """ - if _has_wide_glyph(text): - self._invalidate_last_frame() - return ANSI(text) - def _render_base(self) -> ANSI: """Render the persistent frame around the background screen.""" cols, rows = self._size() @@ -1040,9 +1009,9 @@ class TuiSession: # A chromeless base (the startup splash) forgoes the header/footer bars and is # centered under its banner instead of stretched across the terminal. if not base.chrome: - return self._emit(frame.compose_startup(base, cols, rows)) + return ANSI(frame.compose_startup(base, cols, rows)) footer = self.top.footer_hint if self.top else base.footer_hint - return self._emit(frame.compose_base(self._header(cols), base, footer, cols, rows)) + return ANSI(frame.compose_base(self._header(cols), base, footer, cols, rows)) def _render_float_layer(self, index: int) -> ANSI: """Render the ``index``-th floating dialog (bottom-to-top) as a centered box.""" @@ -1050,7 +1019,7 @@ class TuiSession: if index >= len(layers): return ANSI("") cols, rows = self._size() - return self._emit(frame.compose_dialog(layers[index], cols, rows)) + return ANSI(frame.compose_dialog(layers[index], cols, rows)) def _overlay_visible(self) -> bool: """Whether the busy overlay should be painted this frame. @@ -1067,7 +1036,7 @@ class TuiSession: """Render the busy overlay's skeleton card (only when :meth:`_overlay_visible`).""" if self._overlay is None: return ANSI("") - return self._emit(self._overlay.render()) + return ANSI(self._overlay.render()) # --- input --------------------------------------------------------------- @@ -1099,9 +1068,10 @@ class TuiSession: def _dispatch(self, action: str, data: str = "") -> None: """Forward an action to the top screen and repaint. - The repaint keeps prompt_toolkit's fast differential paint; a frame carrying a glyph - the terminal may draw narrower than pt reserves for it (an emoji) upgrades itself to a - full repaint at compose time — see :meth:`_emit`. + The repaint is prompt_toolkit's efficient differential paint. That stays correct across + emoji because the width model is aligned to the terminal at startup (one cell per emoji, + via :func:`~meshterm.ui.tui.glyphwidth.align_emoji_cell_width`), so pt's relative cursor + never drifts and no stale cells are left behind. """ top = self.top if top is not None: diff --git a/tests/test_tui.py b/tests/test_tui.py index 44c413e..2ba8293 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -1480,63 +1480,65 @@ def test_session_background_is_the_topmost_full_frame_screen() -> None: assert session._float_layers() == [dialog] -def test_wide_glyph_detection_flags_emoji_not_marks() -> None: - """The desync only ever comes from a width-2 glyph the terminal may draw narrower — an - emoji. Node-type marks, status marks and chart braille are width-1 everywhere, so they - must not trip the check (that they seemed to was the earlier misdiagnosis).""" - from meshterm.ui.tui.session import _has_wide_glyph +def test_narrow_emoji_predicate_splits_emoji_from_cjk_and_marks() -> None: + """The width model is aligned to the terminal for exactly one class of glyph: an emoji the + terminal draws in one cell while wcwidth calls it two. CJK/fullwidth glyphs (drawn two cells + wide) and the width-1 marks and braille must stay out of it — that split is what keeps the + alignment from breaking CJK layout or double-counting a node mark.""" + from meshterm.ui.tui.glyphwidth import is_narrow_emoji - assert _has_wide_glyph("👋") - assert _has_wide_glyph("Bob 👋 waved") - assert _has_wide_glyph("clock 🕒 sync") - assert _has_wide_glyph("⚡ explore") # a width-2 icon still counts - assert not _has_wide_glyph("plain ascii row") - assert not _has_wide_glyph("★ ▲ ● ■ ◉ ○") # node-type marks: width 1 - assert not _has_wide_glyph("⠿⣿⡇ chart") # braille: width 1 - assert not _has_wide_glyph("✓ ✗ ⚠ … done") # status marks: width 1 + assert is_narrow_emoji("👋") # waving hand — the reported case + assert is_narrow_emoji("🕒") + assert is_narrow_emoji("⚡") # a BMP pictograph icon, still width-2 + assert not is_narrow_emoji("日") # CJK: genuinely two cells, leave it + assert not is_narrow_emoji("A") # fullwidth Latin: two cells + assert not is_narrow_emoji("▲") # node-type mark: one cell everywhere + assert not is_narrow_emoji("⠿") # braille: one cell + assert not is_narrow_emoji("A") # plain ascii + assert not is_narrow_emoji("👋🏽") # multi-codepoint cluster: left to its parts -def test_a_wide_glyph_frame_upgrades_to_a_full_repaint() -> None: - """prompt_toolkit paints differentially with a *relative* cursor — sound only while every - glyph is one cell. A width-2 glyph the terminal draws in one cell (an emoji in a chat line) - leaves the row's cursor model off; a later paint that skips the unchanged emoji then strands - stale cells to its right. So a composed frame carrying such a glyph drops pt's cached frame, - upgrading the next paint to a full erase_down + redraw. A frame of only width-1 glyphs keeps - the fast differential paint — this holds wherever the glyph is, floating dialog or not.""" - import types +def test_aligning_emoji_width_agrees_with_the_terminal() -> None: + """Aligning teaches both width authorities the terminal's truth — an emoji is one cell. + prompt_toolkit's get_cwidth (behind every Char.width, so the differential cursor) and Rich's + cell_len (behind every layout and border) both then measure an emoji at one, while CJK stays + two and marks stay one. With all three parties agreeing, the differential paint never drifts + and no whole-frame repaint is needed.""" + from prompt_toolkit.utils import get_cwidth + from rich.cells import cell_len - from prompt_toolkit.data_structures import Size - from prompt_toolkit.layout.screen import Char - from prompt_toolkit.layout.screen import Screen as PtScreen + from meshterm.ui.tui.glyphwidth import align_emoji_cell_width - def _fake_app(last: PtScreen) -> types.SimpleNamespace: - return types.SimpleNamespace( - renderer=types.SimpleNamespace(_last_screen=last), - output=types.SimpleNamespace(get_size=lambda: Size(rows=10, columns=60)), - invalidate=lambda: None, - ) + align_emoji_cell_width() # idempotent; also applied by every TuiSession - def _filled() -> PtScreen: - screen = PtScreen() - for row in range(3): - for x in range(10): - screen.data_buffer[row][x] = Char("A") - return screen + assert get_cwidth("👋") == 1 and cell_len("👋") == 1 + assert get_cwidth("🕒") == 1 and cell_len("🕒") == 1 + assert get_cwidth("日") == 2 and cell_len("日") == 2 # CJK unchanged + assert get_cwidth("▲") == 1 and cell_len("▲") == 1 # mark unchanged - # An emoji anywhere in the composed frame drops the remembered frame, so the next paint is a - # full erase_down + redraw that leaves nothing stale behind. - session = TuiSession() - session._app = _fake_app(_filled()) - session._emit("Bob 👋 says hi") - assert session._app.renderer._last_screen is None - # A frame of only width-1 glyphs — plain text, node marks, chart braille — keeps the - # efficient differential paint. - session = TuiSession() - last = _filled() - session._app = _fake_app(last) - session._emit("★ you ▲ repeater ● node ⠿ chart") - assert session._app.renderer._last_screen is last +def test_emoji_row_borders_sit_flush_after_alignment() -> None: + """The ragged-border artifact: a bordered row with an emoji rendered one cell short, so its + right border sat a column in with a blank beside it. Once Rich measures the emoji at one + cell it pads the row to full width and the border is flush — every panel line is exactly the + panel width, whether it carries no emoji, one, or two.""" + from io import StringIO + + from rich.cells import cell_len + from rich.console import Console + from rich.panel import Panel + from rich.text import Text + + from meshterm.ui.tui.glyphwidth import align_emoji_cell_width + + align_emoji_cell_width() + width = 30 + for message in ["plain ascii row", "Bob 👋 hi", "two 👋 and 🕒 icons"]: + console = Console(width=width, file=StringIO(), color_system=None, highlight=False) + with console.capture() as capture: + console.print(Panel(Text(message), width=width), end="") + for line in capture.get().splitlines(): + assert cell_len(line) == width, f"{line!r} is not flush at {width}" def test_floating_text_prompt_is_a_popup_over_a_blank_base() -> None: