diff --git a/meshterm/ui/tui/glyphwidth.py b/meshterm/ui/tui/glyphwidth.py new file mode 100644 index 0000000..f7ceb6c --- /dev/null +++ b/meshterm/ui/tui/glyphwidth.py @@ -0,0 +1,131 @@ +"""Align the app's glyph-width model with how the terminal actually draws emoji. + +Terminal rendering is a three-party agreement about how many cells each glyph occupies, and +this app has three parties: **Rich** lays out every line (wrapping, padding, and — the part +that bites — the position of a panel's right border), **prompt_toolkit** diffs one frame +against the last and steps a *relative* cursor by its own width model, and the **terminal** +draws the result. Rich and prompt_toolkit both measure an emoji (``👋``, ``🕒``, ``⚡``) as +two cells, via ``wcwidth``. The terminals this app runs on draw it in a *single* cell. + +That one-cell disagreement is the whole bug: + +* prompt_toolkit reserves two cells but the terminal advances one, so from the emoji rightward + the diff's cursor model is a column ahead of the terminal. On the next paint, a cell to the + emoji's right that pt repositions to lands a column early, and the stale cell it meant to + overwrite lingers — the "characters that don't get cleared" smear. +* Rich sizes the row assuming two cells, so a bordered line with one emoji renders one cell + short and its right border sits a column in from the frame edge — a ragged notch with a + blank beside it. + +Both vanish the moment all three parties agree. We can't change the terminal, so we teach the +two width authorities the terminal's truth: an emoji is one cell. Rich then pads the row to a +full-width, flush border, and prompt_toolkit's relative cursor tracks the terminal exactly, so +ordinary differential painting stays synced and nothing lingers — no whole-frame repaint, no +compositing tricks. CJK and fullwidth glyphs, which terminals *do* draw two cells wide, are +left at two; node-type marks and braille are already one cell everywhere and are untouched. + +This is the sibling of :class:`~meshterm.ui.tui.session._WidthExtendedOutput`: both correct a +width assumption the terminal breaks, in the one place the whole pipeline reads it. Gated by +``MESHTERM_NARROW_EMOJI`` (default on); set it to ``0`` on a terminal that genuinely draws +emoji two cells wide, where the alignment would instead *introduce* a one-cell error. +""" + +from __future__ import annotations + +import os + +from wcwidth import wcwidth + +#: Whether to align the width model to a one-cell emoji (see the module docstring). Default on; +#: ``MESHTERM_NARROW_EMOJI=0`` opts out for a terminal that draws emoji two cells wide. +_ALIGN_NARROW_EMOJI = os.environ.get("MESHTERM_NARROW_EMOJI", "1") != "0" + + +def _is_cjk_or_fullwidth(cp: int) -> bool: + """Whether ``cp`` is a codepoint terminals reliably draw *two* cells wide. + + These are the East Asian Wide/Fullwidth blocks — CJK ideographs and their radicals, the + kana and Hangul syllables, and the fullwidth ASCII forms. They share ``wcwidth``'s value of + two with emoji, so width alone can't tell them apart; the split is by block. Everything here + is left at two cells, because the terminal draws it at two — only the *rest* of the width-2 + range (the emoji) is the part the terminal narrows. + """ + return ( + 0x1100 <= cp <= 0x115F # Hangul Jamo + or 0x2E80 <= cp <= 0x9FFF # CJK radicals, Kangxi, kana, CJK Unified Ideographs + or 0xA000 <= cp <= 0xA4CF # Yi syllables + or 0xAC00 <= cp <= 0xD7A3 # Hangul syllables + or 0xF900 <= cp <= 0xFAFF # CJK compatibility ideographs + or 0xFE30 <= cp <= 0xFE4F # CJK compatibility forms + or 0xFF00 <= cp <= 0xFF60 # Fullwidth forms + or 0xFFE0 <= cp <= 0xFFE6 # Fullwidth signs + or 0x20000 <= cp <= 0x3FFFD # CJK Unified Ideographs Extension B and beyond + ) + + +def is_narrow_emoji(char: str) -> bool: + """Whether ``char`` is a glyph ``wcwidth`` calls two cells but the terminal draws in one. + + True for an emoji or pictographic symbol — a single codepoint that ``wcwidth`` measures at + two and that is not in a CJK/fullwidth block. The ``>= 0x1100`` guard skips the ASCII/Latin + bulk before the width lookup; ``len == 1`` keeps this to lone codepoints (multi-codepoint + grapheme clusters — flags, ZWJ sequences — are left to their parts). + """ + if len(char) != 1: + return False + cp = ord(char) + return cp >= 0x1100 and wcwidth(char) == 2 and not _is_cjk_or_fullwidth(cp) + + +#: Set once :func:`align_emoji_cell_width` has patched the width authorities, so repeated calls +#: (one per :class:`~meshterm.ui.tui.session.TuiSession`) patch the global state exactly once. +_applied = False + + +def align_emoji_cell_width() -> None: + """Teach Rich and prompt_toolkit that an emoji is one cell, matching the terminal. + + Idempotent and gated by :data:`_ALIGN_NARROW_EMOJI`. Patches the two width authorities the + render pipeline reads: + + * **prompt_toolkit** — ``prompt_toolkit.utils._CHAR_SIZES_CACHE``, the dict behind + ``get_cwidth`` and therefore every ``Char.width``. We swap in a subclass whose + ``__missing__`` returns one for a narrow emoji and otherwise defers to the original + ``wcwidth`` logic, so the diff's cursor model matches the terminal. + * **Rich** — ``rich.cells.get_character_cell_size``, resolved as a module global by + ``_cell_len`` on every measurement. We wrap it to return one for a narrow emoji, then + clear ``cached_cell_len``'s cache so no pre-patch measurement survives, so Rich lays out + (and borders) each row at the terminal's true width. + + Called at :class:`~meshterm.ui.tui.session.TuiSession` construction, before any frame is + drawn, so no stale two-cell ``Char`` or cached measurement is ever painted. + """ + global _applied + if _applied or not _ALIGN_NARROW_EMOJI: + return + _applied = True + + import prompt_toolkit.utils as ptu + + base_cache_cls = type(ptu._CHAR_SIZES_CACHE) + + class _NarrowEmojiCache(base_cache_cls): # type: ignore[valid-type, misc] + def __missing__(self, string: str) -> int: + if is_narrow_emoji(string): + self[string] = 1 + return 1 + return super().__missing__(string) + + ptu._CHAR_SIZES_CACHE = _NarrowEmojiCache() + + import rich.cells as rc + + _measure = rc.get_character_cell_size + + def _narrow_emoji_cell_size(character: str, unicode_version: str = "auto") -> int: + if is_narrow_emoji(character): + return 1 + return _measure(character, unicode_version) + + rc.get_character_cell_size = _narrow_emoji_cell_size + rc.cached_cell_len.cache_clear() diff --git a/meshterm/ui/tui/session.py b/meshterm/ui/tui/session.py index d190afa..60ac727 100644 --- a/meshterm/ui/tui/session.py +++ b/meshterm/ui/tui/session.py @@ -24,11 +24,11 @@ from prompt_toolkit.key_binding import KeyBindings from prompt_toolkit.keys import Keys from prompt_toolkit.layout import ConditionalContainer, Float, FloatContainer, Layout, Window from prompt_toolkit.layout.controls import FormattedTextControl -from prompt_toolkit.utils import get_cwidth from rich.console import RenderableType from rich.text import Text from . import frame +from .glyphwidth import align_emoji_cell_width from .overlay import BusyOverlay from .progress import TuiProgress from .prompt import ( @@ -91,18 +91,6 @@ _RECLAIM_LAST_COLUMN = os.environ.get("MESHTERM_FULL_WIDTH", "1") != "0" #: real nesting — a tool's list, an item's detail popup, and a confirm over that is only three. _MAX_DIALOG_LAYERS = 8 -def _has_wide_glyph(text: str) -> bool: - """Whether ``text`` holds a glyph prompt_toolkit reserves two cells for. - - A width-2 glyph is where the terminal and the renderer can disagree: an emoji (``👋``) - that the terminal draws in a *single* cell is the common case here — pt reserves two, - the terminal advances one, and from that point the row's cursor model is off. Node-type - marks (``▲●■``) and chart braille are width-1 everywhere, so they never trip this. The - ``>= 0x1100`` guard skips the ASCII/Latin bulk of a frame before the width lookup, which - matters because this runs over the whole composed frame on every repaint. - """ - return any(ord(ch) >= 0x1100 and get_cwidth(ch) == 2 for ch in text) - class _WidthExtendedOutput: """A prompt_toolkit ``Output`` proxy that reports one extra terminal column. @@ -173,6 +161,9 @@ class TuiSession: output: Optional prompt_toolkit output to render to (tests use a dummy); defaults to the real terminal. """ + # Align the width model with how the terminal draws emoji (one cell, not two) before + # anything renders, so Rich borders sit flush and pt's differential cursor stays synced. + align_emoji_cell_width() self._stack: list[Screen] = [] self._header = header or (lambda cols: Text("MeshTerm", style="brand")) self._app: Optional[Application] = None @@ -1006,28 +997,6 @@ class TuiSession: """Whether any dialog floats over the background this frame.""" return bool(self._float_layers()) - def _emit(self, text: str) -> ANSI: - """Wrap a composed frame as prompt_toolkit :class:`ANSI`, forcing a full repaint - when it holds a glyph the terminal may draw narrower than pt reserves for it. - - prompt_toolkit paints differentially: it rewrites only the cells that changed since - the last frame, and it steps the cursor *relative* to its own width model. That is - sound only while every glyph is one cell wide. A width-2 glyph the terminal draws in - a single cell (an emoji in a chat line, a menu icon) leaves the terminal's cursor one - column left of pt's model for the rest of that row; on the next paint, any cell to the - emoji's right that pt repositions to — because the emoji itself didn't change and was - skipped — lands one column off, and the stale cell it should have overwritten lingers. - So when a frame carries such a glyph, drop pt's cached frame (:meth:`_invalidate_last_frame`) - so the next paint is a full ``erase_down`` + redraw: every cell is rewritten - contiguously, letting the terminal's own cursor advance keep the row aligned, and - nothing stale survives. Frames with only width-1 glyphs keep the fast differential - paint. This is checked per frame, so it covers a scroll, a resize, or a timer tick - alike — wherever the glyph is (a full-frame chat, a floating dialog, the overlay). - """ - if _has_wide_glyph(text): - self._invalidate_last_frame() - return ANSI(text) - def _render_base(self) -> ANSI: """Render the persistent frame around the background screen.""" cols, rows = self._size() @@ -1040,9 +1009,9 @@ class TuiSession: # A chromeless base (the startup splash) forgoes the header/footer bars and is # centered under its banner instead of stretched across the terminal. if not base.chrome: - return self._emit(frame.compose_startup(base, cols, rows)) + return ANSI(frame.compose_startup(base, cols, rows)) footer = self.top.footer_hint if self.top else base.footer_hint - return self._emit(frame.compose_base(self._header(cols), base, footer, cols, rows)) + return ANSI(frame.compose_base(self._header(cols), base, footer, cols, rows)) def _render_float_layer(self, index: int) -> ANSI: """Render the ``index``-th floating dialog (bottom-to-top) as a centered box.""" @@ -1050,7 +1019,7 @@ class TuiSession: if index >= len(layers): return ANSI("") cols, rows = self._size() - return self._emit(frame.compose_dialog(layers[index], cols, rows)) + return ANSI(frame.compose_dialog(layers[index], cols, rows)) def _overlay_visible(self) -> bool: """Whether the busy overlay should be painted this frame. @@ -1067,7 +1036,7 @@ class TuiSession: """Render the busy overlay's skeleton card (only when :meth:`_overlay_visible`).""" if self._overlay is None: return ANSI("") - return self._emit(self._overlay.render()) + return ANSI(self._overlay.render()) # --- input --------------------------------------------------------------- @@ -1099,9 +1068,10 @@ class TuiSession: def _dispatch(self, action: str, data: str = "") -> None: """Forward an action to the top screen and repaint. - The repaint keeps prompt_toolkit's fast differential paint; a frame carrying a glyph - the terminal may draw narrower than pt reserves for it (an emoji) upgrades itself to a - full repaint at compose time — see :meth:`_emit`. + The repaint is prompt_toolkit's efficient differential paint. That stays correct across + emoji because the width model is aligned to the terminal at startup (one cell per emoji, + via :func:`~meshterm.ui.tui.glyphwidth.align_emoji_cell_width`), so pt's relative cursor + never drifts and no stale cells are left behind. """ top = self.top if top is not None: diff --git a/tests/test_tui.py b/tests/test_tui.py index 44c413e..2ba8293 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -1480,63 +1480,65 @@ def test_session_background_is_the_topmost_full_frame_screen() -> None: assert session._float_layers() == [dialog] -def test_wide_glyph_detection_flags_emoji_not_marks() -> None: - """The desync only ever comes from a width-2 glyph the terminal may draw narrower — an - emoji. Node-type marks, status marks and chart braille are width-1 everywhere, so they - must not trip the check (that they seemed to was the earlier misdiagnosis).""" - from meshterm.ui.tui.session import _has_wide_glyph +def test_narrow_emoji_predicate_splits_emoji_from_cjk_and_marks() -> None: + """The width model is aligned to the terminal for exactly one class of glyph: an emoji the + terminal draws in one cell while wcwidth calls it two. CJK/fullwidth glyphs (drawn two cells + wide) and the width-1 marks and braille must stay out of it — that split is what keeps the + alignment from breaking CJK layout or double-counting a node mark.""" + from meshterm.ui.tui.glyphwidth import is_narrow_emoji - assert _has_wide_glyph("👋") - assert _has_wide_glyph("Bob 👋 waved") - assert _has_wide_glyph("clock 🕒 sync") - assert _has_wide_glyph("⚡ explore") # a width-2 icon still counts - assert not _has_wide_glyph("plain ascii row") - assert not _has_wide_glyph("★ ▲ ● ■ ◉ ○") # node-type marks: width 1 - assert not _has_wide_glyph("⠿⣿⡇ chart") # braille: width 1 - assert not _has_wide_glyph("✓ ✗ ⚠ … done") # status marks: width 1 + assert is_narrow_emoji("👋") # waving hand — the reported case + assert is_narrow_emoji("🕒") + assert is_narrow_emoji("⚡") # a BMP pictograph icon, still width-2 + assert not is_narrow_emoji("日") # CJK: genuinely two cells, leave it + assert not is_narrow_emoji("A") # fullwidth Latin: two cells + assert not is_narrow_emoji("▲") # node-type mark: one cell everywhere + assert not is_narrow_emoji("⠿") # braille: one cell + assert not is_narrow_emoji("A") # plain ascii + assert not is_narrow_emoji("👋🏽") # multi-codepoint cluster: left to its parts -def test_a_wide_glyph_frame_upgrades_to_a_full_repaint() -> None: - """prompt_toolkit paints differentially with a *relative* cursor — sound only while every - glyph is one cell. A width-2 glyph the terminal draws in one cell (an emoji in a chat line) - leaves the row's cursor model off; a later paint that skips the unchanged emoji then strands - stale cells to its right. So a composed frame carrying such a glyph drops pt's cached frame, - upgrading the next paint to a full erase_down + redraw. A frame of only width-1 glyphs keeps - the fast differential paint — this holds wherever the glyph is, floating dialog or not.""" - import types +def test_aligning_emoji_width_agrees_with_the_terminal() -> None: + """Aligning teaches both width authorities the terminal's truth — an emoji is one cell. + prompt_toolkit's get_cwidth (behind every Char.width, so the differential cursor) and Rich's + cell_len (behind every layout and border) both then measure an emoji at one, while CJK stays + two and marks stay one. With all three parties agreeing, the differential paint never drifts + and no whole-frame repaint is needed.""" + from prompt_toolkit.utils import get_cwidth + from rich.cells import cell_len - from prompt_toolkit.data_structures import Size - from prompt_toolkit.layout.screen import Char - from prompt_toolkit.layout.screen import Screen as PtScreen + from meshterm.ui.tui.glyphwidth import align_emoji_cell_width - def _fake_app(last: PtScreen) -> types.SimpleNamespace: - return types.SimpleNamespace( - renderer=types.SimpleNamespace(_last_screen=last), - output=types.SimpleNamespace(get_size=lambda: Size(rows=10, columns=60)), - invalidate=lambda: None, - ) + align_emoji_cell_width() # idempotent; also applied by every TuiSession - def _filled() -> PtScreen: - screen = PtScreen() - for row in range(3): - for x in range(10): - screen.data_buffer[row][x] = Char("A") - return screen + assert get_cwidth("👋") == 1 and cell_len("👋") == 1 + assert get_cwidth("🕒") == 1 and cell_len("🕒") == 1 + assert get_cwidth("日") == 2 and cell_len("日") == 2 # CJK unchanged + assert get_cwidth("▲") == 1 and cell_len("▲") == 1 # mark unchanged - # An emoji anywhere in the composed frame drops the remembered frame, so the next paint is a - # full erase_down + redraw that leaves nothing stale behind. - session = TuiSession() - session._app = _fake_app(_filled()) - session._emit("Bob 👋 says hi") - assert session._app.renderer._last_screen is None - # A frame of only width-1 glyphs — plain text, node marks, chart braille — keeps the - # efficient differential paint. - session = TuiSession() - last = _filled() - session._app = _fake_app(last) - session._emit("★ you ▲ repeater ● node ⠿ chart") - assert session._app.renderer._last_screen is last +def test_emoji_row_borders_sit_flush_after_alignment() -> None: + """The ragged-border artifact: a bordered row with an emoji rendered one cell short, so its + right border sat a column in with a blank beside it. Once Rich measures the emoji at one + cell it pads the row to full width and the border is flush — every panel line is exactly the + panel width, whether it carries no emoji, one, or two.""" + from io import StringIO + + from rich.cells import cell_len + from rich.console import Console + from rich.panel import Panel + from rich.text import Text + + from meshterm.ui.tui.glyphwidth import align_emoji_cell_width + + align_emoji_cell_width() + width = 30 + for message in ["plain ascii row", "Bob 👋 hi", "two 👋 and 🕒 icons"]: + console = Console(width=width, file=StringIO(), color_system=None, highlight=False) + with console.capture() as capture: + console.print(Panel(Text(message), width=width), end="") + for line in capture.get().splitlines(): + assert cell_len(line) == width, f"{line!r} is not flush at {width}" def test_floating_text_prompt_is_a_popup_over_a_blank_base() -> None: