# SPDX-License-Identifier: Apache-2.0 """Reserve-two emoji widths: one rule in each width authority, and each glyph kept whole. :mod:`meshterm.ui.tui.emoji_width` explains how MeshTerm measures a width. :mod:`meshterm.ui.tui.colsnap` places the glyphs. These tests prove that: * Each glyph that a terminal can draw as an emoji has a reserve of two cells, in Rich and in prompt_toolkit. * The marks of the app, the single-cell ranges of Rich, and ordinary text keep their stock widths. * The three measurements of Rich and the measurement of prompt_toolkit agree for each string. Thus the segment splitter of Rich does not walk past a cut that it can never meet. * The source of the app draws no text-default emoji that its vocabulary does not name. * A glyph that has several codepoints is split, cut, and laid out as the one glyph that it is. :func:`~meshterm.ui.tui.emoji_width.install` patches tables that are global to the process. Thus each test that needs it uses the ``installed`` fixture, which puts each table and cache back afterwards. """ from __future__ import annotations import ast import pathlib import threading from collections.abc import Iterator import prompt_toolkit.utils as ptu import pytest import rich.cells as cells import rich.segment as segment from prompt_toolkit.utils import get_cwidth from meshterm.ui.tui import emoji_width as ew _ZWJ = "\u200d" # the joiner: it shows that the characters next to it are one glyph _VS16 = "\ufe0f" # the request for the emoji presentation _WAVE = "\U0001f44b" # ๐Ÿ‘‹ emoji presentation by default: the stock tables already give two cells _DISH = "\U0001f4e1" # ๐Ÿ“ก a menu icon, with the same width _CA = "\U0001f1e8\U0001f1e6" # ๐Ÿ‡จ๐Ÿ‡ฆ a flag: two Regional Indicators, one glyph _CN = "\U0001f1e8\U0001f1f3" # ๐Ÿ‡จ๐Ÿ‡ณ a second flag with the same "C" indicator _SUN = "โ˜€" # โ˜€ a text-default emoji, by itself _PLANE = "\U0001f6e9" # ๐Ÿ›ฉ another one. A font can draw it in either way _HEART = "โค" # โค another one _ROAD = "\U0001f6e3" # ๐Ÿ›ฃ text-default also, but one of the marks of the app _WEB = "\U0001f578" # ๐Ÿ•ธ the same _BIN = "\U0001f5d1" # ๐Ÿ—‘ the same _WARN = "โš " # โš  the same _SHRUG = f"\U0001f937{_ZWJ}โ™‚{_VS16}" # the shrug: base, joiner, sign, selector _FAMILY = f"\U0001f468{_ZWJ}\U0001f469{_ZWJ}\U0001f467" # the family: three joined people _THUMB = "\U0001f44d\U0001f3fd" # a thumb and its skin tone, no joiner: it is still one glyph _TECHIE = f"\U0001f468\U0001f3fb{_ZWJ}\U0001f4bb" # the technologist: toned, then joined _KEYCAP = f"1{_VS16}\u20e3" # a keycap: a digit, a selector, and the enclosing mark _FLAG = "\U0001f3f4" # a black flag, the base of a pirate flag _STRANDED = f"{_FLAG}{_ZWJ}" # a pirate flag that the byte limit of a name cut after its joiner #: Strings that each authority must measure in the same way: emoji of each shape, the marks of #: the app, and the scripts and marks that the rule must not change. _SAMPLES = ( "plain ascii", f"Bob {_FAMILY} x", f"{_CA}{_CN}", f"{_SUN}{_VS16}{_SHRUG}", f"{_SUN} bare, {_PLANE} bare, {_HEART} bare", f"a{_ZWJ}b", f"{_STRANDED} x", f"{_KEYCAP}!", f"โ”‚ {_WARN}{_VS16} warn โ”‚ {_WARN} mark โ”‚", f"Tech {_TECHIE} x {_THUMB}{_THUMB}", "ๆผขๅญ— names", "เคท\u094d\u200dเค•", # a Devanagari conjunct: a joiner between letters, not pictographs f"{_CA[0]} a lone indicator", "e\u0301 a combining accent", f"{_ROAD}{_WEB}{_BIN} โ†” โ†•", ) @pytest.fixture def installed() -> Iterator[None]: """Install the reserve-two rule for one test, and put each table and cache back after it.""" from meshterm.ui.tui import colsnap from meshterm.ui.tui.render import _ANSI_CACHE saved = ( cells._cell_len, cells.get_character_cell_size, cells.split_graphemes, segment.get_character_cell_size, ptu._CHAR_SIZES_CACHE, ew._INSTALLED, ew._CLUSTERS, ) ew._INSTALLED = False ew.install() try: yield finally: ( cells._cell_len, cells.get_character_cell_size, cells.split_graphemes, segment.get_character_cell_size, ptu._CHAR_SIZES_CACHE, ew._INSTALLED, ew._CLUSTERS, ) = saved cells.cached_cell_len.cache_clear() _ANSI_CACHE.clear() colsnap._CACHE.clear() def test_whatever_may_be_drawn_as_an_emoji_is_reserved_two_cells(installed) -> None: # noqa: ANN001 """Each shape of emoji measures two in both authorities, whatever the stock tables said. The stock answers were not consistent. prompt_toolkit gave a flag four cells, a sun that asks for the emoji presentation one cell, a toned thumb four cells, and a family six cells. An answer below two is not safe for a glyph that a font can draw in two cells, because the pin after the glyph would overwrite its right half. """ for glyph in ( _WAVE, _DISH, _SUN, f"{_SUN}{_VS16}", _PLANE, f"{_PLANE}{_VS16}", _HEART, _CA, _CA[0], _SHRUG, _FAMILY, _THUMB, _TECHIE, _KEYCAP, f"{_WARN}{_VS16}", # a mark that asks for the emoji presentation is an emoji ): assert cells.cell_len(glyph) == 2, f"Rich: {glyph!r}" assert get_cwidth(glyph) == 2, f"prompt_toolkit: {glyph!r}" def test_text_the_app_draws_keeps_its_stock_width(installed) -> None: # noqa: ANN001 """The rule reserves nothing without a reason: text, chrome, and the marks of the app. The marks are also text-default emoji. But they are the vocabulary of the app, and the terminals that run the app draw them in one cell. Thus they keep the one cell that each screen uses in its layout. Rich measures its single-cell ranges with a fast path that no patch reaches, so all other measurements must agree with that path. """ for text, width in ( ("a", 1), ("#", 1), ("1", 1), ("รฉ", 1), ("ๆผข", 2), ("โ”€", 1), ("โ ฟ", 1), ("ยฉ", 1), ("โ–ถ", 1), (_WARN, 1), (_ROAD, 1), (_WEB, 1), (_BIN, 1), ("โ†•", 1), ("โ†”", 1), ("โ†ฉ", 1), ("โŒจ", 1), ("โš™", 1), ("a\U0001f3fd", 1), # a skin tone after a letter leaves the letter a letter ): assert cells.cell_len(text) == width, f"Rich: {text!r}" assert get_cwidth(text) == width, f"prompt_toolkit: {text!r}" def test_every_authority_measures_every_string_the_same(installed) -> None: # noqa: ANN001 """The whole-string width of Rich, its grapheme spans, and the cache of prompt_toolkit agree. Also, each single character has the same width in all of them. These are the four places that read a width. MeshTerm lays out a row correctly only if they give one answer. """ for text in _SAMPLES: spans, total = cells.split_graphemes(text) assert cells.cell_len(text) == total == get_cwidth(text), text assert total == sum(width for _start, _end, width in spans), text assert spans[0][0] == 0 and spans[-1][1] == len(text), f"spans cover {text!r}" for char in text: size = cells.get_character_cell_size(char) assert size == segment.get_character_cell_size(char) == get_cwidth(char), repr(char) def test_no_prefix_outgrows_the_character_that_ends_it(installed) -> None: # noqa: ANN001 """The invariant that the segment splitter of Rich needs to end, then the splitter itself. ``Segment.split_cells`` steps through a string one codepoint at a time until a whole-string width reaches the cut. It steps over a two-cell character only where that character measures two by itself. Suppose a prefix grew by two at a character that measures one. The cut would be a place that the walk steps past in both directions for ever. Thus the patch changes the character width and the string width. This test checks the invariant before it uses the splitter. """ for text in _SAMPLES: for index, char in enumerate(text): grew = cells.cell_len(text[: index + 1]) - cells.cell_len(text[:index]) assert 0 <= grew <= 2, f"{text!r} at {index}" if grew == 2: assert cells.get_character_cell_size(char) == 2, f"{text!r} at {index}" def split_everywhere() -> None: for text in _SAMPLES: for cut in range(cells.cell_len(text) + 1): segment.Segment(text).split_cells(cut) worker = threading.Thread(target=split_everywhere, daemon=True) worker.start() worker.join(timeout=20) assert not worker.is_alive(), "a segment split never met its cut" def test_a_stranded_joiner_leaves_the_next_character_its_cell(installed) -> None: # noqa: ANN001 """A joiner with no pictograph after it joins nothing, in both authorities. MeshCore cuts a name at a byte limit. Thus a name that ends with a pirate flag arrives as the black flag and a bare joiner, followed by the padding of the lane. The stock loop of Rich folded the padding into the flag. Thus the name lane measured one cell less than the terminal drew. """ padded = f"{_STRANDED} x" # the two cells of the flag, two of padding, then the next lane assert cells.cell_len(padded) == 5 and get_cwidth(padded) == 5 # A real sequence still joins: the male sign of the shrug is a pictograph, not a space. assert cells.cell_len(f"{_SHRUG} x") == 4 and get_cwidth(f"{_SHRUG} x") == 4 def test_install_happens_once(installed) -> None: # noqa: ANN001 """A second call does nothing: the tables that the first call patched do not change.""" patched = (cells._cell_len, ptu._CHAR_SIZES_CACHE) ew.install() assert (cells._cell_len, ptu._CHAR_SIZES_CACHE) == patched def test_the_app_draws_no_text_default_emoji_its_vocabulary_does_not_name() -> None: """Each text-default emoji that the package draws is one of its marks, and each mark is used. This test keeps :data:`~meshterm.ui.tui.emoji_width._APP_TEXT_MARKS` a closed vocabulary. Without it, the list would depend on care. A new one-cell mark that a developer adds to a screen, and does not name in the list, fails here. Without the test, the mark would get a blank cell next to it with no warning. The test skips prose, because MeshTerm reads docstrings and bare strings and does not draw them. It also skips the module that defines the list. It also skips the vocabulary of reactions, which is content that other apps send. """ from rich._unicode_data import load text_default = { char for char in load("auto").narrow_to_wide if not char.isascii() and char not in cells._SINGLE_CELLS } # Inputs to the substitution table of the PicoCalc, which maps them to glyphs that its # font has. A terminal that shows emoji never draws them, so they are not marks. The # heart is the key of an emoticon (theme._EMOTICONS), which only a handheld uses. fold_inputs = {"โ†ช", "โœ”", "โœ–", "โค"} package = pathlib.Path(ew.__file__).parents[2] drawn: set[str] = set() for path in package.rglob("*.py"): # The reactions module holds the emoji that other apps send (the defaults of # MeshCore One, the table of meshcore-open). These are content, not marks of the # app, and each one that is text-default carries its variation selector. if path.name in ("emoji_width.py", "reactions.py"): continue tree = ast.parse(path.read_text(encoding="utf-8")) prose = { id(node.value) for node in ast.walk(tree) if isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) } for node in ast.walk(tree): if isinstance(node, ast.Constant) and isinstance(node.value, str): if id(node) not in prose: drawn |= set(node.value) & text_default assert drawn - fold_inputs <= ew._APP_TEXT_MARKS, ( f"name these in _APP_TEXT_MARKS: {sorted(drawn - fold_inputs - ew._APP_TEXT_MARKS)}" ) assert ew._APP_TEXT_MARKS <= drawn, ( f"no longer drawn anywhere: {sorted(ew._APP_TEXT_MARKS - drawn)}" ) def test_clusters_split_between_glyphs_and_never_inside_one() -> None: """The unit where a lane can cut and by which a width is measured: one entry for each glyph. A joined run, the indicator pair of a flag, a toned base, and a keycap each return whole, because each is one glyph. A joiner that joins nothing is a separate entry (refer to :func:`~meshterm.ui.tui.emoji_width._joins`), and the letters around a joiner are also separate entries. """ text = f"a{_FAMILY}{_CA}{_THUMB}{_KEYCAP}b" assert list(ew.clusters(text)) == ["a", _FAMILY, _CA, _THUMB, _KEYCAP, "b"] # Two flags in a row make two pairs, one at a time, and do not run together into four. assert list(ew.clusters(_CA + _CN)) == [_CA, _CN] # A name that the byte limit cut just after a joiner: the joiner is not part of the flag. assert list(ew.clusters(_STRANDED)) == [_FLAG, _ZWJ] # A joiner between letters joins nothing that a font draws as one emoji. assert list(ew.clusters(f"a{_ZWJ}b")) == ["a", _ZWJ, "b"] def test_cut_cells_keeps_every_glyph_whole_and_strands_no_joiner() -> None: """The cut of a lane is between glyphs, so MeshTerm measures nothing that it does not draw.""" assert ew.cut_cells(f"Bob {_FAMILY}", 6) == f"Bob {_FAMILY}" # it fits, so it stays assert ew.cut_cells(f"Bob {_FAMILY}", 5) == "Bob " # it does not fit: the cut removes it whole assert ew.cut_cells(_CA, 1) == "" # half of a flag is a letter, not half of a glyph assert ew.cut_cells(_STRANDED, 4) == _FLAG # the joiner does not stay at the end of the cut assert ew.cut_cells("plain name", 5) == "plain" def test_drawable_folds_only_what_no_terminal_can_draw() -> None: """A newline, an escape, or a bidi override in an advert name changes to a space. None of them is a question of width. They are text that must not reach the terminal at all. The joiner is also in the format class, but the function keeps it, because it holds an emoji together. """ assert ew.drawable("line\nbreak") == "line break" assert ew.drawable("esc\x1b[31mape") == "esc [31mape" assert ew.drawable("flip\u202eme") == "flip me" assert ew.drawable(f"Bob {_FAMILY}{_THUMB}") == f"Bob {_FAMILY}{_THUMB}" def test_join_clusters_merges_every_multi_codepoint_glyph() -> None: """The fragment merge puts each glyph that has several codepoints into one fragment. The ANSI text of prompt_toolkit arrives with one codepoint in each fragment. Thus a glyph is a run that the merge must gather. The merge now gathers each such run, not only the joined runs. A mark that measures one cell and has a selector after it must not fold into a one-cell slot. """ def line(text: str) -> list: return [("", char) for char in text] def texts(fragments: list) -> list: return [text for _style, text in fragments] # A line with nothing to merge is returned as it is: the same object, not a copy. plain = line(f"hi {_DISH} {_WARN}") assert ew._join_clusters(plain) is plain assert texts(ew._join_clusters(line(f"|{_SHRUG}|"))) == ["|", _SHRUG, "|"] assert texts(ew._join_clusters(line(_FAMILY))) == [_FAMILY] burning = f"โค{_VS16}{_ZWJ}\U0001f525" # โค\ufe0f\u200d๐Ÿ”ฅ a selector on the base, then a join assert texts(ew._join_clusters(line(burning))) == [burning] # A selector pair, a toned emoji, and a keycap are one glyph each. The merge joins them # like the other glyphs. assert texts(ew._join_clusters(line(f"{_SUN}{_VS16}{_SHRUG}"))) == [f"{_SUN}{_VS16}", _SHRUG] assert texts(ew._join_clusters(line(f"{_WARN}{_VS16}|"))) == [f"{_WARN}{_VS16}", "|"] assert texts(ew._join_clusters(line(f"{_THUMB}{_KEYCAP}"))) == [_THUMB, _KEYCAP] # The two indicators of a flag are one glyph, and the code writes them as one. If a cursor # pin were between the halves, the terminal would draw two letters instead of a flag. assert texts(ew._join_clusters(line(f"|{_CA}|"))) == ["|", _CA, "|"] assert texts(ew._join_clusters(line(f"{_CA}{_CN}"))) == [_CA, _CN] # A single indicator is not a pair, and a trailing or stranded joiner joins nothing. assert texts(ew._join_clusters(line(f"{_CA[0]} x"))) == [_CA[0], " ", "x"] assert texts(ew._join_clusters(line(f"a{_ZWJ}"))) == ["a", _ZWJ] assert texts(ew._join_clusters(line(f"{_STRANDED} x"))) == [_FLAG, _ZWJ, " ", "x"] # A merged fragment iterates as the whole glyph. Thus prompt_toolkit builds one Char for # it instead of one Char for each codepoint. (merged,) = texts(ew._join_clusters(line(_SHRUG))) assert list(merged) == [_SHRUG] and merged == _SHRUG def test_cluster_control_lays_each_glyph_into_a_single_screen_cell(installed) -> None: # noqa: ANN001 """The delivery half: one two-cell ``Char`` for each glyph, with each codepoint still written. A correct measurement of a glyph is not enough. prompt_toolkit lays out one codepoint at a time. Thus an unmerged shrug puts its male sign in a separate cell, and a warning mark with a selector becomes a two-cell glyph in a one-cell slot. When the glyphs are merged, each is one ``Char`` that is two cells wide, and the text after it is where the pins will put it. """ from prompt_toolkit.application import Application from prompt_toolkit.application.current import set_app from prompt_toolkit.formatted_text import ANSI from prompt_toolkit.input import DummyInput from prompt_toolkit.layout import Layout, Window from prompt_toolkit.layout.mouse_handlers import MouseHandlers from prompt_toolkit.layout.screen import Screen, WritePosition from prompt_toolkit.output import DummyOutput warn = f"{_WARN}{_VS16}" text = f"|{_SHRUG}|{_FAMILY}|{warn}|end" window = Window(ew.ClusterTextControl(lambda: ANSI(text)), always_hide_cursor=True) app = Application(layout=Layout(window), input=DummyInput(), output=DummyOutput()) with set_app(app): screen = Screen(default_char=None, initial_width=40, initial_height=1) window.write_to_screen(screen, MouseHandlers(), WritePosition(0, 0, 40, 1), "", False, None) row = screen.data_buffer[0] assert (row[1].char, row[1].width) == (_SHRUG, 2) assert (row[4].char, row[4].width) == (_FAMILY, 2) assert (row[7].char, row[7].width) == (warn, 2) assert "".join(row[x].char for x in range(9, 13)) == "|end" # Nothing was lost: the row still spells the source exactly. assert "".join(row[x].char for x in range(40)).rstrip() == text