From 077f8d6a1b749adab98addb88ce53edf5ec5cacc Mon Sep 17 00:00:00 2001 From: Jean-Pierre Martineau Date: Sun, 9 Aug 2026 02:43:47 -0400 Subject: [PATCH] Mesh walk: the prefix settle was scanning the whole graph, per merge Opening the mesh walk took 4.9 s, and 4.2 s of it was build_topology. coalesce_prefixes folds a short hop hash onto the longer id it abbreviates. It restarts the whole pass after each merge; each pass asks _extensions once per node; and _extensions scanned every node in the graph. On this device - 3053 stored packet paths, 1145 links, ~80 merges to settle - that came to 24 million len calls and 7.6 million startswith per build. Ids sharing a prefix are contiguous once sorted, so the extensions of a short id are a range, not a search: bisect to its start and stop at the first id that no longer carries the prefix. build_topology 4.22 -> 1.23 s with real contacts, and the graph is byte-identical - 1133 links, 345 nodes, both dumped and diffed. On the console the open is 4.94 -> 1.95 s. Contacts' 1.3 s open turns out not to be ours at all: heard_nodes 23 ms, the screen 131 ms, first render 43 ms. The rest is the first devstate read over the UART, already cached for every later open - a warm one is 230 ms. Warming devstate at connect is the fix, and it is now first in the plan. The harness needed two things to measure any of this, both folded back into the skill. An open is not a repaint - it paints the menu closing, falls silent while it loads, then paints the new screen - so quiescence stopped the clock on the first burst and reported the walk opening in 124 ms. Widening the silence window does not help either: past a second the 2 s header tick lands inside every measurement. `expect TEXT` watches /dev/vcs1 until the screen is actually there. Its needle has to be unique to the destination, because every main-menu row carries its own screen's name - `expect Mesh walk` matched the menu and made a 2.3x win read as no change at all. Co-Authored-By: Claude Fable 5 --- .claude/skills/profile-picocalc/SKILL.md | 24 +++++++ .../profile-picocalc/scripts/drive_console.py | 50 +++++++++++++- .../profile-picocalc/tours/nav_opens.txt | 47 +++++++++++++ meshterm/services/topology.py | 33 ++++++--- tests/test_topology.py | 67 +++++++++++++++++++ 5 files changed, 210 insertions(+), 11 deletions(-) create mode 100644 .claude/skills/profile-picocalc/tours/nav_opens.txt diff --git a/.claude/skills/profile-picocalc/SKILL.md b/.claude/skills/profile-picocalc/SKILL.md index 9cc125a..f2095c2 100644 --- a/.claude/skills/profile-picocalc/SKILL.md +++ b/.claude/skills/profile-picocalc/SKILL.md @@ -48,6 +48,30 @@ Tour scripts are one step per line: `down 8`, `enter`, `text:y`, `sleep 3`, `tours/` has `nav_all.txt` (every page), `nav_ab.txt` (short, for A/B runs) and `nav_map.txt`. +## Timing a screen *open* — use `expect`, not silence + +A repaint is one burst of output, so "silent for 45 ms" cleanly ends it. An **open** is not: +it paints the menu closing, falls silent for however long the tool takes to load and build, +then paints the new screen. At the repaint threshold the clock stops on that first burst and +reports an open that never happened (the mesh walk read as 124 ms when it truly took 4.9 s). +Widening the window with `--quiet-ms` does not rescue it either — past about a second the +2 s header tick lands inside every measurement and inflates all of them. + +So watch the panel instead: + +``` +label: open-walk +enter +expect links +``` + +`expect TEXT` polls `/dev/vcs1` until `TEXT` is on screen and reports the time since the +last key. **The needle must be unique to the destination screen.** Every main-menu row +carries its screen's name, so `expect Mesh walk` and `expect Contacts` match the *menu* and +return instantly — a silent false pass that made a 2.3× improvement look like no change. +Use something only the opened screen draws: `links` for the walk (`… 334 nodes · 1145 links`), +` known` for Contacts (`Contacts · 141 known`), `mesh overview` for the Dashboard. + **The app boots to a device picker** — it does not auto-connect. A tour must press `enter` on the splash and then `sleep 25` for the radio, or everything after it measures the splash. Check `shots/*_boot.txt` if the numbers look uniform and wrong. diff --git a/.claude/skills/profile-picocalc/scripts/drive_console.py b/.claude/skills/profile-picocalc/scripts/drive_console.py index 12d5ef4..c08d94d 100644 --- a/.claude/skills/profile-picocalc/scripts/drive_console.py +++ b/.claude/skills/profile-picocalc/scripts/drive_console.py @@ -65,6 +65,11 @@ KEYS = { #: A keystroke is "done" once the app has written nothing for this long. The app writes #: its frame in one burst, so this cleanly separates one repaint from the next without #: charging the gap to the keystroke. +#: +#: Raise it (``--quiet-ms``) when measuring something that *opens* a screen rather than +#: repainting one. An open paints the menu closing, then goes quiet for as long as it takes +#: to load and build, then paints the new screen — at the repaint threshold the clock stops +#: on that first burst and reports a fast open that never happened. QUIET_S = 0.045 #: Give up on a keystroke that never produces output (a key the screen ignores). @@ -85,6 +90,8 @@ def main(): ap.add_argument("--boot-wait", type=float, default=12.0) ap.add_argument("--out", default="") ap.add_argument("--exe", default="$HOME/MeshTerm/.venv/bin/meshterm") + ap.add_argument("--quiet-ms", type=float, default=QUIET_S * 1000, + help="silence that ends a keystroke; raise it to time screen opens") ap.add_argument("--tee", default="", help="also copy the app's output to this file") ap.add_argument("--shots", default="", help="dump the console text after each group here") args = ap.parse_args() @@ -94,6 +101,15 @@ def main(): os.makedirs(args.shots, exist_ok=True) shot_n = [0] + last_key = [None] # when the most recent key was delivered, for `expect` + + def screen_text(): + """The panel's current text, straight out of the console's screen memory.""" + try: + with open("/dev/vcs1", "rb") as fh: + return fh.read(args.rows * args.cols).decode("utf-8", "replace") + except OSError: + return "" def shot(tag): """Read the console's own screen memory — exactly what is on the panel.""" @@ -132,8 +148,12 @@ def main(): set_winsize(fd, args.rows, args.cols) - def drain(quiet=QUIET_S, timeout=TIMEOUT_S, mirror_out=True): + quiet_s = args.quiet_ms / 1000.0 + + def drain(quiet=None, timeout=TIMEOUT_S, mirror_out=True): """Read until the app has been silent for `quiet`; return (bytes, first, last).""" + if quiet is None: + quiet = quiet_s total = 0 t_start = time.perf_counter() t_first = None @@ -198,6 +218,31 @@ def main(): if line.startswith("shot"): shot(line.split(None, 1)[1].strip() if " " in line else label) continue + if line.startswith("expect "): + # Time how long until the panel actually shows the screen we asked for. + # Quiescence cannot answer this: an *open* paints the menu closing, falls + # silent while it loads and builds, then paints the new screen — and widening + # the silence window far enough to span that gap lets the 2 s header tick + # into every other measurement instead. So watch the console itself. + needle = line[7:].strip() + deadline = time.perf_counter() + 30.0 + found = None + while time.perf_counter() < deadline: + drain(quiet=0.005, timeout=0.005) + if needle in screen_text(): + found = time.perf_counter() + break + base = last_key[0] if last_key[0] else time.perf_counter() + if found is None: + print(f"{label:<20} {'expect':<10} {'--':>3} " + f"{'NEVER APPEARED: ' + needle:>40}") + else: + rows_out.append({"label": label, "key": f"expect:{needle}", + "n": 1, "p50": (found - base) * 1000}) + print(f"{label:<20} {'appear':<10} {1:3d} " + f"{(found - base) * 1000:8.1f} {'':>8} {'':>8} {'':>7}" + f" <- {needle}") + continue m = re.match(r"^(\S+?)(?:\s+(\d+))?$", line) name, count = m.group(1), int(m.group(2) or 1) if name.startswith("text:"): @@ -214,11 +259,12 @@ def main(): # Drain anything still in flight from the previous key. Without this a slow # app's tail lands on the next key's clock and reads as a 0.3 ms response. while True: - left, first, _, _ = drain(quiet=0.05, timeout=0.05, mirror_out=True) + left, first, _, _ = drain(timeout=quiet_s, mirror_out=True) if left == 0: break os.write(fd, seq.encode()) t_send = time.perf_counter() + last_key[0] = t_send nbytes, t_first, t_last, _ = drain() if t_first is None: continue diff --git a/.claude/skills/profile-picocalc/tours/nav_opens.txt b/.claude/skills/profile-picocalc/tours/nav_opens.txt new file mode 100644 index 0000000..17ecaf7 --- /dev/null +++ b/.claude/skills/profile-picocalc/tours/nav_opens.txt @@ -0,0 +1,47 @@ +# How long a screen takes to *appear* — the cost no repaint tracer can see. +# +# Each open is timed with `expect`, whose needle must be unique to the destination: +# every main-menu row carries its own screen's name, so `expect Contacts` would match +# the menu and pass instantly. See SKILL.md. + +label: splash +down 1 +up 1 +label: connect +enter +sleep 25 + +# Dashboard (menu row 3) +label: nav +down 3 +label: open-dashboard +enter +expect mesh overview +sleep 2 +esc +sleep 3 + +# Contacts (row 5) — cold pays the device read, warm is served from devstate +label: nav +down 2 +label: open-contacts-cold +enter +expect known +sleep 2 +esc +sleep 3 +label: open-contacts-warm +enter +expect known +sleep 2 +esc +sleep 3 + +# Mesh walk (row 7) — the heaviest open: build_topology over every stored path +label: nav +down 2 +label: open-walk +enter +expect links +sleep 2 +esc diff --git a/meshterm/services/topology.py b/meshterm/services/topology.py index b0b0773..2a5198b 100644 --- a/meshterm/services/topology.py +++ b/meshterm/services/topology.py @@ -29,6 +29,7 @@ so the graph is unit-testable without a device or a database. from __future__ import annotations +import bisect import heapq import math import statistics @@ -464,8 +465,9 @@ class MeshTopology: offered first, so a ``65`` → ``6532`` → ``6532eb`` chain collapses from the tail end inward over successive calls. """ + ordered = sorted(nodes) for short in sorted(nodes, key=len): - exts = self._extensions(short, nodes) + exts = self._extensions(short, ordered) if not exts: continue longest = max(exts, key=len) @@ -474,19 +476,31 @@ class MeshTopology: return None @staticmethod - def _extensions(short: str, nodes: set[str]) -> list[str]: - """Every node id in ``nodes`` that strictly extends the under-specified ``short``. + def _extensions(short: str, ordered: list[str]) -> list[str]: + """Every node id in ``ordered`` that strictly extends the under-specified ``short``. Empty for a full-width canonical id (12 hex is never under-specified) and for a short id nothing in the graph extends. + + ``ordered`` must be the node ids sorted lexicographically, which is what makes this + cheap: every id sharing a prefix sorts into one contiguous run, so the extensions + are a slice found by binary search rather than a scan of the whole graph. That + matters because the caller asks this question once per node, and its own caller + starts the whole pass again after every merge — a mesh with ~1100 links and a couple + of hundred merges to settle turned a linear scan here into ~24 million ``len`` + calls and sixteen seconds of opening the mesh walk. """ if len(short) >= 12: return [] - return [ - other - for other in nodes - if other != short and len(other) > len(short) and other.startswith(short) - ] + out: list[str] = [] + width = len(short) + for i in range(bisect.bisect_left(ordered, short), len(ordered)): + other = ordered[i] + if not other.startswith(short): + break # past the run: nothing further can share the prefix + if len(other) > width: + out.append(other) + return out def _next_corroborated_merge(self, nodes: set[str]) -> Optional[tuple[str, str]]: """The next ambiguous stub the *neighbourhood evidence* resolves, or ``None``. @@ -512,8 +526,9 @@ class MeshTopology: ``f062eb…``) are the same node, so they never split their own vote. """ adjacency = self._adjacency() + ordered = sorted(nodes) for short in sorted(nodes, key=len): - exts = self._extensions(short, nodes) + exts = self._extensions(short, ordered) candidates = [ ext for ext in exts if not any(other != ext and other.startswith(ext) for other in exts) diff --git a/tests/test_topology.py b/tests/test_topology.py index 31410d5..24253f8 100644 --- a/tests/test_topology.py +++ b/tests/test_topology.py @@ -793,3 +793,70 @@ async def test_composer_path_mode_suggests_through_anything_and_commits_verbatim empty.handle("up") empty.handle("enter") assert not empty.future.done() + + +# --- the prefix settle is a sorted-range scan, not a sweep of the graph -------------- + + +def test_extensions_matches_a_naive_scan_over_awkward_shapes() -> None: + """The binary-searched range must return exactly what scanning every node would. + + ``_extensions`` answers "which ids strictly extend this short one" by binary-searching + a lexicographically sorted list, because the caller asks it once per node and its own + caller restarts the whole pass after every merge — on a real mesh that was ~24 million + ``len`` calls and sixteen seconds. The shapes below are the ones a hex-prefix graph + actually produces: chains, shared stems, a full-width id, and near-misses either side + of the run that a sloppy range would swallow. + """ + from meshterm.services.topology import MeshTopology + + nodes = { + "6", "65", "6532", "6532eb", "6532eb00aa11", + "6533", "653300ff", "66", "6600", + "a1", "a1b2", "a1b3", "b0", + "ffffffffffff", + } + ordered = sorted(nodes) + for short in sorted(nodes): + naive = sorted( + other + for other in nodes + if other != short and len(other) > len(short) and other.startswith(short) + ) + assert sorted(MeshTopology._extensions(short, ordered)) == naive, short + + +def test_extensions_ignores_a_full_width_id() -> None: + """Twelve hex is canonical: it is nobody's prefix, however the graph sorts.""" + from meshterm.services.topology import MeshTopology + + ordered = sorted({"6532eb00aa11", "6532eb00aa1122", "65"}) + assert MeshTopology._extensions("6532eb00aa11", ordered) == [] + + +def test_prefix_settle_folds_a_chain_but_not_a_fork() -> None: + """The end-to-end behaviour the fast lookup has to preserve. + + ``65`` -> ``6532`` -> ``6532eb...`` is one node named three ways, so the graph ends up + holding it once. Add a sibling that genuinely forks the stem and ``65`` stops being an + abbreviation and starts being a coin toss between two real nodes — it must then stay + exactly as it is, under-named but honest, rather than be folded onto either. + """ + from meshterm.core.models import utcnow + from meshterm.services.topology import MeshTopology + + def graph(hops): + topo = MeshTopology(self_id="local", contacts=[]) + now = utcnow() + for hop in hops: + topo.add_walk(["local", hop], when=now, source="trace") + topo.coalesce_prefixes() + return {end for link in topo._links for end in link} + + chain = graph(("65", "6532", "6532eb00aa11")) + assert chain == {"local", "6532eb00aa11"}, f"chain not folded: {sorted(chain)}" + + fork = graph(("65", "6532", "6532eb00aa11", "653300ff2211")) + assert "6532eb00aa11" in fork and "653300ff2211" in fork + assert "6532" not in fork, "an unambiguous link in the chain should still fold" + assert "65" in fork, "a stub opening two real nodes must not be guessed onto one"