diff --git a/meshterm/core/connection.py b/meshterm/core/connection.py index 555d684..91a4651 100644 --- a/meshterm/core/connection.py +++ b/meshterm/core/connection.py @@ -1600,7 +1600,8 @@ class MeshCoreDevice(Device): mc = self._require() tag = random.randint(0, 0xFFFFFFFF) - started = asyncio.get_event_loop().time() + loop = asyncio.get_event_loop() + started = loop.time() # A trace packet has no destination field — it walks an explicit path of # repeater hops. Send the path as raw bytes so any uniform hash width @@ -1627,13 +1628,54 @@ class MeshCoreDevice(Device): if timeout is None: hops_walked = len(path_bytes) // (1 << flags) if path_bytes else 0 timeout = trace_timeout(hops_walked) - await mc.commands.send_trace(auth_code=0, tag=tag, flags=flags, path=path_bytes) - event = await mc.wait_for_event( - EventType.TRACE_DATA, - attribute_filters={"tag": tag}, - timeout=timeout, + + # Listen for our tag *before* transmitting, and keep listening for the whole + # trace. ``send_trace`` doesn't return until the companion's ``MSG_SENT`` + # arrives, and the library correlates that acknowledgement by nothing but its + # event type — so any other command in flight (a scheduled advert, a telemetry + # poll, the courier) can consume ours and leave the send blocked on its own + # 15-second default. Subscribing afterwards would mean every reply that landed + # during that stall was dispatched to no listener and dropped, and the trace + # recorded as "no reply" though the mesh answered it — which is why a burst of + # background traffic used to fail *every* trace for as long as it lasted, not + # just the one it collided with. The reply's own arrival time is stamped in the + # handler so a stalled send inflates no round trip. + reply: asyncio.Future = loop.create_future() + landed = started + + def on_trace_reply(event) -> None: # noqa: ANN001 - meshcore Event + nonlocal landed + if not reply.done(): + landed = loop.time() + reply.set_result(event) + + subscription = mc.subscribe( + EventType.TRACE_DATA, on_trace_reply, {"tag": tag} + ) + try: + sent = await mc.commands.send_trace( + auth_code=0, tag=tag, flags=flags, path=path_bytes + ) + if getattr(sent, "is_error", None) is not None and sent.is_error(): + # Uncorrelated acknowledgements make this ambiguous — the error may + # belong to another command entirely — so it is evidence, not a verdict: + # the trace is still on the air and its reply may yet arrive. + _log.debug("trace %08x: send reported %s", tag, sent.payload) + try: + event = await asyncio.wait_for(reply, timeout) + except asyncio.TimeoutError: + event = None + finally: + subscription.unsubscribe() + elapsed_ms = (landed - started) * 1000.0 + _log.debug( + "trace %08x: path=%s flags=%d timeout=%.1fs -> %s", + tag, + path_bytes.hex() if path_bytes else "(none)", + flags, + timeout, + f"{elapsed_ms:.0f}ms" if event is not None else "no reply", ) - elapsed_ms = (asyncio.get_event_loop().time() - started) * 1000.0 # The firmware addresses each hop by a hash of ``1 << flags`` bytes; record it # so the summary can show node hashes at the width the command actually used. hash_bytes = 1 << flags @@ -1789,6 +1831,21 @@ class MeshCoreDevice(Device): etype = getattr(EventType, "ACK", None) if etype is not None: subs.append(subscribe(etype, ack_handler)) + # Trace replies aren't observations — the issuing ``run_trace`` consumes them by + # tag — but logging every one that lands is what makes a "no reply" diagnosable: + # a reply logged here with no matching ``trace `` line means the walk came + # home and we weren't listening, which is a different fault from silence on air. + etype = getattr(EventType, "TRACE_DATA", None) + if etype is not None: + subs.append( + subscribe( + etype, + lambda event: _log.debug( + "trace reply heard: tag=%08x", + (getattr(event, "payload", {}) or {}).get("tag", 0), + ), + ) + ) # Drive the inbound-message pull ourselves (see ``_message_pump``): MeshCore never # pushes message bodies, so without this sending works but nothing is received. diff --git a/meshterm/services/path_probe.py b/meshterm/services/path_probe.py index 0255331..eff639c 100644 --- a/meshterm/services/path_probe.py +++ b/meshterm/services/path_probe.py @@ -12,6 +12,7 @@ the winner — measurement proposes, the user disposes. from __future__ import annotations +import asyncio from dataclasses import dataclass from typing import Awaitable, Callable, Optional @@ -102,6 +103,13 @@ async def probe_paths( outcomes: list[ProbeOutcome] = [] for index, candidate in enumerate(candidates): + if index and cooldown_s > 0: + # ``run_traces`` paces *between* its own samples and never after the last + # one, so at the default of one trace per candidate it paces nothing at + # all: without this the sweep walks every candidate back to back and the + # repeaters see a burst — the traffic pattern that gets a node ignored, + # after which every later trace comes home empty. + await asyncio.sleep(cooldown_s) results = await trace_runner.run_traces( device, target, diff --git a/meshterm/services/tx_optimizer.py b/meshterm/services/tx_optimizer.py index e1d0c53..0e638e5 100644 --- a/meshterm/services/tx_optimizer.py +++ b/meshterm/services/tx_optimizer.py @@ -23,6 +23,7 @@ when ``apply`` is set (the whole point of the run is to leave it tuned). from __future__ import annotations +import asyncio import statistics from typing import Callable, Optional @@ -209,6 +210,11 @@ async def optimize_tx_power( async def measure_level(tx: int) -> TxLevelResult: """Run a batch of traces at ``tx`` (accumulating) and refresh its aggregate.""" nonlocal completed + if completed and cooldown_s > 0: + # ``run_traces`` paces between its own samples but not after the last one, + # so without this the first trace of a level follows the previous level's + # last one with no gap — a burst across every level boundary. + await asyncio.sleep(cooldown_s) await device.set_remote_tx_power(admin_node, tx) batch = await trace_runner.run_traces( device, diff --git a/meshterm/ui/trace_screen.py b/meshterm/ui/trace_screen.py index e8019a4..a4856c5 100644 --- a/meshterm/ui/trace_screen.py +++ b/meshterm/ui/trace_screen.py @@ -473,6 +473,9 @@ class TraceScreen(Screen): self._status = "" self._progress: Optional[tuple[int, int]] = None # (current, total) mid-run self._spinner = Spinner() + #: Loop clock at this screen's last transmission — what :meth:`_pace_remaining` + #: measures the cooldown from, so pacing survives an abort-and-retry. + self._last_tx: Optional[float] = None self._worker: Optional[asyncio.Task] = None self._flight: Optional[TracingDialog] = None # The action rows, in display order. The build-path group leads: Compose, @@ -515,12 +518,17 @@ class TraceScreen(Screen): async def _run_trace(self) -> None: """Drive one Trace commit — the chosen number of traces — under the dialog. - Multi-sample runs are paced: :attr:`_pace_s` sleeps between transmissions so - the repeaters never see a burst, and the dialog counts the run off as replies - land. The dialog is pushed for the duration and popped however the run ends — - completion, failure, or abort — and its Abort wires straight to :meth:`cancel`, - so the cancellation path is the same whether Esc lands on the dialog or the - screen. An aborted run keeps every trace already recorded. + Every transmission is paced, not just the ones inside a multi-sample run: + :attr:`_pace_s` is measured from the *last* transmission this screen made, so + aborting a walk and immediately re-running it waits out the same gap a second + sample would. Pacing only between samples left the one hole that matters — a + user watching a failure and hitting Enter again straight away is exactly the + burst repeaters penalize, and a penalized node's every later trace comes home + empty. The dialog counts the run off as replies land; it is pushed for the + duration and popped however the run ends — completion, failure, or abort — and + its Abort wires straight to :meth:`cancel`, so the cancellation path is the + same whether Esc lands on the dialog or the screen. An aborted run keeps every + trace already recorded. """ total = max(1, int(self._sample_count())) dialog = TracingDialog( @@ -532,14 +540,18 @@ class TraceScreen(Screen): try: for done in range(total): self._progress = (done + 1, total) + wait = self._pace_remaining() + if wait > 0: + dialog.status = ( + f"trace {done + 1}/{total} · pacing…" if total > 1 else "pacing…" + ) + self._session.invalidate() + await asyncio.sleep(wait) if total > 1: dialog.status = f"trace {done + 1}/{total} · transmitting…" self._session.invalidate() + self._last_tx = asyncio.get_event_loop().time() await self._trace_once(self._path_spec, self._on_trace) - if done + 1 < total and self._pace_s > 0: - dialog.status = f"trace {done + 1}/{total} landed · pacing…" - self._session.invalidate() - await asyncio.sleep(self._pace_s) except asyncio.CancelledError: raise except Exception as exc: # noqa: BLE001 - report inline, keep the screen alive @@ -565,6 +577,13 @@ class TraceScreen(Screen): # a popup must never chase the user out of the screen. asyncio.ensure_future(self._announce_records()) + def _pace_remaining(self) -> float: + """Seconds still owed before the next transmission may go out (0 when clear).""" + if self._last_tx is None or self._pace_s <= 0: + return 0.0 + elapsed = asyncio.get_event_loop().time() - self._last_tx + return max(0.0, self._pace_s - elapsed) + async def _animate(self) -> None: """Advance the in-flight spinner and repaint on a steady cadence, until cancelled.""" while True: diff --git a/tests/test_skeleton.py b/tests/test_skeleton.py index 68dbfc9..6a35d03 100644 --- a/tests/test_skeleton.py +++ b/tests/test_skeleton.py @@ -366,16 +366,26 @@ def test_trace_timeout_scales_with_hops() -> None: assert trace_timeout(10_000) == TRACE_TIMEOUT_CEILING_S -async def test_run_trace_sizes_wait_to_the_forced_route() -> None: +async def test_run_trace_sizes_wait_to_the_forced_route(monkeypatch) -> None: # noqa: ANN001 """A forced path's reply-wait is sized to its hop count, not a flat default. - The real device must hand ``wait_for_event`` the route-scaled budget so a long walk - isn't cut off before its reply can travel out and back. + The real device must give the reply the route-scaled budget so a long walk isn't + cut off before it can travel out and back. """ + import asyncio as _asyncio + + from meshterm.core import connection as connection_module from meshterm.core.connection import MeshCoreDevice from meshterm.services.trace_runner import trace_timeout seen: dict[str, float] = {} + real_wait_for = _asyncio.wait_for + + async def spy_wait_for(awaitable, timeout): # noqa: ANN001, ANN202 + seen["timeout"] = timeout + return await real_wait_for(awaitable, 0) # a miss, without burning the budget + + monkeypatch.setattr(connection_module.asyncio, "wait_for", spy_wait_for) class _Commands: async def send_trace(self, *, auth_code, tag, flags, path): # noqa: ANN001, ANN201 @@ -384,9 +394,12 @@ async def test_run_trace_sizes_wait_to_the_forced_route() -> None: class _MC: commands = _Commands() - async def wait_for_event(self, event_type, *, attribute_filters=None, timeout=None): # noqa: ANN001, ANN201 - seen["timeout"] = timeout - return None # a miss: run_trace returns a clean failure, no further calls + def subscribe(self, event_type, callback, attribute_filters=None): # noqa: ANN001, ANN201 + return _Subscription() + + class _Subscription: + def unsubscribe(self) -> None: + return None device = MeshCoreDevice(port="COM-test") device._mc = _MC() @@ -401,6 +414,55 @@ async def test_run_trace_sizes_wait_to_the_forced_route() -> None: assert seen["timeout"] == 2.0 +async def test_run_trace_catches_a_reply_that_beats_the_send_acknowledgement() -> None: + """A reply landing while ``send_trace`` is still blocked is still our trace's reply. + + The companion's send acknowledgement is correlated by nothing but its event type, so + a concurrent command can consume ours and leave the send waiting out its own default. + Listening for the tag only *after* the send returns dropped every reply that arrived + in that window — the trace read as "no reply" though the mesh had answered it. + """ + import asyncio as _asyncio + + from meshterm.core.connection import MeshCoreDevice + + class _Event: + payload = {"tag": 1, "path": [{"hash": "3d", "snr": 4.0}, {"snr": 2.0}]} + + listeners: list = [] + + class _Commands: + async def send_trace(self, *, auth_code, tag, flags, path): # noqa: ANN001, ANN201 + # The reply overtakes the acknowledgement: deliver it, then keep stalling. + for callback in listeners: + callback(_Event()) + await _asyncio.sleep(0) + return None + + async def send_appstart(self): # noqa: ANN201 - the tx-power read on success + return _SelfInfo() + + class _SelfInfo: + payload = {"tx_power": 20} + + class _MC: + commands = _Commands() + + def subscribe(self, event_type, callback, attribute_filters=None): # noqa: ANN001, ANN201 + listeners.append(callback) + return _Subscription() + + class _Subscription: + def unsubscribe(self) -> None: + listeners.clear() + + device = MeshCoreDevice(port="COM-test") + device._mc = _MC() + result = await device.run_trace("Alice", path="3d,f2,3d") + assert result.success is True + assert [h.node for h in result.hops] == ["3d", None] + + def test_trace_edges_endpoints_are_our_device() -> None: """The first edge originates at us and the last edge returns to us (#3/#4).""" from meshterm.core.models import Hop