Traces: listen for the reply before the send goes out

A trace subscribed to its tag only after send_trace returned. That send
waits on the companion's MSG_SENT, which the meshcore library correlates
by event type alone — so any other command in flight (a scheduled advert,
a telemetry poll, the courier) can consume ours and leave the send
blocked on its own 15-second default. Every reply that landed inside that
window was dispatched to no listener and dropped, and the walk recorded
as "no reply" though the mesh had answered it. Because the traffic that
steals the acknowledgement is periodic, the condition outlives the trace
that hit it: traces keep failing for as long as it lasts. Subscribe
first, stamp the reply's own arrival so a stalled send inflates no round
trip, and log every trace and every reply heard so a silent walk can be
told apart from a deaf one.

The other half is pacing. run_traces sleeps between its own samples and
never after the last one, so at one trace per candidate — the path
probe's default — it paced nothing at all, and the sweep walked every
candidate back to back. Same hole across the TX optimizer's level
boundaries, and on the trace screen, where aborting a walk and hitting
Enter again transmitted immediately. Repeaters penalize a node that
bursts, and a penalized node's every later trace comes home empty. Pace
between probe candidates and TX levels, and measure the screen's cooldown
from its last transmission rather than only from the previous sample.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jean-Pierre MartineauandClaude Opus 5 committed 2026-08-11 12:30:34 -04:00
1 parent 284e35edb4
commit 936ab10770
5 files changed
+175 -23

No files matched your search

+64 -7
View File
@@ -1600,7 +1600,8 @@ class MeshCoreDevice(Device):
mc = self._require()
tag = random.randint(0, 0xFFFFFFFF)
started = asyncio.get_event_loop().time()
loop = asyncio.get_event_loop()
started = loop.time()
# A trace packet has no destination field — it walks an explicit path of
# repeater hops. Send the path as raw bytes so any uniform hash width
@@ -1627,13 +1628,54 @@ class MeshCoreDevice(Device):
if timeout is None:
hops_walked = len(path_bytes) // (1 << flags) if path_bytes else 0
timeout = trace_timeout(hops_walked)
await mc.commands.send_trace(auth_code=0, tag=tag, flags=flags, path=path_bytes)
event = await mc.wait_for_event(
EventType.TRACE_DATA,
attribute_filters={"tag": tag},
timeout=timeout,
# Listen for our tag *before* transmitting, and keep listening for the whole
# trace. ``send_trace`` doesn't return until the companion's ``MSG_SENT``
# arrives, and the library correlates that acknowledgement by nothing but its
# event type — so any other command in flight (a scheduled advert, a telemetry
# poll, the courier) can consume ours and leave the send blocked on its own
# 15-second default. Subscribing afterwards would mean every reply that landed
# during that stall was dispatched to no listener and dropped, and the trace
# recorded as "no reply" though the mesh answered it — which is why a burst of
# background traffic used to fail *every* trace for as long as it lasted, not
# just the one it collided with. The reply's own arrival time is stamped in the
# handler so a stalled send inflates no round trip.
reply: asyncio.Future = loop.create_future()
landed = started
def on_trace_reply(event) -> None: # noqa: ANN001 - meshcore Event
nonlocal landed
if not reply.done():
landed = loop.time()
reply.set_result(event)
subscription = mc.subscribe(
EventType.TRACE_DATA, on_trace_reply, {"tag": tag}
)
try:
sent = await mc.commands.send_trace(
auth_code=0, tag=tag, flags=flags, path=path_bytes
)
if getattr(sent, "is_error", None) is not None and sent.is_error():
# Uncorrelated acknowledgements make this ambiguous — the error may
# belong to another command entirely — so it is evidence, not a verdict:
# the trace is still on the air and its reply may yet arrive.
_log.debug("trace %08x: send reported %s", tag, sent.payload)
try:
event = await asyncio.wait_for(reply, timeout)
except asyncio.TimeoutError:
event = None
finally:
subscription.unsubscribe()
elapsed_ms = (landed - started) * 1000.0
_log.debug(
"trace %08x: path=%s flags=%d timeout=%.1fs -> %s",
tag,
path_bytes.hex() if path_bytes else "(none)",
flags,
timeout,
f"{elapsed_ms:.0f}ms" if event is not None else "no reply",
)
elapsed_ms = (asyncio.get_event_loop().time() - started) * 1000.0
# The firmware addresses each hop by a hash of ``1 << flags`` bytes; record it
# so the summary can show node hashes at the width the command actually used.
hash_bytes = 1 << flags
@@ -1789,6 +1831,21 @@ class MeshCoreDevice(Device):
etype = getattr(EventType, "ACK", None)
if etype is not None:
subs.append(subscribe(etype, ack_handler))
# Trace replies aren't observations — the issuing ``run_trace`` consumes them by
# tag — but logging every one that lands is what makes a "no reply" diagnosable:
# a reply logged here with no matching ``trace <tag>`` line means the walk came
# home and we weren't listening, which is a different fault from silence on air.
etype = getattr(EventType, "TRACE_DATA", None)
if etype is not None:
subs.append(
subscribe(
etype,
lambda event: _log.debug(
"trace reply heard: tag=%08x",
(getattr(event, "payload", {}) or {}).get("tag", 0),
),
)
)
# Drive the inbound-message pull ourselves (see ``_message_pump``): MeshCore never
# pushes message bodies, so without this sending works but nothing is received.
+8
View File
@@ -12,6 +12,7 @@ the winner — measurement proposes, the user disposes.
from __future__ import annotations
import asyncio
from dataclasses import dataclass
from typing import Awaitable, Callable, Optional
@@ -102,6 +103,13 @@ async def probe_paths(
outcomes: list[ProbeOutcome] = []
for index, candidate in enumerate(candidates):
if index and cooldown_s > 0:
# ``run_traces`` paces *between* its own samples and never after the last
# one, so at the default of one trace per candidate it paces nothing at
# all: without this the sweep walks every candidate back to back and the
# repeaters see a burst — the traffic pattern that gets a node ignored,
# after which every later trace comes home empty.
await asyncio.sleep(cooldown_s)
results = await trace_runner.run_traces(
device,
target,
+6
View File
@@ -23,6 +23,7 @@ when ``apply`` is set (the whole point of the run is to leave it tuned).
from __future__ import annotations
import asyncio
import statistics
from typing import Callable, Optional
@@ -209,6 +210,11 @@ async def optimize_tx_power(
async def measure_level(tx: int) -> TxLevelResult:
"""Run a batch of traces at ``tx`` (accumulating) and refresh its aggregate."""
nonlocal completed
if completed and cooldown_s > 0:
# ``run_traces`` paces between its own samples but not after the last one,
# so without this the first trace of a level follows the previous level's
# last one with no gap — a burst across every level boundary.
await asyncio.sleep(cooldown_s)
await device.set_remote_tx_power(admin_node, tx)
batch = await trace_runner.run_traces(
device,
+29 -10
View File
@@ -473,6 +473,9 @@ class TraceScreen(Screen):
self._status = ""
self._progress: Optional[tuple[int, int]] = None # (current, total) mid-run
self._spinner = Spinner()
#: Loop clock at this screen's last transmission — what :meth:`_pace_remaining`
#: measures the cooldown from, so pacing survives an abort-and-retry.
self._last_tx: Optional[float] = None
self._worker: Optional[asyncio.Task] = None
self._flight: Optional[TracingDialog] = None
# The action rows, in display order. The build-path group leads: Compose,
@@ -515,12 +518,17 @@ class TraceScreen(Screen):
async def _run_trace(self) -> None:
"""Drive one Trace commit — the chosen number of traces — under the dialog.
Multi-sample runs are paced: :attr:`_pace_s` sleeps between transmissions so
the repeaters never see a burst, and the dialog counts the run off as replies
land. The dialog is pushed for the duration and popped however the run ends —
completion, failure, or abort — and its Abort wires straight to :meth:`cancel`,
so the cancellation path is the same whether Esc lands on the dialog or the
screen. An aborted run keeps every trace already recorded.
Every transmission is paced, not just the ones inside a multi-sample run:
:attr:`_pace_s` is measured from the *last* transmission this screen made, so
aborting a walk and immediately re-running it waits out the same gap a second
sample would. Pacing only between samples left the one hole that matters — a
user watching a failure and hitting Enter again straight away is exactly the
burst repeaters penalize, and a penalized node's every later trace comes home
empty. The dialog counts the run off as replies land; it is pushed for the
duration and popped however the run ends — completion, failure, or abort — and
its Abort wires straight to :meth:`cancel`, so the cancellation path is the
same whether Esc lands on the dialog or the screen. An aborted run keeps every
trace already recorded.
"""
total = max(1, int(self._sample_count()))
dialog = TracingDialog(
@@ -532,14 +540,18 @@ class TraceScreen(Screen):
try:
for done in range(total):
self._progress = (done + 1, total)
wait = self._pace_remaining()
if wait > 0:
dialog.status = (
f"trace {done + 1}/{total} · pacing…" if total > 1 else "pacing…"
)
self._session.invalidate()
await asyncio.sleep(wait)
if total > 1:
dialog.status = f"trace {done + 1}/{total} · transmitting…"
self._session.invalidate()
self._last_tx = asyncio.get_event_loop().time()
await self._trace_once(self._path_spec, self._on_trace)
if done + 1 < total and self._pace_s > 0:
dialog.status = f"trace {done + 1}/{total} landed · pacing…"
self._session.invalidate()
await asyncio.sleep(self._pace_s)
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001 - report inline, keep the screen alive
@@ -565,6 +577,13 @@ class TraceScreen(Screen):
# a popup must never chase the user out of the screen.
asyncio.ensure_future(self._announce_records())
def _pace_remaining(self) -> float:
"""Seconds still owed before the next transmission may go out (0 when clear)."""
if self._last_tx is None or self._pace_s <= 0:
return 0.0
elapsed = asyncio.get_event_loop().time() - self._last_tx
return max(0.0, self._pace_s - elapsed)
async def _animate(self) -> None:
"""Advance the in-flight spinner and repaint on a steady cadence, until cancelled."""
while True:
+68 -6
View File
@@ -366,16 +366,26 @@ def test_trace_timeout_scales_with_hops() -> None:
assert trace_timeout(10_000) == TRACE_TIMEOUT_CEILING_S
async def test_run_trace_sizes_wait_to_the_forced_route() -> None:
async def test_run_trace_sizes_wait_to_the_forced_route(monkeypatch) -> None: # noqa: ANN001
"""A forced path's reply-wait is sized to its hop count, not a flat default.
The real device must hand ``wait_for_event`` the route-scaled budget so a long walk
isn't cut off before its reply can travel out and back.
The real device must give the reply the route-scaled budget so a long walk isn't
cut off before it can travel out and back.
"""
import asyncio as _asyncio
from meshterm.core import connection as connection_module
from meshterm.core.connection import MeshCoreDevice
from meshterm.services.trace_runner import trace_timeout
seen: dict[str, float] = {}
real_wait_for = _asyncio.wait_for
async def spy_wait_for(awaitable, timeout): # noqa: ANN001, ANN202
seen["timeout"] = timeout
return await real_wait_for(awaitable, 0) # a miss, without burning the budget
monkeypatch.setattr(connection_module.asyncio, "wait_for", spy_wait_for)
class _Commands:
async def send_trace(self, *, auth_code, tag, flags, path): # noqa: ANN001, ANN201
@@ -384,9 +394,12 @@ async def test_run_trace_sizes_wait_to_the_forced_route() -> None:
class _MC:
commands = _Commands()
async def wait_for_event(self, event_type, *, attribute_filters=None, timeout=None): # noqa: ANN001, ANN201
seen["timeout"] = timeout
return None # a miss: run_trace returns a clean failure, no further calls
def subscribe(self, event_type, callback, attribute_filters=None): # noqa: ANN001, ANN201
return _Subscription()
class _Subscription:
def unsubscribe(self) -> None:
return None
device = MeshCoreDevice(port="COM-test")
device._mc = _MC()
@@ -401,6 +414,55 @@ async def test_run_trace_sizes_wait_to_the_forced_route() -> None:
assert seen["timeout"] == 2.0
async def test_run_trace_catches_a_reply_that_beats_the_send_acknowledgement() -> None:
"""A reply landing while ``send_trace`` is still blocked is still our trace's reply.
The companion's send acknowledgement is correlated by nothing but its event type, so
a concurrent command can consume ours and leave the send waiting out its own default.
Listening for the tag only *after* the send returns dropped every reply that arrived
in that window — the trace read as "no reply" though the mesh had answered it.
"""
import asyncio as _asyncio
from meshterm.core.connection import MeshCoreDevice
class _Event:
payload = {"tag": 1, "path": [{"hash": "3d", "snr": 4.0}, {"snr": 2.0}]}
listeners: list = []
class _Commands:
async def send_trace(self, *, auth_code, tag, flags, path): # noqa: ANN001, ANN201
# The reply overtakes the acknowledgement: deliver it, then keep stalling.
for callback in listeners:
callback(_Event())
await _asyncio.sleep(0)
return None
async def send_appstart(self): # noqa: ANN201 - the tx-power read on success
return _SelfInfo()
class _SelfInfo:
payload = {"tx_power": 20}
class _MC:
commands = _Commands()
def subscribe(self, event_type, callback, attribute_filters=None): # noqa: ANN001, ANN201
listeners.append(callback)
return _Subscription()
class _Subscription:
def unsubscribe(self) -> None:
listeners.clear()
device = MeshCoreDevice(port="COM-test")
device._mc = _MC()
result = await device.run_trace("Alice", path="3d,f2,3d")
assert result.success is True
assert [h.node for h in result.hops] == ["3d", None]
def test_trace_edges_endpoints_are_our_device() -> None:
"""The first edge originates at us and the last edge returns to us (#3/#4)."""
from meshterm.core.models import Hop