Traces: listen for the reply before the send goes out
A trace subscribed to its tag only after send_trace returned. That send waits on the companion's MSG_SENT, which the meshcore library correlates by event type alone — so any other command in flight (a scheduled advert, a telemetry poll, the courier) can consume ours and leave the send blocked on its own 15-second default. Every reply that landed inside that window was dispatched to no listener and dropped, and the walk recorded as "no reply" though the mesh had answered it. Because the traffic that steals the acknowledgement is periodic, the condition outlives the trace that hit it: traces keep failing for as long as it lasts. Subscribe first, stamp the reply's own arrival so a stalled send inflates no round trip, and log every trace and every reply heard so a silent walk can be told apart from a deaf one. The other half is pacing. run_traces sleeps between its own samples and never after the last one, so at one trace per candidate — the path probe's default — it paced nothing at all, and the sweep walked every candidate back to back. Same hole across the TX optimizer's level boundaries, and on the trace screen, where aborting a walk and hitting Enter again transmitted immediately. Repeaters penalize a node that bursts, and a penalized node's every later trace comes home empty. Pace between probe candidates and TX levels, and measure the screen's cooldown from its last transmission rather than only from the previous sample. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
284e35edb4
commit
936ab10770
5 files changed
+175
-23
No files matched your search
@@ -1600,7 +1600,8 @@ class MeshCoreDevice(Device):
|
||||
|
||||
mc = self._require()
|
||||
tag = random.randint(0, 0xFFFFFFFF)
|
||||
started = asyncio.get_event_loop().time()
|
||||
loop = asyncio.get_event_loop()
|
||||
started = loop.time()
|
||||
|
||||
# A trace packet has no destination field — it walks an explicit path of
|
||||
# repeater hops. Send the path as raw bytes so any uniform hash width
|
||||
@@ -1627,13 +1628,54 @@ class MeshCoreDevice(Device):
|
||||
if timeout is None:
|
||||
hops_walked = len(path_bytes) // (1 << flags) if path_bytes else 0
|
||||
timeout = trace_timeout(hops_walked)
|
||||
await mc.commands.send_trace(auth_code=0, tag=tag, flags=flags, path=path_bytes)
|
||||
event = await mc.wait_for_event(
|
||||
EventType.TRACE_DATA,
|
||||
attribute_filters={"tag": tag},
|
||||
timeout=timeout,
|
||||
|
||||
# Listen for our tag *before* transmitting, and keep listening for the whole
|
||||
# trace. ``send_trace`` doesn't return until the companion's ``MSG_SENT``
|
||||
# arrives, and the library correlates that acknowledgement by nothing but its
|
||||
# event type — so any other command in flight (a scheduled advert, a telemetry
|
||||
# poll, the courier) can consume ours and leave the send blocked on its own
|
||||
# 15-second default. Subscribing afterwards would mean every reply that landed
|
||||
# during that stall was dispatched to no listener and dropped, and the trace
|
||||
# recorded as "no reply" though the mesh answered it — which is why a burst of
|
||||
# background traffic used to fail *every* trace for as long as it lasted, not
|
||||
# just the one it collided with. The reply's own arrival time is stamped in the
|
||||
# handler so a stalled send inflates no round trip.
|
||||
reply: asyncio.Future = loop.create_future()
|
||||
landed = started
|
||||
|
||||
def on_trace_reply(event) -> None: # noqa: ANN001 - meshcore Event
|
||||
nonlocal landed
|
||||
if not reply.done():
|
||||
landed = loop.time()
|
||||
reply.set_result(event)
|
||||
|
||||
subscription = mc.subscribe(
|
||||
EventType.TRACE_DATA, on_trace_reply, {"tag": tag}
|
||||
)
|
||||
try:
|
||||
sent = await mc.commands.send_trace(
|
||||
auth_code=0, tag=tag, flags=flags, path=path_bytes
|
||||
)
|
||||
if getattr(sent, "is_error", None) is not None and sent.is_error():
|
||||
# Uncorrelated acknowledgements make this ambiguous — the error may
|
||||
# belong to another command entirely — so it is evidence, not a verdict:
|
||||
# the trace is still on the air and its reply may yet arrive.
|
||||
_log.debug("trace %08x: send reported %s", tag, sent.payload)
|
||||
try:
|
||||
event = await asyncio.wait_for(reply, timeout)
|
||||
except asyncio.TimeoutError:
|
||||
event = None
|
||||
finally:
|
||||
subscription.unsubscribe()
|
||||
elapsed_ms = (landed - started) * 1000.0
|
||||
_log.debug(
|
||||
"trace %08x: path=%s flags=%d timeout=%.1fs -> %s",
|
||||
tag,
|
||||
path_bytes.hex() if path_bytes else "(none)",
|
||||
flags,
|
||||
timeout,
|
||||
f"{elapsed_ms:.0f}ms" if event is not None else "no reply",
|
||||
)
|
||||
elapsed_ms = (asyncio.get_event_loop().time() - started) * 1000.0
|
||||
# The firmware addresses each hop by a hash of ``1 << flags`` bytes; record it
|
||||
# so the summary can show node hashes at the width the command actually used.
|
||||
hash_bytes = 1 << flags
|
||||
@@ -1789,6 +1831,21 @@ class MeshCoreDevice(Device):
|
||||
etype = getattr(EventType, "ACK", None)
|
||||
if etype is not None:
|
||||
subs.append(subscribe(etype, ack_handler))
|
||||
# Trace replies aren't observations — the issuing ``run_trace`` consumes them by
|
||||
# tag — but logging every one that lands is what makes a "no reply" diagnosable:
|
||||
# a reply logged here with no matching ``trace <tag>`` line means the walk came
|
||||
# home and we weren't listening, which is a different fault from silence on air.
|
||||
etype = getattr(EventType, "TRACE_DATA", None)
|
||||
if etype is not None:
|
||||
subs.append(
|
||||
subscribe(
|
||||
etype,
|
||||
lambda event: _log.debug(
|
||||
"trace reply heard: tag=%08x",
|
||||
(getattr(event, "payload", {}) or {}).get("tag", 0),
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
# Drive the inbound-message pull ourselves (see ``_message_pump``): MeshCore never
|
||||
# pushes message bodies, so without this sending works but nothing is received.
|
||||
|
||||
@@ -12,6 +12,7 @@ the winner — measurement proposes, the user disposes.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from typing import Awaitable, Callable, Optional
|
||||
|
||||
@@ -102,6 +103,13 @@ async def probe_paths(
|
||||
|
||||
outcomes: list[ProbeOutcome] = []
|
||||
for index, candidate in enumerate(candidates):
|
||||
if index and cooldown_s > 0:
|
||||
# ``run_traces`` paces *between* its own samples and never after the last
|
||||
# one, so at the default of one trace per candidate it paces nothing at
|
||||
# all: without this the sweep walks every candidate back to back and the
|
||||
# repeaters see a burst — the traffic pattern that gets a node ignored,
|
||||
# after which every later trace comes home empty.
|
||||
await asyncio.sleep(cooldown_s)
|
||||
results = await trace_runner.run_traces(
|
||||
device,
|
||||
target,
|
||||
|
||||
@@ -23,6 +23,7 @@ when ``apply`` is set (the whole point of the run is to leave it tuned).
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import statistics
|
||||
from typing import Callable, Optional
|
||||
|
||||
@@ -209,6 +210,11 @@ async def optimize_tx_power(
|
||||
async def measure_level(tx: int) -> TxLevelResult:
|
||||
"""Run a batch of traces at ``tx`` (accumulating) and refresh its aggregate."""
|
||||
nonlocal completed
|
||||
if completed and cooldown_s > 0:
|
||||
# ``run_traces`` paces between its own samples but not after the last one,
|
||||
# so without this the first trace of a level follows the previous level's
|
||||
# last one with no gap — a burst across every level boundary.
|
||||
await asyncio.sleep(cooldown_s)
|
||||
await device.set_remote_tx_power(admin_node, tx)
|
||||
batch = await trace_runner.run_traces(
|
||||
device,
|
||||
|
||||
+29
-10
@@ -473,6 +473,9 @@ class TraceScreen(Screen):
|
||||
self._status = ""
|
||||
self._progress: Optional[tuple[int, int]] = None # (current, total) mid-run
|
||||
self._spinner = Spinner()
|
||||
#: Loop clock at this screen's last transmission — what :meth:`_pace_remaining`
|
||||
#: measures the cooldown from, so pacing survives an abort-and-retry.
|
||||
self._last_tx: Optional[float] = None
|
||||
self._worker: Optional[asyncio.Task] = None
|
||||
self._flight: Optional[TracingDialog] = None
|
||||
# The action rows, in display order. The build-path group leads: Compose,
|
||||
@@ -515,12 +518,17 @@ class TraceScreen(Screen):
|
||||
async def _run_trace(self) -> None:
|
||||
"""Drive one Trace commit — the chosen number of traces — under the dialog.
|
||||
|
||||
Multi-sample runs are paced: :attr:`_pace_s` sleeps between transmissions so
|
||||
the repeaters never see a burst, and the dialog counts the run off as replies
|
||||
land. The dialog is pushed for the duration and popped however the run ends —
|
||||
completion, failure, or abort — and its Abort wires straight to :meth:`cancel`,
|
||||
so the cancellation path is the same whether Esc lands on the dialog or the
|
||||
screen. An aborted run keeps every trace already recorded.
|
||||
Every transmission is paced, not just the ones inside a multi-sample run:
|
||||
:attr:`_pace_s` is measured from the *last* transmission this screen made, so
|
||||
aborting a walk and immediately re-running it waits out the same gap a second
|
||||
sample would. Pacing only between samples left the one hole that matters — a
|
||||
user watching a failure and hitting Enter again straight away is exactly the
|
||||
burst repeaters penalize, and a penalized node's every later trace comes home
|
||||
empty. The dialog counts the run off as replies land; it is pushed for the
|
||||
duration and popped however the run ends — completion, failure, or abort — and
|
||||
its Abort wires straight to :meth:`cancel`, so the cancellation path is the
|
||||
same whether Esc lands on the dialog or the screen. An aborted run keeps every
|
||||
trace already recorded.
|
||||
"""
|
||||
total = max(1, int(self._sample_count()))
|
||||
dialog = TracingDialog(
|
||||
@@ -532,14 +540,18 @@ class TraceScreen(Screen):
|
||||
try:
|
||||
for done in range(total):
|
||||
self._progress = (done + 1, total)
|
||||
wait = self._pace_remaining()
|
||||
if wait > 0:
|
||||
dialog.status = (
|
||||
f"trace {done + 1}/{total} · pacing…" if total > 1 else "pacing…"
|
||||
)
|
||||
self._session.invalidate()
|
||||
await asyncio.sleep(wait)
|
||||
if total > 1:
|
||||
dialog.status = f"trace {done + 1}/{total} · transmitting…"
|
||||
self._session.invalidate()
|
||||
self._last_tx = asyncio.get_event_loop().time()
|
||||
await self._trace_once(self._path_spec, self._on_trace)
|
||||
if done + 1 < total and self._pace_s > 0:
|
||||
dialog.status = f"trace {done + 1}/{total} landed · pacing…"
|
||||
self._session.invalidate()
|
||||
await asyncio.sleep(self._pace_s)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - report inline, keep the screen alive
|
||||
@@ -565,6 +577,13 @@ class TraceScreen(Screen):
|
||||
# a popup must never chase the user out of the screen.
|
||||
asyncio.ensure_future(self._announce_records())
|
||||
|
||||
def _pace_remaining(self) -> float:
|
||||
"""Seconds still owed before the next transmission may go out (0 when clear)."""
|
||||
if self._last_tx is None or self._pace_s <= 0:
|
||||
return 0.0
|
||||
elapsed = asyncio.get_event_loop().time() - self._last_tx
|
||||
return max(0.0, self._pace_s - elapsed)
|
||||
|
||||
async def _animate(self) -> None:
|
||||
"""Advance the in-flight spinner and repaint on a steady cadence, until cancelled."""
|
||||
while True:
|
||||
|
||||
+68
-6
@@ -366,16 +366,26 @@ def test_trace_timeout_scales_with_hops() -> None:
|
||||
assert trace_timeout(10_000) == TRACE_TIMEOUT_CEILING_S
|
||||
|
||||
|
||||
async def test_run_trace_sizes_wait_to_the_forced_route() -> None:
|
||||
async def test_run_trace_sizes_wait_to_the_forced_route(monkeypatch) -> None: # noqa: ANN001
|
||||
"""A forced path's reply-wait is sized to its hop count, not a flat default.
|
||||
|
||||
The real device must hand ``wait_for_event`` the route-scaled budget so a long walk
|
||||
isn't cut off before its reply can travel out and back.
|
||||
The real device must give the reply the route-scaled budget so a long walk isn't
|
||||
cut off before it can travel out and back.
|
||||
"""
|
||||
import asyncio as _asyncio
|
||||
|
||||
from meshterm.core import connection as connection_module
|
||||
from meshterm.core.connection import MeshCoreDevice
|
||||
from meshterm.services.trace_runner import trace_timeout
|
||||
|
||||
seen: dict[str, float] = {}
|
||||
real_wait_for = _asyncio.wait_for
|
||||
|
||||
async def spy_wait_for(awaitable, timeout): # noqa: ANN001, ANN202
|
||||
seen["timeout"] = timeout
|
||||
return await real_wait_for(awaitable, 0) # a miss, without burning the budget
|
||||
|
||||
monkeypatch.setattr(connection_module.asyncio, "wait_for", spy_wait_for)
|
||||
|
||||
class _Commands:
|
||||
async def send_trace(self, *, auth_code, tag, flags, path): # noqa: ANN001, ANN201
|
||||
@@ -384,9 +394,12 @@ async def test_run_trace_sizes_wait_to_the_forced_route() -> None:
|
||||
class _MC:
|
||||
commands = _Commands()
|
||||
|
||||
async def wait_for_event(self, event_type, *, attribute_filters=None, timeout=None): # noqa: ANN001, ANN201
|
||||
seen["timeout"] = timeout
|
||||
return None # a miss: run_trace returns a clean failure, no further calls
|
||||
def subscribe(self, event_type, callback, attribute_filters=None): # noqa: ANN001, ANN201
|
||||
return _Subscription()
|
||||
|
||||
class _Subscription:
|
||||
def unsubscribe(self) -> None:
|
||||
return None
|
||||
|
||||
device = MeshCoreDevice(port="COM-test")
|
||||
device._mc = _MC()
|
||||
@@ -401,6 +414,55 @@ async def test_run_trace_sizes_wait_to_the_forced_route() -> None:
|
||||
assert seen["timeout"] == 2.0
|
||||
|
||||
|
||||
async def test_run_trace_catches_a_reply_that_beats_the_send_acknowledgement() -> None:
|
||||
"""A reply landing while ``send_trace`` is still blocked is still our trace's reply.
|
||||
|
||||
The companion's send acknowledgement is correlated by nothing but its event type, so
|
||||
a concurrent command can consume ours and leave the send waiting out its own default.
|
||||
Listening for the tag only *after* the send returns dropped every reply that arrived
|
||||
in that window — the trace read as "no reply" though the mesh had answered it.
|
||||
"""
|
||||
import asyncio as _asyncio
|
||||
|
||||
from meshterm.core.connection import MeshCoreDevice
|
||||
|
||||
class _Event:
|
||||
payload = {"tag": 1, "path": [{"hash": "3d", "snr": 4.0}, {"snr": 2.0}]}
|
||||
|
||||
listeners: list = []
|
||||
|
||||
class _Commands:
|
||||
async def send_trace(self, *, auth_code, tag, flags, path): # noqa: ANN001, ANN201
|
||||
# The reply overtakes the acknowledgement: deliver it, then keep stalling.
|
||||
for callback in listeners:
|
||||
callback(_Event())
|
||||
await _asyncio.sleep(0)
|
||||
return None
|
||||
|
||||
async def send_appstart(self): # noqa: ANN201 - the tx-power read on success
|
||||
return _SelfInfo()
|
||||
|
||||
class _SelfInfo:
|
||||
payload = {"tx_power": 20}
|
||||
|
||||
class _MC:
|
||||
commands = _Commands()
|
||||
|
||||
def subscribe(self, event_type, callback, attribute_filters=None): # noqa: ANN001, ANN201
|
||||
listeners.append(callback)
|
||||
return _Subscription()
|
||||
|
||||
class _Subscription:
|
||||
def unsubscribe(self) -> None:
|
||||
listeners.clear()
|
||||
|
||||
device = MeshCoreDevice(port="COM-test")
|
||||
device._mc = _MC()
|
||||
result = await device.run_trace("Alice", path="3d,f2,3d")
|
||||
assert result.success is True
|
||||
assert [h.node for h in result.hops] == ["3d", None]
|
||||
|
||||
|
||||
def test_trace_edges_endpoints_are_our_device() -> None:
|
||||
"""The first edge originates at us and the last edge returns to us (#3/#4)."""
|
||||
from meshterm.core.models import Hop
|
||||
|
||||
Reference in new issue
Block a user