From 4e3208b1c25e442a33663e98a51c9a0e5915d17c Mon Sep 17 00:00:00 2001 From: Jean-Pierre Martineau Date: Mon, 24 Aug 2026 20:18:04 -0400 Subject: [PATCH] Admin login: the repeater is not charged for the queue ahead of it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remote administration still failed. My last two commits classified the failure correctly and fixed the credential policy, but the login itself kept coming back as no reply from repeaters that were up, traced, and holding the right password. The wait I added was counted from the wrong moment. `budget` is the *node's* answer window, but the clock started before `send_login_sync`, and almost none of what that call spends is the node's: it first waits its turn on meshcore's mesh-request lock — which every telemetry poll, status read and courier retry holds for a whole round trip — and only then blocks until a MSG_SENT the library correlates by event type alone arrives. Whatever that came to was subtracted from the repeater's window, so the fix read as "wait longer" and behaved as "wait the same ten seconds, minus the queue". A send that outlasted the budget left the node no window at all, and the log bears the shape out: both attempts to YUL-UpperSalaberry ended at 9985 ms and 10015 ms — the budget exactly, spent from before the request was on the air. So the answer is timed from the send, the way `run_trace` times its reply. And the route sizing is a floor rather than a verdict: a login is an admin exchange out and back, not a trace's single small packet, so a contact we hold a one-hop route for was being given a narrower window (7.2 s) than the routeless contact beside it that floods (10 s). Knowing where a node is must not make us give up on it sooner. The rest is making the next failure readable, because this one was not. A no-reply has three causes that the old line could not tell apart, so it now reports where the time went — the send's own duration, the budget, and how long the node had — and quotes the frames it heard but could not act on: a LOGIN_SUCCESS/LOGIN_FAILED naming some other node (a key prefix that did not match, not a quiet node), and a companion ERROR, which is the one failure that never reaches the mesh and which `send_login_sync` erases into the same bare `None`. Uncorrelated, so it is evidence in the log and never a verdict. Co-Authored-By: Claude Opus 5 (1M context) --- meshterm/core/connection.py | 83 +++++++++++++++++++++++++++++------ tests/test_admin_login.py | 86 ++++++++++++++++++++++++++++++++++--- 2 files changed, 148 insertions(+), 21 deletions(-) diff --git a/meshterm/core/connection.py b/meshterm/core/connection.py index d75dfcb..240310c 100644 --- a/meshterm/core/connection.py +++ b/meshterm/core/connection.py @@ -104,6 +104,12 @@ def charging_from_battery_level_status(data: bytes) -> Optional[bool]: #: drain loop (seconds). _MESSAGE_GET_TIMEOUT_S = 5.0 +#: How many uncorrelated frames one :meth:`MeshCoreDevice.admin_login` may quote in its log +#: line. They are evidence for reading a failure afterwards, not a record to keep, and a busy +#: mesh can push a great many through the window — a handful names the fault, and the rest +#: would only bury the outcome they sit beside. +_LOGIN_STRAY_LOG_CAP = 4 + #: How long a graceful ``meshcore`` client teardown may take before it is abandoned and the #: transport is force-closed instead (seconds). A healthy disconnect completes in well under a #: second; the bound exists because the library's dispatcher shutdown can deadlock — its @@ -1638,33 +1644,71 @@ class MeshCoreDevice(Device): # Both go away by owning the wait. ``send_login_sync`` is still what transmits (it is # the library's supported path, and its own listener is harmless — the dispatcher # delivers to every matching subscription), but ours is armed first and outlives it: - # when it gives up early we keep waiting for the rest of a budget sized to the route, - # the way a trace to the same node would be. + # when it gives up early, the node still gets a full budget of its own, sized to the + # route the way a trace to the same node would be, and counted from the send rather + # than from the queuing — see the wait below for why that distinction is the fix. answer: asyncio.Future = loop.create_future() + # Frames that landed during the exchange but are not our verdict. Kept only to be + # logged beside the outcome: a "no reply" with one of these next to it is a + # different fault from a node that stayed silent, and nothing else would show it. + stray: list[str] = [] + + def note(what: str, event) -> None: # noqa: ANN001 - meshcore Event + if len(stray) < _LOGIN_STRAY_LOG_CAP: + stray.append(f"{what} {getattr(event, 'payload', None)!r}") def on_answer(event) -> None: # noqa: ANN001 - meshcore Event - if not answer.done() and self._refers_to(event, pub): + if answer.done(): + return + if self._refers_to(event, pub): answer.set_result(event) + else: + # An answer naming some other node — two admin flows can overlap. Not ours + # to act on, but worth saying we heard it: a login that reports silence with + # one of these logged is a key prefix that did not match, not a quiet node. + note("login frame for another node:", event) + + def on_local_error(event) -> None: # noqa: ANN001 - meshcore Event + # The one failure that never reaches the mesh: the companion refusing to send. + # ``send_login_sync`` erases it — it turns its own ERROR into a bare ``None``, + # indistinguishable from an acknowledgement that was merely slow — so the reason + # is only recoverable by listening for the frame. Uncorrelated (an ERROR carries + # no request id and may belong to another command in flight), so it is evidence + # for the log and never a verdict. + note("companion error:", event) subscriptions = [ mc.subscribe(EventType.LOGIN_SUCCESS, on_answer), mc.subscribe(EventType.LOGIN_FAILED, on_answer), + mc.subscribe(EventType.ERROR, on_local_error), ] # A login is a round trip along the contact's route and back, which is the shape a # trace budget already describes; the stored route is one-way, so the wire carries - # twice its hops. A contact we hold no route for floods, and takes the flood budget. - budget = trace_timeout(2 * len(node.route_hops or ())) + # twice its hops. Knowing the route only ever buys *more* patience, never less: an + # admin exchange is heavier than a trace's single small packet, so a contact we hold + # a short route for must not be given a narrower window than the routeless one + # beside it — which is exactly what ``trace_timeout(0)``, the flood budget, is. + hops = 2 * len(node.route_hops or ()) + budget = max(trace_timeout(hops), trace_timeout(0)) started = loop.time() try: sent = await mc.commands.send_login_sync(pub, password) + # The budget times the *node's* answer, so it is spent from the moment the + # request is on the air — not from the moment we began queuing it. Everything + # before that belongs to the companion, and it can be most of a minute: + # ``send_login_sync`` first waits its turn on the library's mesh-request lock, + # which every telemetry poll and courier retry holds for a whole round trip, and + # then blocks until a MSG_SENT it correlates by event type alone arrives. Timing + # the repeater from before all that is what made the last fix a longer way of + # failing at the same ten seconds: the wait was widened, then handed a window + # some other command had already spent. + sent_at = loop.time() if not answer.done(): - remaining = budget - (loop.time() - started) - if remaining > 0: - # Shielded: a timeout here must leave the future readable, not cancel it. - try: - await asyncio.wait_for(asyncio.shield(answer), remaining) - except asyncio.TimeoutError: - pass + # Shielded: a timeout here must leave the future readable, not cancel it. + try: + await asyncio.wait_for(asyncio.shield(answer), budget) + except asyncio.TimeoutError: + pass finally: for subscription in subscriptions: subscription.unsubscribe() @@ -1677,9 +1721,20 @@ class MeshCoreDevice(Device): # the direction this whole method is trying to fail in. event = sent etype = getattr(event, "type", None) + # Three numbers, because a no-reply has three different causes and they are told + # apart only by where the time went: a send that took seconds means the request sat + # behind another command, a send that returned at once with no acknowledgement means + # the companion never put it on the air, and a full budget spent after a clean send + # means the node really did stay silent. _log.debug( - "admin login to %s: budget=%.1fs -> %s after %.0fms", - node.name, budget, etype, (loop.time() - started) * 1000.0, + "admin login to %s: send %s in %.0fms, budget=%.1fs -> %s after %.0fms%s", + node.name, + "acknowledged" if sent is not None else "unacknowledged", + (sent_at - started) * 1000.0, + budget, + etype, + (loop.time() - sent_at) * 1000.0, + f" [also heard: {'; '.join(stray)}]" if stray else "", ) if etype is EventType.LOGIN_SUCCESS: return LoginResult.ACCEPTED diff --git a/tests/test_admin_login.py b/tests/test_admin_login.py index f4c2fd1..6b6656e 100644 --- a/tests/test_admin_login.py +++ b/tests/test_admin_login.py @@ -139,18 +139,23 @@ class _FakeCommands: listener outliving it can catch. """ - def __init__(self, mc, *, answer=None, late=None, late_delay: float = 0.0) -> None: # noqa: ANN001 + def __init__( # noqa: ANN003 + self, mc, *, answer=None, late=None, late_delay: float = 0.0, send_delay: float = 0.0 + ) -> None: # noqa: ANN001 self._mc = mc self._answer = answer self._late = late self._late_delay = late_delay + # How long the send takes to return — the library's mesh-request lock, then a + # MSG_SENT it correlates by event type alone. None of it is the node's time. + self._send_delay = send_delay async def send_login_sync(self, pubkey, password): # noqa: ANN001 from meshcore import EventType self._mc.sent.append((pubkey, password)) - self._mc.armed_before_send = len(self._mc.subscriptions) - await asyncio.sleep(0) + self._mc.armed = [sub.event_type for sub in self._mc.subscriptions] + await asyncio.sleep(self._send_delay) if self._answer is not None: self._mc.dispatch(self._answer) if self._answer.type is EventType.LOGIN_SUCCESS: @@ -167,7 +172,7 @@ class _FakeMeshCore: def __init__(self, **commands) -> None: # noqa: ANN003 self.subscriptions: list[_Subscription] = [] self.sent: list[tuple] = [] - self.armed_before_send = 0 + self.armed: list = [] # the event types subscribed by the time the send went out self.commands = _FakeCommands(self, **commands) def subscribe(self, event_type, callback, attribute_filters=None): # noqa: ANN001 @@ -273,12 +278,46 @@ def test_an_answer_meant_for_a_different_node_is_not_ours(quick_budget) -> None: def test_the_listener_is_armed_before_the_request_goes_out(quick_budget) -> None: # noqa: ANN001 - """Both frames, both subscribed — an answer landing mid-send has somewhere to go.""" + """Both answers subscribed before the send — one landing mid-send has somewhere to go.""" + from meshcore import EventType + mc = _FakeMeshCore(answer=_login_success()) asyncio.run(_device(mc).admin_login(_NODE, "hunter2")) - assert mc.armed_before_send == 2 # LOGIN_SUCCESS and LOGIN_FAILED + assert EventType.LOGIN_SUCCESS in mc.armed + assert EventType.LOGIN_FAILED in mc.armed + + +def test_a_companion_error_is_watched_for_because_the_library_erases_it() -> None: + """``send_login_sync`` turns its own ERROR into a bare ``None``, so the reason is lost. + + A companion that refuses to send and a node that never answers are the same + :attr:`LoginResult.NO_REPLY` — correctly, since neither says anything about the + password — but they are opposite faults to go and fix, and only the frame tells them + apart. It is uncorrelated, so it is logged and never read as a verdict. + """ + from meshcore import EventType + + mc = _FakeMeshCore(answer=_login_success()) + + asyncio.run(_device(mc).admin_login(_NODE, "hunter2")) + + assert EventType.ERROR in mc.armed + + +def test_the_node_gets_its_whole_budget_even_when_the_send_was_slow(quick_budget) -> None: # noqa: ANN001 + """THE regression this fix is for: the repeater was charged for the queue ahead of it. + + The budget times the *node's* answer, but it used to be counted from before + ``send_login_sync`` — which waits its turn on the library's mesh-request lock (held by + every telemetry poll and courier retry for a whole round trip) before it transmits at + all. A send that outlasted the budget therefore left the node no window whatsoever, and + a repeater that was up, listening and about to answer was recorded as silent. + """ + mc = _FakeMeshCore(late=_login_success(), late_delay=0.01, send_delay=0.2) + + assert asyncio.run(_device(mc).admin_login(_NODE, "hunter2")) is LoginResult.ACCEPTED def test_the_listeners_are_released_even_when_the_send_blows_up() -> None: @@ -335,10 +374,43 @@ def test_the_reply_budget_is_sized_to_the_route_the_login_has_to_walk() -> None: finally: trace_runner.trace_timeout = real - assert asked == [4, 0] # out and back over two hops; then a contact with no route + # Each login asks twice: for its own walk, and for the routeless floor (hops ``0``) + # it may never be given less than. + assert asked == [4, 0, 0, 0] assert real(4) > real(2) > real(1) # and the budget grows with the walk +def test_a_known_short_route_never_buys_less_patience_than_no_route_at_all() -> None: + """Knowing where a node is must not make us give up on it sooner. + + A trace budget is sized for one small packet on an explicit path; a login is an admin + exchange out and back. Sized literally, a contact with a one-hop route would be given a + narrower window than the routeless contact beside it that floods — so the flood budget + is the floor, and the route only ever widens it. + """ + import meshterm.services.trace_runner as trace_runner + + budgets: list[float] = [] + real = trace_runner.trace_timeout + + def spy(hops): # noqa: ANN001 + # The real shape, scaled down so the no-reply this provokes is not a real wait. + budget = real(hops) / 1000 + budgets.append(budget) + return budget + + trace_runner.trace_timeout = spy + try: + near = Contact(name="YUL-Near", public_key="a1b2c3d4" * 8, route_hops=("3d",)) + asyncio.run(_device(_FakeMeshCore()).admin_login(near, "hunter2")) + finally: + trace_runner.trace_timeout = real + + walked, floor = budgets + assert walked < floor # the literal trace sizing really is the narrower of the two + assert real(2) < real(0) == trace_runner.TRACE_TIMEOUT_FLOOD_S # and so at full scale + + # --- the simulator speaks the same three answers --------------------------------------