diff --git a/meshterm/core/connection.py b/meshterm/core/connection.py index d75dfcb..240310c 100644 --- a/meshterm/core/connection.py +++ b/meshterm/core/connection.py @@ -104,6 +104,12 @@ def charging_from_battery_level_status(data: bytes) -> Optional[bool]: #: drain loop (seconds). _MESSAGE_GET_TIMEOUT_S = 5.0 +#: How many uncorrelated frames one :meth:`MeshCoreDevice.admin_login` may quote in its log +#: line. They are evidence for reading a failure afterwards, not a record to keep, and a busy +#: mesh can push a great many through the window — a handful names the fault, and the rest +#: would only bury the outcome they sit beside. +_LOGIN_STRAY_LOG_CAP = 4 + #: How long a graceful ``meshcore`` client teardown may take before it is abandoned and the #: transport is force-closed instead (seconds). A healthy disconnect completes in well under a #: second; the bound exists because the library's dispatcher shutdown can deadlock — its @@ -1638,33 +1644,71 @@ class MeshCoreDevice(Device): # Both go away by owning the wait. ``send_login_sync`` is still what transmits (it is # the library's supported path, and its own listener is harmless — the dispatcher # delivers to every matching subscription), but ours is armed first and outlives it: - # when it gives up early we keep waiting for the rest of a budget sized to the route, - # the way a trace to the same node would be. + # when it gives up early, the node still gets a full budget of its own, sized to the + # route the way a trace to the same node would be, and counted from the send rather + # than from the queuing — see the wait below for why that distinction is the fix. answer: asyncio.Future = loop.create_future() + # Frames that landed during the exchange but are not our verdict. Kept only to be + # logged beside the outcome: a "no reply" with one of these next to it is a + # different fault from a node that stayed silent, and nothing else would show it. + stray: list[str] = [] + + def note(what: str, event) -> None: # noqa: ANN001 - meshcore Event + if len(stray) < _LOGIN_STRAY_LOG_CAP: + stray.append(f"{what} {getattr(event, 'payload', None)!r}") def on_answer(event) -> None: # noqa: ANN001 - meshcore Event - if not answer.done() and self._refers_to(event, pub): + if answer.done(): + return + if self._refers_to(event, pub): answer.set_result(event) + else: + # An answer naming some other node — two admin flows can overlap. Not ours + # to act on, but worth saying we heard it: a login that reports silence with + # one of these logged is a key prefix that did not match, not a quiet node. + note("login frame for another node:", event) + + def on_local_error(event) -> None: # noqa: ANN001 - meshcore Event + # The one failure that never reaches the mesh: the companion refusing to send. + # ``send_login_sync`` erases it — it turns its own ERROR into a bare ``None``, + # indistinguishable from an acknowledgement that was merely slow — so the reason + # is only recoverable by listening for the frame. Uncorrelated (an ERROR carries + # no request id and may belong to another command in flight), so it is evidence + # for the log and never a verdict. + note("companion error:", event) subscriptions = [ mc.subscribe(EventType.LOGIN_SUCCESS, on_answer), mc.subscribe(EventType.LOGIN_FAILED, on_answer), + mc.subscribe(EventType.ERROR, on_local_error), ] # A login is a round trip along the contact's route and back, which is the shape a # trace budget already describes; the stored route is one-way, so the wire carries - # twice its hops. A contact we hold no route for floods, and takes the flood budget. - budget = trace_timeout(2 * len(node.route_hops or ())) + # twice its hops. Knowing the route only ever buys *more* patience, never less: an + # admin exchange is heavier than a trace's single small packet, so a contact we hold + # a short route for must not be given a narrower window than the routeless one + # beside it — which is exactly what ``trace_timeout(0)``, the flood budget, is. + hops = 2 * len(node.route_hops or ()) + budget = max(trace_timeout(hops), trace_timeout(0)) started = loop.time() try: sent = await mc.commands.send_login_sync(pub, password) + # The budget times the *node's* answer, so it is spent from the moment the + # request is on the air — not from the moment we began queuing it. Everything + # before that belongs to the companion, and it can be most of a minute: + # ``send_login_sync`` first waits its turn on the library's mesh-request lock, + # which every telemetry poll and courier retry holds for a whole round trip, and + # then blocks until a MSG_SENT it correlates by event type alone arrives. Timing + # the repeater from before all that is what made the last fix a longer way of + # failing at the same ten seconds: the wait was widened, then handed a window + # some other command had already spent. + sent_at = loop.time() if not answer.done(): - remaining = budget - (loop.time() - started) - if remaining > 0: - # Shielded: a timeout here must leave the future readable, not cancel it. - try: - await asyncio.wait_for(asyncio.shield(answer), remaining) - except asyncio.TimeoutError: - pass + # Shielded: a timeout here must leave the future readable, not cancel it. + try: + await asyncio.wait_for(asyncio.shield(answer), budget) + except asyncio.TimeoutError: + pass finally: for subscription in subscriptions: subscription.unsubscribe() @@ -1677,9 +1721,20 @@ class MeshCoreDevice(Device): # the direction this whole method is trying to fail in. event = sent etype = getattr(event, "type", None) + # Three numbers, because a no-reply has three different causes and they are told + # apart only by where the time went: a send that took seconds means the request sat + # behind another command, a send that returned at once with no acknowledgement means + # the companion never put it on the air, and a full budget spent after a clean send + # means the node really did stay silent. _log.debug( - "admin login to %s: budget=%.1fs -> %s after %.0fms", - node.name, budget, etype, (loop.time() - started) * 1000.0, + "admin login to %s: send %s in %.0fms, budget=%.1fs -> %s after %.0fms%s", + node.name, + "acknowledged" if sent is not None else "unacknowledged", + (sent_at - started) * 1000.0, + budget, + etype, + (loop.time() - sent_at) * 1000.0, + f" [also heard: {'; '.join(stray)}]" if stray else "", ) if etype is EventType.LOGIN_SUCCESS: return LoginResult.ACCEPTED diff --git a/tests/test_admin_login.py b/tests/test_admin_login.py index f4c2fd1..6b6656e 100644 --- a/tests/test_admin_login.py +++ b/tests/test_admin_login.py @@ -139,18 +139,23 @@ class _FakeCommands: listener outliving it can catch. """ - def __init__(self, mc, *, answer=None, late=None, late_delay: float = 0.0) -> None: # noqa: ANN001 + def __init__( # noqa: ANN003 + self, mc, *, answer=None, late=None, late_delay: float = 0.0, send_delay: float = 0.0 + ) -> None: # noqa: ANN001 self._mc = mc self._answer = answer self._late = late self._late_delay = late_delay + # How long the send takes to return — the library's mesh-request lock, then a + # MSG_SENT it correlates by event type alone. None of it is the node's time. + self._send_delay = send_delay async def send_login_sync(self, pubkey, password): # noqa: ANN001 from meshcore import EventType self._mc.sent.append((pubkey, password)) - self._mc.armed_before_send = len(self._mc.subscriptions) - await asyncio.sleep(0) + self._mc.armed = [sub.event_type for sub in self._mc.subscriptions] + await asyncio.sleep(self._send_delay) if self._answer is not None: self._mc.dispatch(self._answer) if self._answer.type is EventType.LOGIN_SUCCESS: @@ -167,7 +172,7 @@ class _FakeMeshCore: def __init__(self, **commands) -> None: # noqa: ANN003 self.subscriptions: list[_Subscription] = [] self.sent: list[tuple] = [] - self.armed_before_send = 0 + self.armed: list = [] # the event types subscribed by the time the send went out self.commands = _FakeCommands(self, **commands) def subscribe(self, event_type, callback, attribute_filters=None): # noqa: ANN001 @@ -273,12 +278,46 @@ def test_an_answer_meant_for_a_different_node_is_not_ours(quick_budget) -> None: def test_the_listener_is_armed_before_the_request_goes_out(quick_budget) -> None: # noqa: ANN001 - """Both frames, both subscribed — an answer landing mid-send has somewhere to go.""" + """Both answers subscribed before the send — one landing mid-send has somewhere to go.""" + from meshcore import EventType + mc = _FakeMeshCore(answer=_login_success()) asyncio.run(_device(mc).admin_login(_NODE, "hunter2")) - assert mc.armed_before_send == 2 # LOGIN_SUCCESS and LOGIN_FAILED + assert EventType.LOGIN_SUCCESS in mc.armed + assert EventType.LOGIN_FAILED in mc.armed + + +def test_a_companion_error_is_watched_for_because_the_library_erases_it() -> None: + """``send_login_sync`` turns its own ERROR into a bare ``None``, so the reason is lost. + + A companion that refuses to send and a node that never answers are the same + :attr:`LoginResult.NO_REPLY` — correctly, since neither says anything about the + password — but they are opposite faults to go and fix, and only the frame tells them + apart. It is uncorrelated, so it is logged and never read as a verdict. + """ + from meshcore import EventType + + mc = _FakeMeshCore(answer=_login_success()) + + asyncio.run(_device(mc).admin_login(_NODE, "hunter2")) + + assert EventType.ERROR in mc.armed + + +def test_the_node_gets_its_whole_budget_even_when_the_send_was_slow(quick_budget) -> None: # noqa: ANN001 + """THE regression this fix is for: the repeater was charged for the queue ahead of it. + + The budget times the *node's* answer, but it used to be counted from before + ``send_login_sync`` — which waits its turn on the library's mesh-request lock (held by + every telemetry poll and courier retry for a whole round trip) before it transmits at + all. A send that outlasted the budget therefore left the node no window whatsoever, and + a repeater that was up, listening and about to answer was recorded as silent. + """ + mc = _FakeMeshCore(late=_login_success(), late_delay=0.01, send_delay=0.2) + + assert asyncio.run(_device(mc).admin_login(_NODE, "hunter2")) is LoginResult.ACCEPTED def test_the_listeners_are_released_even_when_the_send_blows_up() -> None: @@ -335,10 +374,43 @@ def test_the_reply_budget_is_sized_to_the_route_the_login_has_to_walk() -> None: finally: trace_runner.trace_timeout = real - assert asked == [4, 0] # out and back over two hops; then a contact with no route + # Each login asks twice: for its own walk, and for the routeless floor (hops ``0``) + # it may never be given less than. + assert asked == [4, 0, 0, 0] assert real(4) > real(2) > real(1) # and the budget grows with the walk +def test_a_known_short_route_never_buys_less_patience_than_no_route_at_all() -> None: + """Knowing where a node is must not make us give up on it sooner. + + A trace budget is sized for one small packet on an explicit path; a login is an admin + exchange out and back. Sized literally, a contact with a one-hop route would be given a + narrower window than the routeless contact beside it that floods — so the flood budget + is the floor, and the route only ever widens it. + """ + import meshterm.services.trace_runner as trace_runner + + budgets: list[float] = [] + real = trace_runner.trace_timeout + + def spy(hops): # noqa: ANN001 + # The real shape, scaled down so the no-reply this provokes is not a real wait. + budget = real(hops) / 1000 + budgets.append(budget) + return budget + + trace_runner.trace_timeout = spy + try: + near = Contact(name="YUL-Near", public_key="a1b2c3d4" * 8, route_hops=("3d",)) + asyncio.run(_device(_FakeMeshCore()).admin_login(near, "hunter2")) + finally: + trace_runner.trace_timeout = real + + walked, floor = budgets + assert walked < floor # the literal trace sizing really is the narrower of the two + assert real(2) < real(0) == trace_runner.TRACE_TIMEOUT_FLOOD_S # and so at full scale + + # --- the simulator speaks the same three answers --------------------------------------