diff --git a/docs/guide/features.md b/docs/guide/features.md index cacc783..36083de 100644 --- a/docs/guide/features.md +++ b/docs/guide/features.md @@ -17,7 +17,7 @@ This page lists each interactive screen with its scripted equivalent, where one | --- | --- | --- | | **πŸ’¬ Chat** | Live full-screen messages on channels and to contacts. The transcript scrolls, the input line stays at the bottom, and sent and received messages appear together. The rooms that you joined are also here. Each post shows under the name of its author. MeshTerm stores each message. Unread counts show in the menu header. | `meshterm chat send / history / list / listen` | | **πŸ“» Channels** | Create, join, reorder, mute, and share mesh channels. You can share a channel with a QR code or a `meshcore://` link. If you give a channel a **send scope**, its messages are flooded into one region. Only the repeaters that carry that region relay them. When you change to a new device, export all the channels to a file, then import the file on the new device. The **channel guide** lists the public channels that MeshTerm heard, also the channels that are not on your device, and adds one with Enter. | `meshterm channels list / add / join / import / share / clear / scope / export / guide` | -| **πŸ“Œ Rooms** | Each room server that your radio knows. A room server is a message board on a radio. It keeps the latest posts and sends you the posts that you missed. Join a room with its password. If a room does not answer, MeshTerm tells you what it knows and offers the next step to try. For example, it can send again by flood when the route that your radio learned is no longer good. | `meshterm rooms list / join / forget` | +| **πŸ“Œ Rooms** | Each room server that your radio knows. A room server is a message board on a radio. It keeps the latest posts and sends you the posts that you missed. Join a room with its password. If a room does not answer, MeshTerm tells you what it knows and offers the next step to try. For example, it can send again by flood when the route that your radio learned is no longer good. When the clock of a room goes back (often after a reboot), the room sends you no new post, because each new post looks older than the ones that you have. The node page of the room then says so and offers to ask for all its posts again. | `meshterm rooms list / join / forget` | | **πŸ“¨ Courier** | A store-and-forward outbox for contacts that you cannot reach yet. Queue a message. MeshTerm sends it when the contact is next heard, or at a time that you set. It tracks the acks, and it waits longer between each try (exponential backoff). | `meshterm courier queue / list / send / cancel / clear` | | **πŸ‘₯ Contacts** | This node and its known contacts. The screen shows a heat map of how recently each contact was heard, the counts of overheard packets, and the full public keys. The hash of each key is highlighted. The scripted listing shows only the contacts. For this node, run `meshterm info`, which gives much more detail. | `meshterm contacts` | diff --git a/meshterm/core/connection.py b/meshterm/core/connection.py index e84e467..8ce2494 100644 --- a/meshterm/core/connection.py +++ b/meshterm/core/connection.py @@ -1358,6 +1358,27 @@ class Device(ABC): write. """ + @abstractmethod + async def reset_room_sync(self, node: Contact) -> None: + """Make the next login to a room ask for all its posts again. + + At a login, the device sends the time of the newest post that it got from the room + (``sync_since``), and the room sends only the posts after that time. When the clock + of the room goes back, its new posts are all older than that time, and the room + never sends them. The device sets ``sync_since`` to zero only for a *new* contact, + and no command changes it for a contact that it has. Thus this method reads the + contact record, removes the contact, and adds the same record back: the same + flags, path, name, position, and advert time, with ``sync_since`` at zero. + + Args: + node: The room server. It must have a full public key. + + Raises: + ContactNotOnDeviceError: If the device has no contact with that key. + DeviceCommandError: If the device refused the removal, or the add. After a + failed add the contact is not on the device. The message says so. + """ + @abstractmethod async def get_tx_power(self) -> int | None: """Return the current TX power level, or ``None`` if it is not known.""" @@ -3459,6 +3480,35 @@ class MeshCoreDevice(Device): ) return True + async def reset_room_sync(self, node: Contact) -> None: # noqa: D102 - inherited docstring + mc = self._require() + pub = self._node_pubkey(node) # raises DeviceCommandError if it has no key + if len(pub) != 64: + raise DeviceCommandError(f"{node.name} is known only by a key prefix") + record = await self._contact_record(mc, pub) + if record is None: + raise ContactNotOnDeviceError(node) + result = await mc.commands.remove_contact(pub) + if result is not None and getattr(result, "is_error", lambda: False)(): + if error_code(result) == _ERR_NOT_FOUND: + raise ContactNotOnDeviceError(node) + raise DeviceCommandError( + f"couldn't remove {node.name} from the device: {reject_reason(result)}" + ) + # The same record goes back, as a new contact: the firmware gives a new contact a + # ``sync_since`` of zero. Two tries, because the contact is off the device between + # the two commands, and a lost reply must not leave it there. + reason = "" + for _attempt in range(2): + result = await mc.commands.add_contact(dict(record)) + if result is not None and not getattr(result, "is_error", lambda: False)(): + return + reason = reject_reason(result) + raise DeviceCommandError( + f"{node.name} was removed from the device but couldn't be added back ({reason}). " + "MeshTerm still remembers it: add it again from the Contacts list." + ) + async def _contact_record(self, mc, pub: str) -> dict | None: # noqa: ANN001 """The raw contact record that the device has for a full key, or ``None``. @@ -5017,6 +5067,14 @@ class MockDevice(Device): return True raise ContactNotOnDeviceError(node) + async def reset_room_sync(self, node: Contact) -> None: # noqa: D102 - inherited docstring + await asyncio.sleep(0) + # The simulator keeps no sync time. It refuses only a contact that it does not have, + # as the firmware does. + key = self._mock_key(node) + if key not in {self._mock_key(c) for c in self._contacts}: + raise ContactNotOnDeviceError(node) + async def get_tx_power(self) -> int | None: # noqa: D102 - inherited docstring return self._tx_power diff --git a/meshterm/persistence/repository.py b/meshterm/persistence/repository.py index e8767a1..d806741 100644 --- a/meshterm/persistence/repository.py +++ b/meshterm/persistence/repository.py @@ -2418,6 +2418,27 @@ class Repository: samples.append((bool(row["is_channel"]), row["peer"], row["text"], sent, arrived)) return samples + def newest_room_post(self, peer: str) -> datetime | None: + """The time of the newest post that a room sent us, by the clock of the room. + + The companion asks a room for each post newer than the newest post that it got, and + that time is close to this one (both are the highest post time seen). Thus, when the + clock of the room goes back past this time, the room never sends us its new posts + (refer to :func:`~meshterm.services.clock_offsets.room_posts_blocked`). + + Args: + peer: The key prefix of the room, as :meth:`record_chat_message` stores it. + + Returns: + The newest post time, or ``None`` when the room sent no post. + """ + row = self._conn.execute( + "SELECT MAX(created_at) AS newest FROM messages " + "WHERE is_channel = 0 AND outbound = 0 AND author IS NOT NULL AND peer = ?", + ((peer or "").lower(),), + ).fetchone() + return _as_when(row["newest"] if row else None) + def direct_message_bounds( self, peer: str | None, when: datetime, *, outbound: bool ) -> tuple[datetime | None, datetime | None]: diff --git a/meshterm/services/clock_offsets.py b/meshterm/services/clock_offsets.py index 7b5cca1..8fae253 100644 --- a/meshterm/services/clock_offsets.py +++ b/meshterm/services/clock_offsets.py @@ -27,10 +27,11 @@ from __future__ import annotations import statistics from collections.abc import Callable, Iterable from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timedelta from typing import TYPE_CHECKING from ..core.channels import split_channel_sender +from ..core.models import utcnow if TYPE_CHECKING: from ..persistence.repository import Repository @@ -156,6 +157,33 @@ def window_samples( return samples +def room_posts_blocked( + newest_post: datetime | None, offset: ClockOffset | None, tolerance_s: float +) -> bool: + """Whether a room can no longer send us its new posts, because its clock went back. + + A room stamps each post with its own clock. The companion asks for the posts newer than + the newest one that it got (its ``sync_since``), and the room sends only those + (``post_timestamp > sync_since`` in the room firmware). When the clock of the room goes + back, for example after a reboot with no clock source, each new post is older than + that time, and the room never sends it. The room keeps its posts only in memory, so a + reboot also loses the old ones. + + Args: + newest_post: The newest post time that we stored for the room (its clock). + offset: The clock offset of the room now, from its adverts and replies. + tolerance_s: The ``clock_tolerance_s`` preference. + + Returns: + ``True`` when the newest post is later than the clock of the room now, by more than + the tolerance. ``False`` when that is not so, or when a value is not known. + """ + if newest_post is None or offset is None: + return False + room_now = utcnow() + timedelta(seconds=offset.offset_s) + return newest_post > room_now + timedelta(seconds=tolerance_s) + + def node_offset( repo: Repository, node: str, *, name: str | None = None, key_of: KeyOf | None = None ) -> ClockOffset | None: diff --git a/meshterm/ui/node_detail_screen.py b/meshterm/ui/node_detail_screen.py index 25d2997..bebb66f 100644 --- a/meshterm/ui/node_detail_screen.py +++ b/meshterm/ui/node_detail_screen.py @@ -113,7 +113,7 @@ from ..core.connection import ContactNotOnDeviceError, DeviceCommandError from ..core.geo import haversine_km, usable_fix from ..core.models import NODE_TYPE_LABELS, NODE_TYPE_REPEATER, NODE_TYPE_ROOM, Contact, utcnow from ..platforms import Platform, on_platform -from ..services.clock_offsets import node_offset +from ..services.clock_offsets import node_offset, room_posts_blocked from . import menus from .mapcanvas import RGB from .marks import MANAGED_ICON, SELF_MARK @@ -1543,6 +1543,8 @@ async def open_node_detail( # -- the info block (the identity, how the node was heard, and where it is). The routes # go into the Routes tab. info_rows: list[tuple[str, Text]] = [] + # Whether a room sends us no new post, because its clock went back (set below). + room_blocked = False if key: info_rows.append(("key", highlighted_hash(key, prefix_bytes, known=bool(name)))) else: @@ -1580,6 +1582,13 @@ async def open_node_detail( # The device drops each advert of the node until this date (refer to # ``Contact.adverts_refused_until``). The row says why the node looks silent. info_rows.append(("adverts", _refused_text(refused))) + # A room whose clock went back past its newest post sends us no new post (refer to + # ``room_posts_blocked``). The row says why the board looks quiet. + if node_type == NODE_TYPE_ROOM and node_id: + newest_post = ctx.repo.newest_room_post(node_id) + if room_posts_blocked(newest_post, clock_offset, ctx.preferences.clock_tolerance_s): + room_blocked = True + info_rows.append(("posts", _blocked_text(newest_post))) if lat is not None and lon is not None: info_rows.append(("where", _range_text(lat, lon, self_lat, self_lon))) # The regions of a repeater: what it was last heard to relay, from the region store (its @@ -1679,6 +1688,10 @@ async def open_node_detail( # ``Device.accept_adverts_again``). It writes to the device, so it confirms first. if refused is not None and not you and contact is not None and contact.public_key: info_actions.append(_Action("adverts", "πŸ“‘", "warn", "Accept its adverts again…")) + # The room sends no new post (refer to ``room_blocked``). The repair adds the contact back + # with a sync time of zero, then logs in again (refer to ``Device.reset_room_sync``). + if room_blocked and contact is not None and contact.public_key: + info_actions.append(_Action("resync", "πŸ“Œ", "warn", "Ask for all its posts again…")) # -- contact management: the last group of the page, and the only actions that end the # visit. None of them shows when the caller opened the page to look, not to act (refer # to ``manage``), except the lock when the caller lets the page lock (refer to @@ -1794,6 +1807,14 @@ async def open_node_detail( screen.replace_info_row( "adverts", Text("accepted again β€” the next advert updates it", "ok") ) + elif action == "resync": + assert contact is not None # offered only for a room with a full key + if await _resync_room(ctx, contact, label): + base_actions = [a for a in base_actions if a.key != "resync"] + screen.replace_info_actions(management(locked)) + screen.replace_info_row( + "posts", Text("asked again β€” the room sends what it still has", "ok") + ) elif action == "restore": # A restore writes the contact back, and it ends the visit for the same reason # as a delete. The page was opened from the Archived list, and the row that @@ -2160,6 +2181,71 @@ async def _accept_adverts(ctx: AppContext, contact: Contact, label: str) -> bool return True +async def _resync_room(ctx: AppContext, contact: Contact, label: str) -> bool: + """Confirm, reset the post sync of a room on the device, then log in to the room again. + + The room sends only the posts after the sync time that the device gives at a login. + After its clock went back, each new post is older than that time (refer to + :func:`~meshterm.services.clock_offsets.room_posts_blocked`). The device resets the + time only for a new contact, so the repair removes the contact and adds the same + record back (:meth:`~meshterm.core.connection.Device.reset_room_sync`). Then a login + with the stored password asks for each post that the room still has. The room keeps + its posts only in memory, so a reboot lost the older ones. The chat drops the posts + that it already has. + + Args: + ctx: The shared application context (interactive menu). + contact: The room server, with a full key. + label: The name of the room on the page. + + Returns: + ``True`` when the device has the reset sync time. ``False`` when the user cancelled + or the device refused. A failure is shown. + """ + from ..core.models import LoginResult + from .surface import TuiUi + + assert isinstance(ctx.ui, TuiUi) # open_node_detail makes sure of this + session = ctx.ui.session + if not await ctx.ui.dialog( + f"{label}'s clock went back, so it sends no post newer than the ones you have. " + "Remove it from this device and add it back with the same settings, then log in " + "again to get every post it still has?", + [("Cancel", False), ("Ask again", True)], + title="Room posts", + default=1, + danger=True, + ): + return False + device = await ctx.device() + try: + await device.reset_room_sync(contact) + except Exception as exc: # noqa: BLE001 - a refused write is reported, not raised + ctx.log.warning("rooms: sync reset failed for %s: %s", contact.name, exc) + await session.message_dialog(Text(f"βœ— {exc}", style="err"), title="Room posts") + return False + ctx.devstate.invalidate_contacts() + password = ctx.rooms.password(contact) + if password is None: + await session.message_dialog( + Text(f"Log in to {label} from Rooms to get its posts.", style="warn"), + title="Room posts", + ) + return True + try: + async with ctx.ui.busy_overlay(): + login = await ctx.rooms.login(contact, password) + except Exception as exc: # noqa: BLE001 - the reset is done. Only the login failed + ctx.log.warning("rooms: login after the sync reset failed for %s: %s", contact.name, exc) + login = None + if login is None or login.result is not LoginResult.ACCEPTED: + await session.message_dialog( + Text(f"No login yet β€” open {label} in Rooms to log in and get its posts.", "warn"), + title="Room posts", + ) + return True + + #: Why a repeater may not answer the regions request, in the words that the page uses. _REGIONS_REACH = "it answers only a neighbour, or over a known route" @@ -2296,6 +2382,14 @@ def _clock_text(offset, tolerance_s: float) -> Text: # noqa: ANN001 - a ClockOf return text +def _blocked_text(newest_post: datetime | None) -> Text: + """The value of the ``posts`` row of a room whose clock went back past its newest post.""" + text = Text("blocked β€” its clock went back", style="warn") + if newest_post is not None: + text.append(f" Β· newest {newest_post.astimezone():%Y-%m-%d %H:%M}", style="muted") + return text + + def _refused_text(until: datetime) -> Text: """The value of the ``adverts`` row: the date until which the device refuses them.""" text = Text(f"refused until {until.astimezone():%Y-%m-%d}", style="warn") diff --git a/tests/test_room_sync.py b/tests/test_room_sync.py new file mode 100644 index 0000000..0329907 --- /dev/null +++ b/tests/test_room_sync.py @@ -0,0 +1,143 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A room whose clock went back: the test that finds it, and the repair of the post sync. + +A room stamps each post with its own clock, and it sends us only the posts after the newest +one that we got. When its clock goes back, it sends no new post. The repair adds the room +contact back as a new contact, whose sync time the firmware sets to zero. +""" + +from __future__ import annotations + +import asyncio +from datetime import timedelta +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from meshterm.core.connection import ( + ContactNotOnDeviceError, + DeviceCommandError, + MeshCoreDevice, + MockDevice, +) +from meshterm.core.models import ( + CLOCK_TOLERANCE_S, + NODE_TYPE_ROOM, + ChatMessage, + Contact, + Message, + utcnow, +) +from meshterm.persistence.repository import Repository +from meshterm.services.clock_offsets import ClockOffset, room_posts_blocked + +ROOM = "ab" * 32 + + +def _offset(seconds: float) -> ClockOffset: + return ClockOffset(ROOM[:12], True, seconds, 3, utcnow()) + + +def test_a_room_whose_clock_went_back_past_its_newest_post_is_blocked() -> None: + """The newest post is later than the clock of the room now: the room sends nothing new.""" + newest = utcnow() - timedelta(minutes=10) + assert room_posts_blocked(newest, _offset(-3 * 86400), CLOCK_TOLERANCE_S) + assert not room_posts_blocked(newest, _offset(5), CLOCK_TOLERANCE_S) + assert not room_posts_blocked(newest, None, CLOCK_TOLERANCE_S) # no offset: no claim + assert not room_posts_blocked(None, _offset(-3 * 86400), CLOCK_TOLERANCE_S) + + +def test_the_newest_post_counts_only_posts(tmp_path: Path) -> None: + """A command reply of the room and our own posts are not posts that the room sent us.""" + repo = Repository(tmp_path / "rooms.db") + now = utcnow().replace(microsecond=0) + post = Message( + text="hello board", + sender=ROOM[:12], + sender_timestamp=now - timedelta(hours=2), + received_at=now, + author="c0ffee12", + ) + reply = Message( + text="OK", sender=ROOM[:12], sender_timestamp=now + timedelta(days=9), received_at=now + ) + repo.record_chat_message(ChatMessage.from_message(post)) + repo.record_chat_message(ChatMessage.from_message(reply)) + ours = ChatMessage(text="mine", outbound=True, peer=ROOM[:12], created_at=now) + repo.record_chat_message(ours) + assert repo.newest_room_post(ROOM[:12]) == now - timedelta(hours=2) + assert repo.newest_room_post("cd" * 6) is None + repo.close() + + +class _Commands: + """A stand-in for the command set of the library: one stored record and each command.""" + + def __init__(self, record: dict, *, add_fails: int = 0) -> None: + self.record = record + self.calls: list[tuple[str, object]] = [] + self._add_fails = add_fails + + async def get_contact_by_key(self, pubkey: bytes): # noqa: ANN201 + return SimpleNamespace(payload=dict(self.record), is_error=lambda: False) + + async def remove_contact(self, pub: str): # noqa: ANN201 + self.calls.append(("remove", pub)) + return SimpleNamespace(payload={}, is_error=lambda: False) + + async def add_contact(self, record: dict): # noqa: ANN201 + self.calls.append(("add", record)) + failed = self._add_fails > 0 + self._add_fails -= 1 + return SimpleNamespace(payload={"reason": "table full"}, is_error=lambda: failed) + + +def _record() -> dict: + return { + "public_key": ROOM, + "type": NODE_TYPE_ROOM, + "flags": 0, + "out_path_len": 1, + "out_path_hash_mode": 0, + "out_path": "3d", + "adv_name": "MTL BBS", + "last_advert": int(utcnow().timestamp()) - 3600, + "adv_lat": 45.5, + "adv_lon": -73.6, + } + + +def _device(commands: _Commands) -> MeshCoreDevice: + device = MeshCoreDevice(port="mock") + device._mc = SimpleNamespace(commands=commands) + return device + + +def test_the_reset_removes_the_room_and_adds_the_same_record_back() -> None: + """Remove, then add: the record goes back as the device had it, as a new contact.""" + commands = _Commands(_record()) + asyncio.run(_device(commands).reset_room_sync(Contact(name="MTL BBS", public_key=ROOM))) + assert [name for name, _ in commands.calls] == ["remove", "add"] + assert commands.calls[1][1] == _record() | {"last_advert": commands.record["last_advert"]} + + +def test_a_failed_add_back_tries_again_then_says_the_room_is_off_the_device() -> None: + """One lost reply is tried again. Two failures raise, and the message says what is gone.""" + commands = _Commands(_record(), add_fails=1) + asyncio.run(_device(commands).reset_room_sync(Contact(name="MTL BBS", public_key=ROOM))) + assert [name for name, _ in commands.calls] == ["remove", "add", "add"] + + commands = _Commands(_record(), add_fails=2) + with pytest.raises(DeviceCommandError, match="removed from the device"): + asyncio.run(_device(commands).reset_room_sync(Contact(name="MTL BBS", public_key=ROOM))) + + +def test_the_simulator_refuses_a_room_that_it_does_not_have() -> None: + """The simulator keeps no sync time, but it refuses an unknown contact as firmware does.""" + device = MockDevice() + room = Contact(name="Room", public_key=ROOM, key_prefix=ROOM[:12], node_type=NODE_TYPE_ROOM) + with pytest.raises(ContactNotOnDeviceError): + asyncio.run(device.reset_room_sync(room)) + asyncio.run(device.add_contact(room)) + asyncio.run(device.reset_room_sync(room))