diff --git a/CLAUDE.md b/CLAUDE.md index f84bb04..4f76fbb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -443,7 +443,9 @@ as a grouped list does. Filling a page in is editing its `.md`; no Python follow heard times, list order, activity, the packet-log search — reads the arrival time (`ChatMessage.arrived_at`, SQL `_ARRIVED_AT`), never `created_at` alone. The CLI's plain `TIME` carries the same `?`; its JSON gives `created_at`, `received_at`, and - `clock_corrected`. + `clock_corrected`. Channel rows from before schema 21 get their arrival once from the + packet log (`services/arrival_backfill.py`: a stored GRP_TXT with the same text *and* + sender time is the message, its first copy the arrival); a DM has no such proof. - Node types (shared with the map): `★` you (white) · `●` companion · `▲` repeater · `■` room server · `◉` sensor · `○` unknown. **Every type mark is white** (JP, 2026-10-14; they were pink, violet, white, orange): the shape is the type. Each keeps its diff --git a/meshterm/persistence/repository.py b/meshterm/persistence/repository.py index 8baebb3..eea9d96 100644 --- a/meshterm/persistence/repository.py +++ b/meshterm/persistence/repository.py @@ -2467,6 +2467,80 @@ class Repository: for r in rows ] + def channel_messages_without_arrival( + self, channel_ids: Iterable[str] + ) -> list[tuple[int, str, str, str]]: + """Each received channel message from before MeshTerm kept the arrival time. + + These rows have only the sender time. :mod:`~meshterm.services.arrival_backfill` + finds their arrival time in the packet log. + + Args: + channel_ids: The identities of the channels to read. + + Returns: + ``(row id, channel id, created_at, text)`` for each row, with the time as the + stored ISO 8601 text. + """ + ids = sorted(set(channel_ids)) + if not ids: + return [] + marks = ", ".join("?" * len(ids)) + rows = self._conn.execute( + "SELECT id, channel_id, created_at, text FROM messages " + "WHERE outbound = 0 AND is_channel = 1 AND received_at IS NULL " + f"AND channel_id IN ({marks})", + ids, + ).fetchall() + return [(r["id"], r["channel_id"], r["created_at"], r["text"]) for r in rows] + + def set_arrival_times(self, arrivals: Iterable[tuple[int, str]]) -> int: + """Give old rows the arrival time that the packet log proves. + + Only a row with no arrival time changes. Thus a second run, or a run at the same + time as a new message, changes nothing that it must not change. + + Args: + arrivals: ``(row id, arrival time as ISO 8601 text)`` pairs. + + Returns: + The number of rows that changed. + """ + cur = self._conn.executemany( + "UPDATE messages SET received_at = ? WHERE id = ? AND received_at IS NULL", + [(when, row_id) for row_id, when in arrivals], + ) + self._conn.commit() + return cur.rowcount + + def arrival_backfill_done(self) -> set[str]: + """The channels whose old messages :mod:`~meshterm.services.arrival_backfill` searched. + + Returns: + The channel identities. The backfill does not search these again. + """ + row = self._conn.execute( + "SELECT value FROM app_state WHERE key = 'arrival_backfill'" + ).fetchone() + if row is None: + return set() + try: + return {str(cid) for cid in json.loads(row["value"])} + except (ValueError, TypeError): + return set() + + def mark_arrival_backfill_done(self, channel_ids: Iterable[str]) -> None: + """Record the channels that the backfill searched (refer to :meth:`arrival_backfill_done`). + + Args: + channel_ids: All the searched channels, the earlier ones included. + """ + self._conn.execute( + "INSERT OR REPLACE INTO app_state(key, value) VALUES ('arrival_backfill', ?)", + (json.dumps(sorted(set(channel_ids))),), + ) + self._conn.commit() + def message_texts(self) -> list[str]: """The text of each stored chat message, for the names that the channel guide tests. diff --git a/meshterm/services/arrival_backfill.py b/meshterm/services/arrival_backfill.py new file mode 100644 index 0000000..6c11eb8 --- /dev/null +++ b/meshterm/services/arrival_backfill.py @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Find the arrival time of old channel messages in the stored packet log. + +Before schema 21, MeshTerm kept only the sender time of a received message. A sender with +a wrong clock then put its message on the wrong day, and nothing on the row can tell. But +the recorder also stored each channel packet that it heard (``GRP_TXT`` observations), with +the time when our node heard it, and with its ciphertext. MeshTerm has the key of each +channel in its slots. Thus it can decrypt each stored packet, and a packet whose text and +sender time are equal to those of a stored message *is* that message. The first time that +our node heard it is the arrival time of the message. + +The match is exact, not a guess. The sender time is in the encrypted body, to the second, +so two messages with the same text ("hi") are two different keys. A message with no +matching packet (the monitor was not running, or the packet came before MeshTerm kept the +ciphertext) keeps no arrival time, and the app uses its sender time, as before. + +Direct messages cannot use this. Only the recipient can decrypt a direct packet. + +The backfill runs one time for each channel. It records each channel that it searched +(:meth:`~meshterm.persistence.repository.Repository.arrival_backfill_done`), so the next +start does not decrypt the log again. The work is cut into small parts, with a yield to +the event loop between them. A large log costs seconds on a desktop, and much longer on a +handheld, and the screen must stay quick during that time. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Mapping +from datetime import datetime +from typing import TYPE_CHECKING + +from ..core.channels import channel_hash, decrypt_channel_text, effective_secret + +if TYPE_CHECKING: + from ..persistence.repository import Repository + +#: The number of stored packets to decrypt before each yield to the event loop. +_CHUNK = 200 + + +def _key(channel_id: str, sent: datetime, text: str) -> tuple[str, int, str]: + """The identity of one channel message: its channel, its sender time, and its text.""" + return channel_id, int(sent.timestamp()), text.strip() + + +async def backfill_arrivals(repo: Repository, channels: Mapping[str, tuple[str, bytes]]) -> int: + """Give the old messages of these channels the arrival time from the packet log. + + Args: + repo: The repository that holds the messages and the packet log. + channels: The channels on the device: identity to ``(name, secret)``. + + Returns: + The number of messages that got an arrival time. + """ + done = repo.arrival_backfill_done() + todo = {cid: pair for cid, pair in channels.items() if cid not in done} + if not todo: + return 0 + rows = repo.channel_messages_without_arrival(todo) + wanted: dict[tuple[str, int, str], list[int]] = {} + for row_id, channel_id, created_at, text in rows: + try: + sent = datetime.fromisoformat(created_at) + except (TypeError, ValueError): + continue + wanted.setdefault(_key(channel_id, sent, text), []).append(row_id) + + found: dict[int, str] = {} + if wanted: + # A packet names its channel only by a one-byte hash. Thus each packet is tried only + # with the channels whose hash it carries. The MAC check in the decryption is the + # proof. + by_hash: dict[str, list[tuple[str, str, bytes]]] = {} + for cid, (name, secret) in todo.items(): + fingerprint = channel_hash(effective_secret(name, secret)) + by_hash.setdefault(fingerprint.lower(), []).append((cid, name, secret)) + packets = repo.channel_packets() + for start in range(0, len(packets), _CHUNK): + for chan_hash, cipher_mac, crypted, first_at, _last in packets[start : start + _CHUNK]: + for cid, name, secret in by_hash.get(str(chan_hash).lower(), ()): + text = decrypt_channel_text(chan_hash, cipher_mac, crypted, [(name, secret)]) + if text is None: + continue + if text.sent_at is not None: + for row_id in wanted.get(_key(cid, text.sent_at, text.text), ()): + # The copies of one message are separate groups (each retry has + # its own ciphertext). The earliest copy is the arrival. + if row_id not in found or first_at < found[row_id]: + found[row_id] = first_at + break + await asyncio.sleep(0) + + changed = repo.set_arrival_times(found.items()) if found else 0 + repo.mark_arrival_backfill_done(done | set(todo)) + return changed diff --git a/meshterm/services/chat_service.py b/meshterm/services/chat_service.py index 3eaaf7d..39b3d94 100644 --- a/meshterm/services/chat_service.py +++ b/meshterm/services/chat_service.py @@ -35,6 +35,7 @@ from ..core.models import ( utcnow, ) from ..core.regions import WILDCARD, normalize +from .arrival_backfill import backfill_arrivals if TYPE_CHECKING: from ..context import AppContext @@ -92,6 +93,9 @@ class ChatService: # (synchronous) hub callback to that worker. self._queue: asyncio.Queue[Message] | None = None self._worker: asyncio.Task[None] | None = None + # The search of the packet log for the arrival time of old channel messages (refer + # to :meth:`_prime_channels`). The reference keeps the task alive while it runs. + self._backfill: asyncio.Task[None] | None = None # The last known channel slot index -> the intrinsic identity. The resolution of an # inbound message reads the slot again each time (refer to :meth:`channel_id_for`). # Thus this map is not the source of truth. It is only a warm map for the priming @@ -279,14 +283,31 @@ class ChatService: message from before the identities, with the channels that are now in each slot. This is best-effort, because MeshTerm did not store the old map from slot to channel. + + Then it starts the search of the packet log for the arrival time of the old channel + messages (refer to :mod:`~meshterm.services.arrival_backfill`). That search runs in + the background, because a large log takes time to decrypt. """ try: await self.refresh_channels() if self._channel_ids: self._ctx.repo.backfill_channel_ids(dict(self._channel_ids)) + slots = await self._ctx.devstate.channel_slots() + channels = {channel_identity(s.name, s.secret): (s.name, s.secret) for s in slots} + if channels: + self._backfill = asyncio.ensure_future(self._backfill_arrivals(channels)) except Exception as exc: # noqa: BLE001 - the priming is best-effort, and never fatal self._ctx.log.debug("chat: channel priming failed: %s", exc) + async def _backfill_arrivals(self, channels: dict[str, tuple[str, bytes]]) -> None: + """Run the arrival backfill. A failure is logged and never stops the chat service.""" + try: + changed = await backfill_arrivals(self._ctx.repo, channels) + if changed: + self._ctx.log.info("chat: %d old channel messages got their arrival time", changed) + except Exception as exc: # noqa: BLE001 - the backfill is best-effort, and never fatal + self._ctx.log.warning("chat: arrival backfill failed: %s", exc) + async def stop(self) -> None: """Stop the storage of messages, and close the run record. Idempotent. @@ -311,6 +332,15 @@ class ChatService: except asyncio.CancelledError: pass self._worker = None + if self._backfill is not None: + # It yields after each small part, so a cancel stops it at once. A backfill that + # did not end searches again at the next start. + self._backfill.cancel() + try: + await self._backfill + except asyncio.CancelledError: + pass + self._backfill = None self._queue = None if self._run_id is not None: self._ctx.repo.finish_run(self._run_id, "ok", {"messages": self._session_count}) diff --git a/tests/test_clock_skew.py b/tests/test_clock_skew.py index 3951efb..16d2a81 100644 --- a/tests/test_clock_skew.py +++ b/tests/test_clock_skew.py @@ -247,6 +247,72 @@ def test_the_paths_of_a_received_message_are_found_by_its_arrival(tmp_path: Path repo.close() +# -- the backfill from the packet log ------------------------------------------------------- + + +def _old_channel_row(repo: Repository, channel_id: str, text: str, sent) -> int: # noqa: ANN001 + """A received channel message stored before the arrival time: only the sender time.""" + message = ChatMessage(text=text, is_channel=True, channel_idx=0, channel_id=channel_id) + message.created_at = sent + return repo.record_chat_message(message) + + +def _heard(repo: Repository, run: int, body: dict, when) -> None: # noqa: ANN001 + from meshterm.core.models import Observation + + repo.record_observation( + run, Observation(node=None, kind="packet", path="3d", snr=1.0, observed_at=when, raw=body) + ) + + +def test_the_backfill_finds_the_arrival_in_the_packet_log(tmp_path: Path) -> None: + """A stored packet with the same text and sender time is the message. Its first copy wins. + + A packet with the same text but another sender time is another message, and a message + with no packet keeps no arrival time. + """ + import asyncio + + from meshterm.core.channels import DEFAULT_PUBLIC_SECRET, channel_identity + from meshterm.demo.traffic import grp_txt_body + from meshterm.services.arrival_backfill import backfill_arrivals + + repo = Repository(tmp_path / "chat.db") + run = repo.start_run("monitor", {}, None) + cid = channel_identity("Public", DEFAULT_PUBLIC_SECRET) + now = utcnow().replace(microsecond=0) + behind = now - timedelta(days=4) # the clock of the sender + + late = _old_channel_row(repo, cid, "klkdeck: hi", behind) + lost = _old_channel_row(repo, cid, "klkdeck: nobody heard this", behind) + for when, attempt in ( + (now, 0), + (now + timedelta(seconds=1), 0), + (now + timedelta(seconds=9), 1), + ): + _heard(repo, run, grp_txt_body(DEFAULT_PUBLIC_SECRET, "klkdeck: hi", behind, attempt), when) + # The same text at another sender time: another message, which matches nothing. + other = behind + timedelta(hours=1) + _heard( + repo, + run, + grp_txt_body(DEFAULT_PUBLIC_SECRET, "klkdeck: hi", other), + now - timedelta(days=1), + ) + + channels = {cid: ("Public", DEFAULT_PUBLIC_SECRET)} + assert asyncio.run(backfill_arrivals(repo, channels)) == 1 + rows = {m.row_id: m for m in repo.recent_chat_messages(is_channel=True, channel_id=cid)} + assert rows[late].received_at == now + assert rows[late].corrected(TOLERANCE) + assert rows[lost].received_at is None + + # The channel is done: a second start does not decrypt the log again. + assert repo.arrival_backfill_done() == {cid} + assert asyncio.run(backfill_arrivals(repo, channels)) == 0 + repo.close() + + # -- the screens ---------------------------------------------------------------------------- @@ -280,9 +346,7 @@ def test_a_picked_corrected_message_gives_the_sender_time() -> None: assert "behind" in text screen._selected = None - assert "its clock:" not in "\n".join( - _strip_ansi(line) for line in screen._render_grouped(72) - ) + assert "its clock:" not in "\n".join(_strip_ansi(line) for line in screen._render_grouped(72)) def test_the_cli_history_marks_a_corrected_time_and_gives_both_times() -> None: