Old channel messages get their arrival time from the packet log

Rows stored before the arrival time had only the sender time, so a
wrong clock still put them on the wrong day. A one-time background
backfill decrypts the stored GRP_TXT packets with the device's channel
keys; a packet with the same text and sender time is the message, and
its first copy is the arrival. On a real history it matched 12,071 of
14,197 old Public messages in 1.6 s. DMs cannot be decrypted and keep
their sender time.

Co-Authored-By: Claude Haiku 5.5 <noreply@anthropic.com>
This commit is contained in:
Jean-Pierre MartineauandClaude Haiku 5.5 committed 2026-11-02 23:28:55 -05:00
1 parent 8107256ebb
commit 2cf0e09eb4
5 files changed
+271 -4

No files matched your search

+3 -1
View File
@@ -443,7 +443,9 @@ as a grouped list does. Filling a page in is editing its `.md`; no Python follow
heard times, list order, activity, the packet-log search — reads the arrival time
(`ChatMessage.arrived_at`, SQL `_ARRIVED_AT`), never `created_at` alone. The CLI's
plain `TIME` carries the same `?`; its JSON gives `created_at`, `received_at`, and
`clock_corrected`.
`clock_corrected`. Channel rows from before schema 21 get their arrival once from the
packet log (`services/arrival_backfill.py`: a stored GRP_TXT with the same text *and*
sender time is the message, its first copy the arrival); a DM has no such proof.
- Node types (shared with the map): `★` you (white) · `●` companion · `▲` repeater ·
`■` room server · `◉` sensor · `○` unknown. **Every type mark is white** (JP,
2026-10-14; they were pink, violet, white, orange): the shape is the type. Each keeps its
+74
View File
@@ -2467,6 +2467,80 @@ class Repository:
for r in rows
]
def channel_messages_without_arrival(
self, channel_ids: Iterable[str]
) -> list[tuple[int, str, str, str]]:
"""Each received channel message from before MeshTerm kept the arrival time.
These rows have only the sender time. :mod:`~meshterm.services.arrival_backfill`
finds their arrival time in the packet log.
Args:
channel_ids: The identities of the channels to read.
Returns:
``(row id, channel id, created_at, text)`` for each row, with the time as the
stored ISO 8601 text.
"""
ids = sorted(set(channel_ids))
if not ids:
return []
marks = ", ".join("?" * len(ids))
rows = self._conn.execute(
"SELECT id, channel_id, created_at, text FROM messages "
"WHERE outbound = 0 AND is_channel = 1 AND received_at IS NULL "
f"AND channel_id IN ({marks})",
ids,
).fetchall()
return [(r["id"], r["channel_id"], r["created_at"], r["text"]) for r in rows]
def set_arrival_times(self, arrivals: Iterable[tuple[int, str]]) -> int:
"""Give old rows the arrival time that the packet log proves.
Only a row with no arrival time changes. Thus a second run, or a run at the same
time as a new message, changes nothing that it must not change.
Args:
arrivals: ``(row id, arrival time as ISO 8601 text)`` pairs.
Returns:
The number of rows that changed.
"""
cur = self._conn.executemany(
"UPDATE messages SET received_at = ? WHERE id = ? AND received_at IS NULL",
[(when, row_id) for row_id, when in arrivals],
)
self._conn.commit()
return cur.rowcount
def arrival_backfill_done(self) -> set[str]:
"""The channels whose old messages :mod:`~meshterm.services.arrival_backfill` searched.
Returns:
The channel identities. The backfill does not search these again.
"""
row = self._conn.execute(
"SELECT value FROM app_state WHERE key = 'arrival_backfill'"
).fetchone()
if row is None:
return set()
try:
return {str(cid) for cid in json.loads(row["value"])}
except (ValueError, TypeError):
return set()
def mark_arrival_backfill_done(self, channel_ids: Iterable[str]) -> None:
"""Record the channels that the backfill searched (refer to :meth:`arrival_backfill_done`).
Args:
channel_ids: All the searched channels, the earlier ones included.
"""
self._conn.execute(
"INSERT OR REPLACE INTO app_state(key, value) VALUES ('arrival_backfill', ?)",
(json.dumps(sorted(set(channel_ids))),),
)
self._conn.commit()
def message_texts(self) -> list[str]:
"""The text of each stored chat message, for the names that the channel guide tests.
+97
View File
@@ -0,0 +1,97 @@
# SPDX-License-Identifier: Apache-2.0
"""Find the arrival time of old channel messages in the stored packet log.
Before schema 21, MeshTerm kept only the sender time of a received message. A sender with
a wrong clock then put its message on the wrong day, and nothing on the row can tell. But
the recorder also stored each channel packet that it heard (``GRP_TXT`` observations), with
the time when our node heard it, and with its ciphertext. MeshTerm has the key of each
channel in its slots. Thus it can decrypt each stored packet, and a packet whose text and
sender time are equal to those of a stored message *is* that message. The first time that
our node heard it is the arrival time of the message.
The match is exact, not a guess. The sender time is in the encrypted body, to the second,
so two messages with the same text ("hi") are two different keys. A message with no
matching packet (the monitor was not running, or the packet came before MeshTerm kept the
ciphertext) keeps no arrival time, and the app uses its sender time, as before.
Direct messages cannot use this. Only the recipient can decrypt a direct packet.
The backfill runs one time for each channel. It records each channel that it searched
(:meth:`~meshterm.persistence.repository.Repository.arrival_backfill_done`), so the next
start does not decrypt the log again. The work is cut into small parts, with a yield to
the event loop between them. A large log costs seconds on a desktop, and much longer on a
handheld, and the screen must stay quick during that time.
"""
from __future__ import annotations
import asyncio
from collections.abc import Mapping
from datetime import datetime
from typing import TYPE_CHECKING
from ..core.channels import channel_hash, decrypt_channel_text, effective_secret
if TYPE_CHECKING:
from ..persistence.repository import Repository
#: The number of stored packets to decrypt before each yield to the event loop.
_CHUNK = 200
def _key(channel_id: str, sent: datetime, text: str) -> tuple[str, int, str]:
"""The identity of one channel message: its channel, its sender time, and its text."""
return channel_id, int(sent.timestamp()), text.strip()
async def backfill_arrivals(repo: Repository, channels: Mapping[str, tuple[str, bytes]]) -> int:
"""Give the old messages of these channels the arrival time from the packet log.
Args:
repo: The repository that holds the messages and the packet log.
channels: The channels on the device: identity to ``(name, secret)``.
Returns:
The number of messages that got an arrival time.
"""
done = repo.arrival_backfill_done()
todo = {cid: pair for cid, pair in channels.items() if cid not in done}
if not todo:
return 0
rows = repo.channel_messages_without_arrival(todo)
wanted: dict[tuple[str, int, str], list[int]] = {}
for row_id, channel_id, created_at, text in rows:
try:
sent = datetime.fromisoformat(created_at)
except (TypeError, ValueError):
continue
wanted.setdefault(_key(channel_id, sent, text), []).append(row_id)
found: dict[int, str] = {}
if wanted:
# A packet names its channel only by a one-byte hash. Thus each packet is tried only
# with the channels whose hash it carries. The MAC check in the decryption is the
# proof.
by_hash: dict[str, list[tuple[str, str, bytes]]] = {}
for cid, (name, secret) in todo.items():
fingerprint = channel_hash(effective_secret(name, secret))
by_hash.setdefault(fingerprint.lower(), []).append((cid, name, secret))
packets = repo.channel_packets()
for start in range(0, len(packets), _CHUNK):
for chan_hash, cipher_mac, crypted, first_at, _last in packets[start : start + _CHUNK]:
for cid, name, secret in by_hash.get(str(chan_hash).lower(), ()):
text = decrypt_channel_text(chan_hash, cipher_mac, crypted, [(name, secret)])
if text is None:
continue
if text.sent_at is not None:
for row_id in wanted.get(_key(cid, text.sent_at, text.text), ()):
# The copies of one message are separate groups (each retry has
# its own ciphertext). The earliest copy is the arrival.
if row_id not in found or first_at < found[row_id]:
found[row_id] = first_at
break
await asyncio.sleep(0)
changed = repo.set_arrival_times(found.items()) if found else 0
repo.mark_arrival_backfill_done(done | set(todo))
return changed
+30
View File
@@ -35,6 +35,7 @@ from ..core.models import (
utcnow,
)
from ..core.regions import WILDCARD, normalize
from .arrival_backfill import backfill_arrivals
if TYPE_CHECKING:
from ..context import AppContext
@@ -92,6 +93,9 @@ class ChatService:
# (synchronous) hub callback to that worker.
self._queue: asyncio.Queue[Message] | None = None
self._worker: asyncio.Task[None] | None = None
# The search of the packet log for the arrival time of old channel messages (refer
# to :meth:`_prime_channels`). The reference keeps the task alive while it runs.
self._backfill: asyncio.Task[None] | None = None
# The last known channel slot index -> the intrinsic identity. The resolution of an
# inbound message reads the slot again each time (refer to :meth:`channel_id_for`).
# Thus this map is not the source of truth. It is only a warm map for the priming
@@ -279,14 +283,31 @@ class ChatService:
message from before the identities, with the channels that are now in each slot.
This is best-effort, because MeshTerm did not store the old map from slot to
channel.
Then it starts the search of the packet log for the arrival time of the old channel
messages (refer to :mod:`~meshterm.services.arrival_backfill`). That search runs in
the background, because a large log takes time to decrypt.
"""
try:
await self.refresh_channels()
if self._channel_ids:
self._ctx.repo.backfill_channel_ids(dict(self._channel_ids))
slots = await self._ctx.devstate.channel_slots()
channels = {channel_identity(s.name, s.secret): (s.name, s.secret) for s in slots}
if channels:
self._backfill = asyncio.ensure_future(self._backfill_arrivals(channels))
except Exception as exc: # noqa: BLE001 - the priming is best-effort, and never fatal
self._ctx.log.debug("chat: channel priming failed: %s", exc)
async def _backfill_arrivals(self, channels: dict[str, tuple[str, bytes]]) -> None:
"""Run the arrival backfill. A failure is logged and never stops the chat service."""
try:
changed = await backfill_arrivals(self._ctx.repo, channels)
if changed:
self._ctx.log.info("chat: %d old channel messages got their arrival time", changed)
except Exception as exc: # noqa: BLE001 - the backfill is best-effort, and never fatal
self._ctx.log.warning("chat: arrival backfill failed: %s", exc)
async def stop(self) -> None:
"""Stop the storage of messages, and close the run record. Idempotent.
@@ -311,6 +332,15 @@ class ChatService:
except asyncio.CancelledError:
pass
self._worker = None
if self._backfill is not None:
# It yields after each small part, so a cancel stops it at once. A backfill that
# did not end searches again at the next start.
self._backfill.cancel()
try:
await self._backfill
except asyncio.CancelledError:
pass
self._backfill = None
self._queue = None
if self._run_id is not None:
self._ctx.repo.finish_run(self._run_id, "ok", {"messages": self._session_count})
+67 -3
View File
@@ -247,6 +247,72 @@ def test_the_paths_of_a_received_message_are_found_by_its_arrival(tmp_path: Path
repo.close()
# -- the backfill from the packet log -------------------------------------------------------
def _old_channel_row(repo: Repository, channel_id: str, text: str, sent) -> int: # noqa: ANN001
"""A received channel message stored before the arrival time: only the sender time."""
message = ChatMessage(text=text, is_channel=True, channel_idx=0, channel_id=channel_id)
message.created_at = sent
return repo.record_chat_message(message)
def _heard(repo: Repository, run: int, body: dict, when) -> None: # noqa: ANN001
from meshterm.core.models import Observation
repo.record_observation(
run, Observation(node=None, kind="packet", path="3d", snr=1.0, observed_at=when, raw=body)
)
def test_the_backfill_finds_the_arrival_in_the_packet_log(tmp_path: Path) -> None:
"""A stored packet with the same text and sender time is the message. Its first copy wins.
A packet with the same text but another sender time is another message, and a message
with no packet keeps no arrival time.
"""
import asyncio
from meshterm.core.channels import DEFAULT_PUBLIC_SECRET, channel_identity
from meshterm.demo.traffic import grp_txt_body
from meshterm.services.arrival_backfill import backfill_arrivals
repo = Repository(tmp_path / "chat.db")
run = repo.start_run("monitor", {}, None)
cid = channel_identity("Public", DEFAULT_PUBLIC_SECRET)
now = utcnow().replace(microsecond=0)
behind = now - timedelta(days=4) # the clock of the sender
late = _old_channel_row(repo, cid, "klkdeck: hi", behind)
lost = _old_channel_row(repo, cid, "klkdeck: nobody heard this", behind)
for when, attempt in (
(now, 0),
(now + timedelta(seconds=1), 0),
(now + timedelta(seconds=9), 1),
):
_heard(repo, run, grp_txt_body(DEFAULT_PUBLIC_SECRET, "klkdeck: hi", behind, attempt), when)
# The same text at another sender time: another message, which matches nothing.
other = behind + timedelta(hours=1)
_heard(
repo,
run,
grp_txt_body(DEFAULT_PUBLIC_SECRET, "klkdeck: hi", other),
now - timedelta(days=1),
)
channels = {cid: ("Public", DEFAULT_PUBLIC_SECRET)}
assert asyncio.run(backfill_arrivals(repo, channels)) == 1
rows = {m.row_id: m for m in repo.recent_chat_messages(is_channel=True, channel_id=cid)}
assert rows[late].received_at == now
assert rows[late].corrected(TOLERANCE)
assert rows[lost].received_at is None
# The channel is done: a second start does not decrypt the log again.
assert repo.arrival_backfill_done() == {cid}
assert asyncio.run(backfill_arrivals(repo, channels)) == 0
repo.close()
# -- the screens ----------------------------------------------------------------------------
@@ -280,9 +346,7 @@ def test_a_picked_corrected_message_gives_the_sender_time() -> None:
assert "behind" in text
screen._selected = None
assert "its clock:" not in "\n".join(
_strip_ansi(line) for line in screen._render_grouped(72)
)
assert "its clock:" not in "\n".join(_strip_ansi(line) for line in screen._render_grouped(72))
def test_the_cli_history_marks_a_corrected_time_and_gives_both_times() -> None: