Old channel messages get their arrival time from the packet log
Rows stored before the arrival time had only the sender time, so a wrong clock still put them on the wrong day. A one-time background backfill decrypts the stored GRP_TXT packets with the device's channel keys; a packet with the same text and sender time is the message, and its first copy is the arrival. On a real history it matched 12,071 of 14,197 old Public messages in 1.6 s. DMs cannot be decrypted and keep their sender time. Co-Authored-By: Claude Haiku 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8107256ebb
commit
2cf0e09eb4
5 files changed
+271
-4
No files matched your search
@@ -443,7 +443,9 @@ as a grouped list does. Filling a page in is editing its `.md`; no Python follow
|
||||
heard times, list order, activity, the packet-log search — reads the arrival time
|
||||
(`ChatMessage.arrived_at`, SQL `_ARRIVED_AT`), never `created_at` alone. The CLI's
|
||||
plain `TIME` carries the same `?`; its JSON gives `created_at`, `received_at`, and
|
||||
`clock_corrected`.
|
||||
`clock_corrected`. Channel rows from before schema 21 get their arrival once from the
|
||||
packet log (`services/arrival_backfill.py`: a stored GRP_TXT with the same text *and*
|
||||
sender time is the message, its first copy the arrival); a DM has no such proof.
|
||||
- Node types (shared with the map): `★` you (white) · `●` companion · `▲` repeater ·
|
||||
`■` room server · `◉` sensor · `○` unknown. **Every type mark is white** (JP,
|
||||
2026-10-14; they were pink, violet, white, orange): the shape is the type. Each keeps its
|
||||
|
||||
@@ -2467,6 +2467,80 @@ class Repository:
|
||||
for r in rows
|
||||
]
|
||||
|
||||
def channel_messages_without_arrival(
|
||||
self, channel_ids: Iterable[str]
|
||||
) -> list[tuple[int, str, str, str]]:
|
||||
"""Each received channel message from before MeshTerm kept the arrival time.
|
||||
|
||||
These rows have only the sender time. :mod:`~meshterm.services.arrival_backfill`
|
||||
finds their arrival time in the packet log.
|
||||
|
||||
Args:
|
||||
channel_ids: The identities of the channels to read.
|
||||
|
||||
Returns:
|
||||
``(row id, channel id, created_at, text)`` for each row, with the time as the
|
||||
stored ISO 8601 text.
|
||||
"""
|
||||
ids = sorted(set(channel_ids))
|
||||
if not ids:
|
||||
return []
|
||||
marks = ", ".join("?" * len(ids))
|
||||
rows = self._conn.execute(
|
||||
"SELECT id, channel_id, created_at, text FROM messages "
|
||||
"WHERE outbound = 0 AND is_channel = 1 AND received_at IS NULL "
|
||||
f"AND channel_id IN ({marks})",
|
||||
ids,
|
||||
).fetchall()
|
||||
return [(r["id"], r["channel_id"], r["created_at"], r["text"]) for r in rows]
|
||||
|
||||
def set_arrival_times(self, arrivals: Iterable[tuple[int, str]]) -> int:
|
||||
"""Give old rows the arrival time that the packet log proves.
|
||||
|
||||
Only a row with no arrival time changes. Thus a second run, or a run at the same
|
||||
time as a new message, changes nothing that it must not change.
|
||||
|
||||
Args:
|
||||
arrivals: ``(row id, arrival time as ISO 8601 text)`` pairs.
|
||||
|
||||
Returns:
|
||||
The number of rows that changed.
|
||||
"""
|
||||
cur = self._conn.executemany(
|
||||
"UPDATE messages SET received_at = ? WHERE id = ? AND received_at IS NULL",
|
||||
[(when, row_id) for row_id, when in arrivals],
|
||||
)
|
||||
self._conn.commit()
|
||||
return cur.rowcount
|
||||
|
||||
def arrival_backfill_done(self) -> set[str]:
|
||||
"""The channels whose old messages :mod:`~meshterm.services.arrival_backfill` searched.
|
||||
|
||||
Returns:
|
||||
The channel identities. The backfill does not search these again.
|
||||
"""
|
||||
row = self._conn.execute(
|
||||
"SELECT value FROM app_state WHERE key = 'arrival_backfill'"
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return set()
|
||||
try:
|
||||
return {str(cid) for cid in json.loads(row["value"])}
|
||||
except (ValueError, TypeError):
|
||||
return set()
|
||||
|
||||
def mark_arrival_backfill_done(self, channel_ids: Iterable[str]) -> None:
|
||||
"""Record the channels that the backfill searched (refer to :meth:`arrival_backfill_done`).
|
||||
|
||||
Args:
|
||||
channel_ids: All the searched channels, the earlier ones included.
|
||||
"""
|
||||
self._conn.execute(
|
||||
"INSERT OR REPLACE INTO app_state(key, value) VALUES ('arrival_backfill', ?)",
|
||||
(json.dumps(sorted(set(channel_ids))),),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def message_texts(self) -> list[str]:
|
||||
"""The text of each stored chat message, for the names that the channel guide tests.
|
||||
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Find the arrival time of old channel messages in the stored packet log.
|
||||
|
||||
Before schema 21, MeshTerm kept only the sender time of a received message. A sender with
|
||||
a wrong clock then put its message on the wrong day, and nothing on the row can tell. But
|
||||
the recorder also stored each channel packet that it heard (``GRP_TXT`` observations), with
|
||||
the time when our node heard it, and with its ciphertext. MeshTerm has the key of each
|
||||
channel in its slots. Thus it can decrypt each stored packet, and a packet whose text and
|
||||
sender time are equal to those of a stored message *is* that message. The first time that
|
||||
our node heard it is the arrival time of the message.
|
||||
|
||||
The match is exact, not a guess. The sender time is in the encrypted body, to the second,
|
||||
so two messages with the same text ("hi") are two different keys. A message with no
|
||||
matching packet (the monitor was not running, or the packet came before MeshTerm kept the
|
||||
ciphertext) keeps no arrival time, and the app uses its sender time, as before.
|
||||
|
||||
Direct messages cannot use this. Only the recipient can decrypt a direct packet.
|
||||
|
||||
The backfill runs one time for each channel. It records each channel that it searched
|
||||
(:meth:`~meshterm.persistence.repository.Repository.arrival_backfill_done`), so the next
|
||||
start does not decrypt the log again. The work is cut into small parts, with a yield to
|
||||
the event loop between them. A large log costs seconds on a desktop, and much longer on a
|
||||
handheld, and the screen must stay quick during that time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from ..core.channels import channel_hash, decrypt_channel_text, effective_secret
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..persistence.repository import Repository
|
||||
|
||||
#: The number of stored packets to decrypt before each yield to the event loop.
|
||||
_CHUNK = 200
|
||||
|
||||
|
||||
def _key(channel_id: str, sent: datetime, text: str) -> tuple[str, int, str]:
|
||||
"""The identity of one channel message: its channel, its sender time, and its text."""
|
||||
return channel_id, int(sent.timestamp()), text.strip()
|
||||
|
||||
|
||||
async def backfill_arrivals(repo: Repository, channels: Mapping[str, tuple[str, bytes]]) -> int:
|
||||
"""Give the old messages of these channels the arrival time from the packet log.
|
||||
|
||||
Args:
|
||||
repo: The repository that holds the messages and the packet log.
|
||||
channels: The channels on the device: identity to ``(name, secret)``.
|
||||
|
||||
Returns:
|
||||
The number of messages that got an arrival time.
|
||||
"""
|
||||
done = repo.arrival_backfill_done()
|
||||
todo = {cid: pair for cid, pair in channels.items() if cid not in done}
|
||||
if not todo:
|
||||
return 0
|
||||
rows = repo.channel_messages_without_arrival(todo)
|
||||
wanted: dict[tuple[str, int, str], list[int]] = {}
|
||||
for row_id, channel_id, created_at, text in rows:
|
||||
try:
|
||||
sent = datetime.fromisoformat(created_at)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
wanted.setdefault(_key(channel_id, sent, text), []).append(row_id)
|
||||
|
||||
found: dict[int, str] = {}
|
||||
if wanted:
|
||||
# A packet names its channel only by a one-byte hash. Thus each packet is tried only
|
||||
# with the channels whose hash it carries. The MAC check in the decryption is the
|
||||
# proof.
|
||||
by_hash: dict[str, list[tuple[str, str, bytes]]] = {}
|
||||
for cid, (name, secret) in todo.items():
|
||||
fingerprint = channel_hash(effective_secret(name, secret))
|
||||
by_hash.setdefault(fingerprint.lower(), []).append((cid, name, secret))
|
||||
packets = repo.channel_packets()
|
||||
for start in range(0, len(packets), _CHUNK):
|
||||
for chan_hash, cipher_mac, crypted, first_at, _last in packets[start : start + _CHUNK]:
|
||||
for cid, name, secret in by_hash.get(str(chan_hash).lower(), ()):
|
||||
text = decrypt_channel_text(chan_hash, cipher_mac, crypted, [(name, secret)])
|
||||
if text is None:
|
||||
continue
|
||||
if text.sent_at is not None:
|
||||
for row_id in wanted.get(_key(cid, text.sent_at, text.text), ()):
|
||||
# The copies of one message are separate groups (each retry has
|
||||
# its own ciphertext). The earliest copy is the arrival.
|
||||
if row_id not in found or first_at < found[row_id]:
|
||||
found[row_id] = first_at
|
||||
break
|
||||
await asyncio.sleep(0)
|
||||
|
||||
changed = repo.set_arrival_times(found.items()) if found else 0
|
||||
repo.mark_arrival_backfill_done(done | set(todo))
|
||||
return changed
|
||||
@@ -35,6 +35,7 @@ from ..core.models import (
|
||||
utcnow,
|
||||
)
|
||||
from ..core.regions import WILDCARD, normalize
|
||||
from .arrival_backfill import backfill_arrivals
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..context import AppContext
|
||||
@@ -92,6 +93,9 @@ class ChatService:
|
||||
# (synchronous) hub callback to that worker.
|
||||
self._queue: asyncio.Queue[Message] | None = None
|
||||
self._worker: asyncio.Task[None] | None = None
|
||||
# The search of the packet log for the arrival time of old channel messages (refer
|
||||
# to :meth:`_prime_channels`). The reference keeps the task alive while it runs.
|
||||
self._backfill: asyncio.Task[None] | None = None
|
||||
# The last known channel slot index -> the intrinsic identity. The resolution of an
|
||||
# inbound message reads the slot again each time (refer to :meth:`channel_id_for`).
|
||||
# Thus this map is not the source of truth. It is only a warm map for the priming
|
||||
@@ -279,14 +283,31 @@ class ChatService:
|
||||
message from before the identities, with the channels that are now in each slot.
|
||||
This is best-effort, because MeshTerm did not store the old map from slot to
|
||||
channel.
|
||||
|
||||
Then it starts the search of the packet log for the arrival time of the old channel
|
||||
messages (refer to :mod:`~meshterm.services.arrival_backfill`). That search runs in
|
||||
the background, because a large log takes time to decrypt.
|
||||
"""
|
||||
try:
|
||||
await self.refresh_channels()
|
||||
if self._channel_ids:
|
||||
self._ctx.repo.backfill_channel_ids(dict(self._channel_ids))
|
||||
slots = await self._ctx.devstate.channel_slots()
|
||||
channels = {channel_identity(s.name, s.secret): (s.name, s.secret) for s in slots}
|
||||
if channels:
|
||||
self._backfill = asyncio.ensure_future(self._backfill_arrivals(channels))
|
||||
except Exception as exc: # noqa: BLE001 - the priming is best-effort, and never fatal
|
||||
self._ctx.log.debug("chat: channel priming failed: %s", exc)
|
||||
|
||||
async def _backfill_arrivals(self, channels: dict[str, tuple[str, bytes]]) -> None:
|
||||
"""Run the arrival backfill. A failure is logged and never stops the chat service."""
|
||||
try:
|
||||
changed = await backfill_arrivals(self._ctx.repo, channels)
|
||||
if changed:
|
||||
self._ctx.log.info("chat: %d old channel messages got their arrival time", changed)
|
||||
except Exception as exc: # noqa: BLE001 - the backfill is best-effort, and never fatal
|
||||
self._ctx.log.warning("chat: arrival backfill failed: %s", exc)
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Stop the storage of messages, and close the run record. Idempotent.
|
||||
|
||||
@@ -311,6 +332,15 @@ class ChatService:
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._worker = None
|
||||
if self._backfill is not None:
|
||||
# It yields after each small part, so a cancel stops it at once. A backfill that
|
||||
# did not end searches again at the next start.
|
||||
self._backfill.cancel()
|
||||
try:
|
||||
await self._backfill
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._backfill = None
|
||||
self._queue = None
|
||||
if self._run_id is not None:
|
||||
self._ctx.repo.finish_run(self._run_id, "ok", {"messages": self._session_count})
|
||||
|
||||
@@ -247,6 +247,72 @@ def test_the_paths_of_a_received_message_are_found_by_its_arrival(tmp_path: Path
|
||||
repo.close()
|
||||
|
||||
|
||||
# -- the backfill from the packet log -------------------------------------------------------
|
||||
|
||||
|
||||
def _old_channel_row(repo: Repository, channel_id: str, text: str, sent) -> int: # noqa: ANN001
|
||||
"""A received channel message stored before the arrival time: only the sender time."""
|
||||
message = ChatMessage(text=text, is_channel=True, channel_idx=0, channel_id=channel_id)
|
||||
message.created_at = sent
|
||||
return repo.record_chat_message(message)
|
||||
|
||||
|
||||
def _heard(repo: Repository, run: int, body: dict, when) -> None: # noqa: ANN001
|
||||
from meshterm.core.models import Observation
|
||||
|
||||
repo.record_observation(
|
||||
run, Observation(node=None, kind="packet", path="3d", snr=1.0, observed_at=when, raw=body)
|
||||
)
|
||||
|
||||
|
||||
def test_the_backfill_finds_the_arrival_in_the_packet_log(tmp_path: Path) -> None:
|
||||
"""A stored packet with the same text and sender time is the message. Its first copy wins.
|
||||
|
||||
A packet with the same text but another sender time is another message, and a message
|
||||
with no packet keeps no arrival time.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
from meshterm.core.channels import DEFAULT_PUBLIC_SECRET, channel_identity
|
||||
from meshterm.demo.traffic import grp_txt_body
|
||||
from meshterm.services.arrival_backfill import backfill_arrivals
|
||||
|
||||
repo = Repository(tmp_path / "chat.db")
|
||||
run = repo.start_run("monitor", {}, None)
|
||||
cid = channel_identity("Public", DEFAULT_PUBLIC_SECRET)
|
||||
now = utcnow().replace(microsecond=0)
|
||||
behind = now - timedelta(days=4) # the clock of the sender
|
||||
|
||||
late = _old_channel_row(repo, cid, "klkdeck: hi", behind)
|
||||
lost = _old_channel_row(repo, cid, "klkdeck: nobody heard this", behind)
|
||||
for when, attempt in (
|
||||
(now, 0),
|
||||
(now + timedelta(seconds=1), 0),
|
||||
(now + timedelta(seconds=9), 1),
|
||||
):
|
||||
_heard(repo, run, grp_txt_body(DEFAULT_PUBLIC_SECRET, "klkdeck: hi", behind, attempt), when)
|
||||
# The same text at another sender time: another message, which matches nothing.
|
||||
other = behind + timedelta(hours=1)
|
||||
_heard(
|
||||
repo,
|
||||
run,
|
||||
grp_txt_body(DEFAULT_PUBLIC_SECRET, "klkdeck: hi", other),
|
||||
now - timedelta(days=1),
|
||||
)
|
||||
|
||||
channels = {cid: ("Public", DEFAULT_PUBLIC_SECRET)}
|
||||
assert asyncio.run(backfill_arrivals(repo, channels)) == 1
|
||||
rows = {m.row_id: m for m in repo.recent_chat_messages(is_channel=True, channel_id=cid)}
|
||||
assert rows[late].received_at == now
|
||||
assert rows[late].corrected(TOLERANCE)
|
||||
assert rows[lost].received_at is None
|
||||
|
||||
# The channel is done: a second start does not decrypt the log again.
|
||||
assert repo.arrival_backfill_done() == {cid}
|
||||
assert asyncio.run(backfill_arrivals(repo, channels)) == 0
|
||||
repo.close()
|
||||
|
||||
|
||||
# -- the screens ----------------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -280,9 +346,7 @@ def test_a_picked_corrected_message_gives_the_sender_time() -> None:
|
||||
assert "behind" in text
|
||||
|
||||
screen._selected = None
|
||||
assert "its clock:" not in "\n".join(
|
||||
_strip_ansi(line) for line in screen._render_grouped(72)
|
||||
)
|
||||
assert "its clock:" not in "\n".join(_strip_ansi(line) for line in screen._render_grouped(72))
|
||||
|
||||
|
||||
def test_the_cli_history_marks_a_corrected_time_and_gives_both_times() -> None:
|
||||
|
||||
Reference in new issue
Block a user