From 8107256ebbdca8ba85195f4c9ee9d176ed54786a Mon Sep 17 00:00:00 2001 From: Jean-Pierre Martineau Date: Mon, 2 Nov 2026 23:09:47 -0500 Subject: [PATCH] A wrong sender clock shows the arrival time with a ? mark A received message now keeps its arrival time (received_at) beside the sender time, and whether it waited in the companion's queue. Heard times, the Messages list order, channel activity, and the packet-log search use the arrival time, so a DM from a clock a year ahead no longer pins its contact at "heard now". The transcript shows a corrected time as 12:31?, and a picked message says what the sender's clock wrote. The CLI history carries the same mark; its JSON gives both times. clock_tolerance_s replaces ADVERT_FUTURE_SKEW_S as a preference. Co-Authored-By: Claude Haiku 5.5 --- CLAUDE.md | 12 ++ docs/development/writing-style.md | 4 + meshterm/core/connection.py | 39 +++- meshterm/core/models.py | 138 +++++++++++-- meshterm/core/preferences.py | 17 +- meshterm/persistence/db.py | 19 +- meshterm/persistence/repository.py | 62 ++++-- meshterm/services/device_state.py | 11 +- meshterm/services/message_paths.py | 43 ++-- meshterm/tools/chat.py | 37 +++- meshterm/ui/chat.py | 59 +++++- meshterm/ui/message_paths_screen.py | 10 +- meshterm/ui/widgets.py | 37 ++++ tests/test_clock_skew.py | 297 ++++++++++++++++++++++++++++ tests/test_skeleton.py | 4 +- 15 files changed, 718 insertions(+), 71 deletions(-) create mode 100644 tests/test_clock_skew.py diff --git a/CLAUDE.md b/CLAUDE.md index dc2db5d..f84bb04 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -432,6 +432,18 @@ as a grouped list does. Filling a page in is editing its `.md`; no Python follow - Status marks (single-width, themed): `✓` ok · `✗` err · `⚠`/`?` warn · `●` unread/ unacked (err) · `○` acked/empty (muted). Never `✔`, `✖`, `✅`, `❌` as status. +- **A corrected time wears `?`** (warn), in the cell after it (JP, 2026-11-02). A + received message keeps two times: the *sender time* (the sender's clock, a claim) and + the *arrival time* (ours, `received_at`). Past `clock_tolerance_s` apart the sender + time is wrong, and the line shows the arrival time instead, `12:31?` — never a guess at + the real send. A room post or a message from the companion's queue (`waited`) arrived + at an unknown time, so only a sender time in the future convicts it. On the picked + message a line under the body says what the sender's clock wrote and how far off it is + (`? its clock: 2024-01-01 03:12 · 1y 44w behind`). Everything about *our* side — + heard times, list order, activity, the packet-log search — reads the arrival time + (`ChatMessage.arrived_at`, SQL `_ARRIVED_AT`), never `created_at` alone. The CLI's + plain `TIME` carries the same `?`; its JSON gives `created_at`, `received_at`, and + `clock_corrected`. - Node types (shared with the map): `★` you (white) · `●` companion · `▲` repeater · `■` room server · `◉` sensor · `○` unknown. **Every type mark is white** (JP, 2026-10-14; they were pink, violet, white, orange): the shape is the type. Each keeps its diff --git a/docs/development/writing-style.md b/docs/development/writing-style.md index bc53873..a6c5ac9 100644 --- a/docs/development/writing-style.md +++ b/docs/development/writing-style.md @@ -194,6 +194,10 @@ and in quoted text. | packet | One radio transmission | frame (for radio data) | | protocol frame | One unit of the companion protocol on USB, Bluetooth, or TCP: a command, a reply, or a push from the companion. In code that handles only this protocol, "frame" alone is permitted | packet (for this) | | message | A text message, direct or on a channel | | +| sender time | The time that a message or an advert carries, from the clock of the node that sent it. It is a claim, and a node with a wrong clock makes it false | timestamp (alone) | +| arrival time | Our clock when MeshTerm got a received message from the companion (`received_at`) | receive time | +| clock offset | The sender time minus the arrival time. A positive offset is a clock ahead of ours | skew, drift (for this) | +| corrected time | The arrival time that the screen shows in place of a wrong sender time, with the `?` mark | | | send | MeshTerm gives a message, a packet, or a command to the device | emit | | transmit | A radio puts a packet on the air | | | broadcast | Transmit to all nodes, with no destination | | diff --git a/meshterm/core/connection.py b/meshterm/core/connection.py index bdd13c2..f83e310 100644 --- a/meshterm/core/connection.py +++ b/meshterm/core/connection.py @@ -3996,8 +3996,13 @@ class MeshCoreDevice(Device): return handler + # True while the pump empties the queue that the companion kept before we connected + # (refer to ``_message_pump``). A message from that queue arrived at an unknown time, + # so it is marked ``waited``. + backlog = [True] + def message_handler(event) -> None: # noqa: ANN001 - msg = message_from_event(event) + msg = message_from_event(event, waited=backlog[0]) if msg is not None: on_event(MeshEvent.message_event(msg)) @@ -4057,7 +4062,9 @@ class MeshCoreDevice(Device): # Do the pull of received messages ourselves (refer to ``_message_pump``). MeshCore # never pushes message bodies, so without this pull, a send works but nothing is # received. - stop_pump = self._message_pump(mc, subs, subscribe) + stop_pump = self._message_pump( + mc, subs, subscribe, on_backlog_done=lambda: backlog.__setitem__(0, False) + ) def unsubscribe() -> None: stop_pump() @@ -4071,7 +4078,9 @@ class MeshCoreDevice(Device): return unsubscribe - def _message_pump(self, mc, subs: list, subscribe) -> Unsubscribe: # type: ignore[no-untyped-def] + def _message_pump( # type: ignore[no-untyped-def] + self, mc, subs: list, subscribe, *, on_backlog_done: Callable[[], None] | None = None + ) -> Unsubscribe: """Pull the received messages from the companion continuously (the RX pull model). MeshCore does not push message bodies unsolicited. The device sends a @@ -4091,11 +4100,20 @@ class MeshCoreDevice(Device): A lock serializes the drains, so the triggers that overlap never send concurrent ``get_msg`` commands. + The first drain is the **backlog**: the messages that the companion kept while no + app was connected. The companion gives no time of arrival with them, so their + arrival time is unknown. When the first drain ends, the pump calls + ``on_backlog_done``. Thus the handler marks only the messages of the backlog as + ``waited``. A message that arrives on the air during the first drain is also + marked. The cost is small: for such a message, only a sender time in the future + counts as a wrong clock. + Args: mc: The connected ``MeshCore`` client. subs: The subscription list to which this method adds the ``MESSAGES_WAITING`` subscription (so it is torn down with the others). subscribe: The ``subscribe`` callable of the client. + on_backlog_done: Called one time, when the first drain ends. Returns: A callable with no arguments that stops the pump (its poll task and drains). @@ -4123,6 +4141,13 @@ class MeshCoreDevice(Device): def schedule_drain(_event=None) -> None: # noqa: ANN001 - MESSAGES_WAITING callback asyncio.ensure_future(drain()) + async def drain_backlog() -> None: + try: + await drain() + finally: + if on_backlog_done is not None: + on_backlog_done() + async def poll_loop() -> None: while not stop.is_set(): try: @@ -4133,7 +4158,8 @@ class MeshCoreDevice(Device): waiting = getattr(EventType, "MESSAGES_WAITING", None) if waiting is not None: subs.append(subscribe(waiting, schedule_drain)) - schedule_drain() # a first, immediate drain of all that is already in the queue + # A first, immediate drain of all that is already in the queue: the backlog. + asyncio.ensure_future(drain_backlog()) poll_task = asyncio.ensure_future(poll_loop()) def stop_pump() -> None: @@ -5838,7 +5864,7 @@ def packet_observation_from_event(event) -> Observation | None: # noqa: ANN001 ) -def message_from_event(event) -> Message | None: # noqa: ANN001 +def message_from_event(event, *, waited: bool = False) -> Message | None: # noqa: ANN001 """Map a meshcore ``CONTACT_MSG_RECV`` or ``CHANNEL_MSG_RECV`` event into a message. Direct messages have a ``pubkey_prefix`` sender. Channel messages have a @@ -5853,6 +5879,8 @@ def message_from_event(event) -> Message | None: # noqa: ANN001 Args: event: A meshcore message event (anything that has a ``payload`` mapping). + waited: Whether the message came out of the queue of the companion at connect + (refer to :attr:`~meshterm.core.models.Message.waited`). Returns: The parsed :class:`Message`, or ``None`` if the payload had no text body. @@ -5876,6 +5904,7 @@ def message_from_event(event) -> Message | None: # noqa: ANN001 is_channel=is_channel, sender_timestamp=sender_ts, snr=_as_float(payload.get("SNR", payload.get("snr"))), + waited=waited, author=author, raw=payload, ) diff --git a/meshterm/core/models.py b/meshterm/core/models.py index 9a2c784..aa4d28d 100644 --- a/meshterm/core/models.py +++ b/meshterm/core/models.py @@ -239,14 +239,30 @@ def utcnow() -> datetime: return datetime.now(timezone.utc) -#: The maximum time (in seconds) that an advert timestamp can be in the future. After this -#: limit, MeshTerm refuses the timestamp as false. The clock of the sender writes the -#: timestamp of an advert, and a mesh node with no time source can be wrong by minutes, or -#: by months. A skew of a few minutes is usual and causes no problem (the age clamps to -#: "now" until the real time catches up). But if MeshTerm accepts a larger skew, that skew -#: pins the contact at the top of each list sorted by heard time. The contact then shows -#: "now" for as long as the false timestamp stays ahead of the wall clock. -ADVERT_FUTURE_SKEW_S = 300 +#: The default of the ``clock_tolerance_s`` preference: how far (in seconds) the clock of a +#: node can be from our clock before MeshTerm calls its time wrong. The clock of the sender +#: writes the time in an advert and in a message, and a mesh node with no time source can be +#: wrong by minutes, or by months. A difference of a few minutes is usual and causes no +#: problem (an age clamps to "now" until the real time catches up). A larger difference is +#: a fault. An advert time past this limit in the future pins the contact at the top of each +#: list sorted by heard time, and a message time past it puts the message on the wrong day. +#: Read the value through :func:`clock_tolerance_s`, never this constant. +CLOCK_TOLERANCE_S = 300 + + +def clock_tolerance_s() -> int: + """The ``clock_tolerance_s`` preference: how far a clock can be off and still be correct. + + The import is here and not at the top, because :mod:`~meshterm.core.preferences` + imports this module for its defaults. + + Returns: + The tolerance in seconds. + """ + from . import preferences + + return int(preferences.current().clock_tolerance_s) + #: How close together (in days) two keys must use one name before MeshTerm calls the name #: unsure. A name is a label that the owner of a key chooses, and a new key for the same @@ -267,7 +283,7 @@ def advert_time(last_advert: object) -> datetime | None: both go through it. Thus the plausibility rule is in one place. The clock of the node that advertised writes the value, so the value is hearsay. A value that is zero, absent, or garbage means "never heard". A timestamp that is more than - :data:`ADVERT_FUTURE_SKEW_S` ahead of our clock comes from a sender clock that is set + :func:`clock_tolerance_s` ahead of our clock comes from a sender clock that is set incorrectly, and the function refuses it in the same way. Callers that have their own evidence of reception (stored observations) use that evidence to fill the ``None`` result. A true "when we heard it" is better than a false "heard in the future". @@ -287,7 +303,7 @@ def advert_time(last_advert: object) -> datetime | None: if seconds <= 0: return None when = datetime.fromtimestamp(seconds, tz=timezone.utc) - if when > utcnow() + timedelta(seconds=ADVERT_FUTURE_SKEW_S): + if when > utcnow() + timedelta(seconds=clock_tolerance_s()): return None return when @@ -473,6 +489,11 @@ class Message: room. A post from the backlog comes long after that time. snr: The signal-to-noise ratio (dB) of the reception, if reported. received_at: When the companion delivered the message to us. + waited: Whether the message came out of the queue of the companion when MeshTerm + connected. The companion keeps each message that arrives while no app is + connected, and it gives no time of arrival with it. Thus the message arrived at + an unknown time before :attr:`received_at`, and only a sender time in the + future is proof of a wrong clock. author: For a room post: the key prefix of the author, in lowercase hex. A room post is a direct message from a room server, signed with the first four bytes of the key of its writer. The room is the :attr:`sender` (it is the node that @@ -488,6 +509,7 @@ class Message: sender_timestamp: datetime | None = None snr: float | None = None received_at: datetime = field(default_factory=utcnow) + waited: bool = False author: str | None = None raw: dict | None = None @@ -600,7 +622,16 @@ class ChatMessage: ``False``: sent but not acknowledged (the user can try again). ``None``: the ack is still expected, or the state does not apply (a channel broadcast or an inbound message). - created_at: When the message was sent or received. + created_at: The time that the message carries. For our own messages, it is our + clock when we sent the message. For a received message, it is the sender time: + the clock of the sender when it composed the message. That clock can be wrong + (refer to :meth:`corrected`). + received_at: The arrival time of a received message: our clock when MeshTerm got + it from the companion. ``None`` for our own messages, and for each message + stored before MeshTerm kept this time. + waited: Whether a received message came out of the queue of the companion when + MeshTerm connected (refer to :attr:`Message.waited`). It then arrived at an + unknown time before :attr:`received_at`. row_id: The primary key in the ``messages`` table, after MeshTerm stores the message. MeshTerm uses it to update the delivery state of an outbound message in place, when the user tries again. ``None`` until the message is stored. @@ -625,6 +656,8 @@ class ChatMessage: snr: float | None = None acked: bool | None = None created_at: datetime = field(default_factory=utcnow) + received_at: datetime | None = None + waited: bool = False row_id: int | None = None scope: str | None = None author: str | None = None @@ -634,6 +667,79 @@ class ChatMessage: """Whether this is an inbound room post, by the member that :attr:`author` names.""" return self.author is not None + @property + def arrived_at(self) -> datetime: + """The best time on our clock for when this message came to us. + + Use this time for everything that is about our side: when we heard the sender, the + order of the conversations, and the search of the packet log. A sender time is only + a claim, and a node with a wrong clock makes it false. The rules: + + * Our own message: :attr:`created_at`, which our clock wrote. + * A received message with an arrival time: :attr:`received_at`. + * A message that waited in the companion: the earlier of the sender time and the + arrival time. It arrived before MeshTerm got it, but MeshTerm does not know how + long before. + * A message stored before MeshTerm kept the arrival time: :attr:`created_at`. + + The SQL form of these rules is ``_ARRIVED_AT`` in + :mod:`~meshterm.persistence.repository`. The two must agree. + """ + if self.received_at is None: + return self.created_at + if self.waited and self.created_at < self.received_at: + return self.created_at + return self.received_at + + @property + def clock_offset_s(self) -> float | None: + """The sender time minus the arrival time, in seconds, or ``None`` if not known. + + A positive value means that the clock of the sender is ahead of ours. The value is + ``None`` for our own messages and for a message with no arrival time. A packet + takes only seconds to arrive, so the value is the error of the sender's clock. + """ + if self.outbound or self.received_at is None: + return None + return (self.created_at - self.received_at).total_seconds() + + def corrected(self, tolerance_s: float) -> bool: + """Whether the sender time is wrong, so that the screen shows the arrival time. + + A sender time is wrong when it is more than ``tolerance_s`` from the arrival time. + For two kinds of message, only a time in the future is proof of a fault: + + * A room post. A room sends its backlog with the times when the members posted, + so an old time is correct. + * A message that waited in the companion (:attr:`waited`). It arrived at an + unknown time before :attr:`received_at`. + + Args: + tolerance_s: The ``clock_tolerance_s`` preference. + + Returns: + ``True`` when the screen must show the arrival time with the corrected mark. + """ + offset = self.clock_offset_s + if offset is None: + return False + if self.waited or self.is_post: + return offset > tolerance_s + return abs(offset) > tolerance_s + + def shown_at(self, tolerance_s: float) -> datetime: + """The time to show for this message: the sender time, or the arrival time if it is wrong. + + Args: + tolerance_s: The ``clock_tolerance_s`` preference. + + Returns: + :attr:`received_at` when :meth:`corrected` is true, else :attr:`created_at`. + """ + if self.received_at is not None and self.corrected(tolerance_s): + return self.received_at + return self.created_at + @property def key(self) -> str: """The key of the conversation that this message is part of.""" @@ -668,11 +774,13 @@ class ChatMessage: peer=None if message.is_channel else (message.sender or None), peer_name=peer_name, snr=message.snr, - # Use the timestamp of the sender (the moment when the message was composed) - # instead of ``received_at`` (when MeshTerm read it from the device). Thus the - # transcript shows the time of the message, not the time of the read. If the - # sender gave no timestamp, use the receive time. + # Keep the two times. The sender time (when the message was composed) is the + # time that the transcript shows, if the clock of the sender is correct. The + # arrival time is what MeshTerm knows for sure (refer to ``arrived_at``). If the + # sender gave no time, use the arrival time for both. created_at=message.sender_timestamp or message.received_at, + received_at=message.received_at, + waited=message.waited, author=None if message.is_channel else message.author, ) diff --git a/meshterm/core/preferences.py b/meshterm/core/preferences.py index 382e3f5..650386f 100644 --- a/meshterm/core/preferences.py +++ b/meshterm/core/preferences.py @@ -54,7 +54,7 @@ from .advert_store import DEFAULT_QUIET_S, QUIET_CHOICES_S from .atomicwrite import write_atomically from .courier_store import DONE_CAP from .geo import DEFAULT_VIEW_FRACTION -from .models import NAME_OVERLAP_DAYS +from .models import CLOCK_TOLERANCE_S, NAME_OVERLAP_DAYS from .watch_store import ALERT_CAP, DEFAULT_SILENCE_HOURS, OFF, SILENCE_CHOICES_H if sys.version_info >= (3, 11): @@ -520,6 +520,21 @@ PREFERENCES: tuple[PrefSpec, ...] = ( maximum=3650, unit="days", ), + PrefSpec( + key="clock_tolerance_s", + label="Clock tolerance", + help="How far a node's clock can be off before its times are wrong", + group="Display", + value_type="int", + # A message whose sender time is farther than this from its arrival shows the + # arrival time, with a ? mark. An advert time farther than this in the future is + # refused. Five minutes: a node with no time source drifts by seconds each day, and + # a larger error is a clock that nobody set. + default=CLOCK_TOLERANCE_S, + minimum=30, + maximum=86400, + unit="s", + ), PrefSpec( key="section_spacing_min_rows", label="Section spacing", diff --git a/meshterm/persistence/db.py b/meshterm/persistence/db.py index 43716bd..6e8b7b2 100644 --- a/meshterm/persistence/db.py +++ b/meshterm/persistence/db.py @@ -12,7 +12,7 @@ import json import sqlite3 from pathlib import Path -SCHEMA_VERSION = 20 +SCHEMA_VERSION = 21 _SCHEMA = """ CREATE TABLE IF NOT EXISTS schema_meta ( @@ -165,7 +165,11 @@ CREATE TABLE IF NOT EXISTS messages ( created_at TEXT NOT NULL, scope TEXT, -- outbound floods: the region it was sent under, '*' for -- unscoped, NULL where it was never known - author TEXT -- a room post: its author's key prefix (peer is the room) + author TEXT, -- a room post: its author's key prefix (peer is the room) + received_at TEXT, -- inbound: our clock when we got it from the companion + -- (created_at is the sender's clock); NULL outbound/old + waited INTEGER NOT NULL DEFAULT 0 -- inbound: it came out of the companion's queue + -- at connect, so it arrived some time before received_at ); CREATE INDEX IF NOT EXISTS idx_traces_run ON traces(run_id); @@ -396,6 +400,17 @@ def _migrate(conn: sqlite3.Connection) -> None: # Rows from before this column are NULL too: when MeshTerm stored such a post, the # author was lost on the way in, and nothing on disk can bring it back. conn.execute("ALTER TABLE messages ADD COLUMN author TEXT") + if "received_at" not in message_cols: + # v20 -> v21: the arrival time of a received message. ``created_at`` holds the time + # that the sender put on the message, from its own clock, and a node with a wrong + # clock makes that time false by hours or by years. ``received_at`` is our clock + # when MeshTerm got the message from the companion. ``waited`` is 1 when the message + # came out of the queue of the companion at connect, so that it arrived at an + # unknown time before that. Old rows keep NULL and 0: MeshTerm did not keep the + # time, and nothing can bring it back. Each query uses ``created_at`` for them, as + # before (refer to Repository._ARRIVED_AT). + conn.execute("ALTER TABLE messages ADD COLUMN received_at TEXT") + conn.execute("ALTER TABLE messages ADD COLUMN waited INTEGER NOT NULL DEFAULT 0") stored = conn.execute("SELECT value FROM schema_meta WHERE key = 'version'").fetchone() if stored is not None and int(stored["value"]) < 19: diff --git a/meshterm/persistence/repository.py b/meshterm/persistence/repository.py index f8cf3c3..8baebb3 100644 --- a/meshterm/persistence/repository.py +++ b/meshterm/persistence/repository.py @@ -87,6 +87,19 @@ _INSERT_OBSERVATION = ( "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" ) +#: The SQL form of :attr:`~meshterm.core.models.ChatMessage.arrived_at`: the best time on +#: our clock for when a ``messages`` row came to us. The arrival time, if it is known. For a +#: message that waited in the companion, the earlier of the sender time and the arrival +#: time. For our own messages and the rows from before MeshTerm kept the arrival time, +#: ``created_at``. Each query about our side (heard times, the order of conversations, +#: channel activity) uses this, never ``created_at`` alone, because a node with a wrong +#: clock writes a false ``created_at``. The two columns hold UTC ISO-8601 text, so the +#: comparison of the text is a comparison of the times (to the second). +_ARRIVED_AT = ( + "COALESCE(CASE WHEN waited = 1 AND created_at < received_at " + "THEN created_at ELSE received_at END, created_at)" +) + def _packet_raw(row: sqlite3.Row) -> dict | None: """Rebuild the minimum raw payload with which MeshTerm reads a stored ``packet`` row. @@ -2037,7 +2050,7 @@ class Repository: # -- direct messages: totals, our share, and the latest, matched by prefix ---- dm_rows = self._conn.execute( "SELECT peer, COUNT(*) AS total, SUM(outbound) AS sent, " - "MAX(created_at) AS last FROM messages " + f"MAX({_ARRIVED_AT}) AS last FROM messages " "WHERE is_channel = 0 AND peer IS NOT NULL GROUP BY peer" ).fetchall() @@ -2116,8 +2129,8 @@ class Repository: cur = self._conn.execute( "INSERT INTO messages " "(run_id, outbound, is_channel, channel_id, channel_idx, peer, peer_name, text, " - "snr, acked, created_at, scope, author) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "snr, acked, created_at, scope, author, received_at, waited) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( run_id, int(msg.outbound), @@ -2132,6 +2145,8 @@ class Repository: msg.created_at.isoformat(), msg.scope, msg.author.lower() if msg.author else None, + msg.received_at.isoformat() if msg.received_at else None, + int(msg.waited), ), ) self._conn.commit() @@ -2287,6 +2302,10 @@ class Repository: Outbound messages are not included: to send to a node is not to hear from it. + The time is the arrival time (:data:`_ARRIVED_AT`), not the sender time. The sender + time comes from the clock of the sender. When this method read it, one message from + a node whose clock was a year ahead showed that node as "heard now" for a year. + Returns: The latest time of an inbound message, keyed by the stored peer prefix (in lowercase, as :meth:`record_chat_message` writes it), in aware UTC. The prefix @@ -2294,7 +2313,7 @@ class Repository: match it as a prefix, not by equality. """ rows = self._conn.execute( - "SELECT peer, MAX(created_at) AS last FROM messages " + f"SELECT peer, MAX({_ARRIVED_AT}) AS last FROM messages " "WHERE is_channel = 0 AND outbound = 0 AND peer IS NOT NULL GROUP BY peer" ).fetchall() return {row["peer"]: datetime.fromisoformat(row["last"]) for row in rows} @@ -2325,9 +2344,14 @@ class Repository: version of this clamp gave a received message an empty window. In one direction, the stamps are consistent, so the neighbours mean what they say. + The times are arrival times (:data:`_ARRIVED_AT`). For a received message that has + one, our clock stamps it, as it stamps a send. The direction rule stays, because + each row from before MeshTerm kept the arrival time still has only the sender time. + Args: peer: The key prefix of the contact, as :meth:`record_chat_message` stores it. - when: The time of the message. + when: The arrival time of the message + (:attr:`~meshterm.core.models.ChatMessage.arrived_at`). outbound: The direction in which to look: the direction of the message. Returns: @@ -2338,14 +2362,14 @@ class Repository: key = (peer or "").lower() args = (key, int(outbound), when.isoformat()) row = self._conn.execute( - "SELECT MAX(created_at) AS edge FROM messages " - "WHERE is_channel = 0 AND peer = ? AND outbound = ? AND created_at < ?", + f"SELECT MAX({_ARRIVED_AT}) AS edge FROM messages " + f"WHERE is_channel = 0 AND peer = ? AND outbound = ? AND {_ARRIVED_AT} < ?", args, ).fetchone() prev = _as_when(row["edge"] if row else None) row = self._conn.execute( - "SELECT MIN(created_at) AS edge FROM messages " - "WHERE is_channel = 0 AND peer = ? AND outbound = ? AND created_at > ?", + f"SELECT MIN({_ARRIVED_AT}) AS edge FROM messages " + f"WHERE is_channel = 0 AND peer = ? AND outbound = ? AND {_ARRIVED_AT} > ?", args, ).fetchone() return prev, _as_when(row["edge"] if row else None) @@ -2364,9 +2388,9 @@ class Repository: timestamps of the window, and Python puts them in buckets (a few hours of messages, so the row set stays small). - The timestamps are compared as strings. ``utcnow().isoformat()`` writes each - ``created_at`` (a fixed-width UTC ISO-8601 form). Thus the lexicographic order is - the chronological order, and the window cutoff does not have to parse each row. + The timestamps are the arrival times (:data:`_ARRIVED_AT`), compared as strings. + Each is a UTC ISO-8601 form. Thus the lexicographic order is the chronological + order, and the window cutoff does not have to parse each row. Messages from before the history had identity keys (a ``NULL`` ``channel_id``: refer to :meth:`backfill_channel_ids`) have no channel to count under, and the method skips them. @@ -2378,25 +2402,27 @@ class Repository: now = utcnow() cutoff = now - ACTIVITY_WINDOW bucket_span = ACTIVITY_WINDOW / ACTIVITY_BUCKETS + # The arrival times (_ARRIVED_AT), not the sender times. A sender with a wrong + # clock put its messages outside the window, or in the bucket of "now" for months. rows = self._conn.execute( - "SELECT channel_id, COUNT(*) AS total, MAX(created_at) AS last_at " + f"SELECT channel_id, COUNT(*) AS total, MAX({_ARRIVED_AT}) AS last_at " "FROM messages WHERE is_channel = 1 AND channel_id IS NOT NULL " "GROUP BY channel_id" ).fetchall() histograms: dict[str, list[int]] = {} recent_rows = self._conn.execute( - "SELECT channel_id, created_at FROM messages " - "WHERE is_channel = 1 AND channel_id IS NOT NULL AND created_at >= ?", + f"SELECT channel_id, {_ARRIVED_AT} AS arrived_at FROM messages " + f"WHERE is_channel = 1 AND channel_id IS NOT NULL AND {_ARRIVED_AT} >= ?", (cutoff.isoformat(),), ).fetchall() for row in recent_rows: try: - created_at = datetime.fromisoformat(row["created_at"]) + arrived_at = datetime.fromisoformat(row["arrived_at"]) # Bucket by age, so that the histogram comes out newest first (bucket 0 has # the current five minutes). This is the order that the sparkline widget # expects. - idx = min(ACTIVITY_BUCKETS - 1, int((now - created_at) / bucket_span)) + idx = min(ACTIVITY_BUCKETS - 1, int((now - arrived_at) / bucket_span)) except (TypeError, ValueError): continue # a malformed or naive timestamp does not go in a bucket histogram = histograms.setdefault(row["channel_id"], [0] * ACTIVITY_BUCKETS) @@ -2525,6 +2551,8 @@ class Repository: snr=row["snr"], acked=None if row["acked"] is None else bool(row["acked"]), created_at=datetime.fromisoformat(row["created_at"]), + received_at=_as_when(row["received_at"]) if "received_at" in row.keys() else None, + waited=bool(row["waited"]) if "waited" in row.keys() else False, row_id=row["id"], scope=row["scope"] if "scope" in row.keys() else None, author=row["author"] if "author" in row.keys() else None, diff --git a/meshterm/services/device_state.py b/meshterm/services/device_state.py index 03b96bf..2356bf7 100644 --- a/meshterm/services/device_state.py +++ b/meshterm/services/device_state.py @@ -49,10 +49,11 @@ from __future__ import annotations import asyncio import time from dataclasses import replace -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING from ..core.channel_probe import ChannelSlot, probe_channel_slots +from ..core.models import clock_tolerance_s, utcnow if TYPE_CHECKING: from ..context import AppContext @@ -246,6 +247,11 @@ class DeviceState: later reception. A contact that we never heard stays ``None`` and shows ``never``. This step is best effort: if a history read fails, the method returns the list without the merge. It never blocks the read. + + A time more than the clock tolerance in the future is not evidence, and the merge + ignores it. The messages stored before MeshTerm kept the arrival time have only the + sender time, and a sender with a clock that is ahead wrote a time in the future + there. If the merge took it, it pinned the contact at "heard now". """ try: heard = self._ctx.repo.last_heard_by_node() @@ -253,6 +259,7 @@ class DeviceState: except Exception as exc: # noqa: BLE001 - the history must never block a contacts read self._ctx.log.debug("devstate: last-heard merge skipped: %s", exc) return contacts + limit = utcnow() + timedelta(seconds=clock_tolerance_s()) merged: list[Contact] = [] for contact in contacts: ident = (contact.public_key or contact.key_prefix or "").lower() @@ -267,7 +274,7 @@ class DeviceState: for peer, when in messaged.items() if ident and peer and (ident.startswith(peer) or peer.startswith(ident)) ] - known = [a for a in (_aware(s) for s in stamps) if a is not None] + known = [a for a in (_aware(s) for s in stamps) if a is not None and a <= limit] latest = max(known) if known else None if latest is not None and latest != contact.last_seen: contact = replace(contact, last_seen=latest) diff --git a/meshterm/services/message_paths.py b/meshterm/services/message_paths.py index 5790ed9..79257b9 100644 --- a/meshterm/services/message_paths.py +++ b/meshterm/services/message_paths.py @@ -191,9 +191,10 @@ def channel_arrivals( The matching arrivals, oldest first. Resends of the same message are included, and each carries its ``resend`` counter. """ - frames = repo.packet_frames_between( - message.created_at - _CHANNEL_WINDOW, message.created_at + _CHANNEL_WINDOW - ) + # Around the arrival time, not the sender time: a sender with a wrong clock put the + # window hours or years away from its packets. + at = message.arrived_at + frames = repo.packet_frames_between(at - _CHANNEL_WINDOW, at + _CHANNEL_WINDOW) arrivals: list[Arrival] = [] for frame in frames: raw = frame.raw if isinstance(frame.raw, dict) else {} @@ -244,9 +245,12 @@ def direct_window(repo: Repository, message: ChatMessage) -> tuple[datetime, dat send can come before it. The window opens at the message (minus a short time for jitter) and goes to the next send. Retries come after the send, so all the span goes forward. - * **A received message** has the time of the sender's clock, which can be before or - after the time when we heard it. Its window stays centred, and it goes halfway to - the messages before and after it. + * **A received message** has its arrival time on our clock + (:attr:`~meshterm.core.models.ChatMessage.arrived_at`). Its packets arrived a moment + before it, and their copies from other relays a moment after. A row from before + MeshTerm kept the arrival time has only the sender time, which can be before or + after the packets. Thus its window stays centred, and it goes halfway to the + messages before and after it. The window is never narrower than :data:`_MIN_HALF_WINDOW` on each side. Thus two messages one second apart still each have a span to search. @@ -259,7 +263,7 @@ def direct_window(repo: Repository, message: ChatMessage) -> tuple[datetime, dat ``(start, end)`` for :meth:`~meshterm.persistence.repository.Repository .packet_frames_between`. """ - at = message.created_at + at = message.arrived_at start, end = at - _DIRECT_WINDOW, at + _DIRECT_WINDOW previous, following = repo.direct_message_bounds(message.peer, at, outbound=message.outbound) if message.outbound: @@ -291,17 +295,28 @@ def _pick_group(groups: dict[str, list[Arrival]], message: ChatMessage) -> list[ body. The log cannot tell which group is this message. Thus the clock decides, and it decides well. The packets of a send start at the time when it was composed, and they continue through the retries. Thus the correct group is the earliest one that did not - start before the message. The fallback to the nearest group covers an inbound message, - whose time comes from the sender's clock and can be a little after the packet that we - heard. + start before the message. + + A received message with an arrival time is the opposite case. The companion gives + MeshTerm the message after it received the packet. Thus the correct group is the + latest one that started before the arrival (with :data:`_MIN_HALF_WINDOW` of slack). + + The fallback to the nearest group covers a received message from before MeshTerm kept + the arrival time, and one that waited in the companion. Its time comes from the + sender's clock, and it can be a little before or after the packet that we heard. """ if not groups: return [] - at = message.created_at + at = message.arrived_at started = {mac: min(a.when for a in arrivals) for mac, arrivals in groups.items()} - after = [mac for mac, first in started.items() if first >= at] - if after: - return groups[min(after, key=lambda mac: started[mac])] + if not message.outbound and message.received_at is not None and not message.waited: + before = [mac for mac, first in started.items() if first <= at + _MIN_HALF_WINDOW] + if before: + return groups[max(before, key=lambda mac: started[mac])] + else: + after = [mac for mac, first in started.items() if first >= at] + if after: + return groups[min(after, key=lambda mac: started[mac])] return groups[min(started, key=lambda mac: abs(started[mac] - at))] diff --git a/meshterm/tools/chat.py b/meshterm/tools/chat.py index b494b9c..e8afdc1 100644 --- a/meshterm/tools/chat.py +++ b/meshterm/tools/chat.py @@ -47,6 +47,7 @@ from ..core.models import ( Contact, Conversation, NodeResolver, + clock_tolerance_s, is_direct_messageable, is_room, ) @@ -75,6 +76,8 @@ from ..ui.widgets import ( from .base import Tool, ToolResult, register if TYPE_CHECKING: + from datetime import datetime + from ..core.channel_probe import ChannelSlot from ..ui.report import Facts, Listing @@ -745,7 +748,9 @@ def _recency_key(conversation: Conversation, lasts: dict) -> tuple: """ last: ChatMessage | None = lasts.get(conversation.key) if last is not None: - return (0, -last.created_at.timestamp()) + # The arrival time, not the sender time. A sender with a clock that was behind put + # its conversation under old ones, where a new message was easy to miss. + return (0, -last.arrived_at.timestamp()) return (1, conversation.label.casefold()) @@ -986,8 +991,9 @@ def _title( # The relative age lane (right-aligned) is between the badge and the message text. Thus # the ages are in one neat column, and all the previews start at the same position. A # last message that names us has our white ``★`` at the left of its age, the same as - # the transcript has it at the left of the time (refer to ChatScreen._body_lines). - age = _ago(last.created_at) if last is not None else "" + # the transcript has it at the left of the time (refer to ChatScreen._body_lines). The + # age is from the arrival time, as the order of the list is (refer to _recency_key). + age = _ago(last.arrived_at) if last is not None else "" lane = f" {age:>{_AGE_WIDTH}}" if age and last is not None and _names_us(last, us): mark = len(lane) - len(age) - 2 # the mark, a space, then the age @@ -1208,14 +1214,32 @@ def _message_columns(*, author_lane: bool = False) -> tuple: conversation is not a room), so that one command answers in one shape. For the transcript of a room, ``author_lane`` also draws it as ``AUTHOR``, because there it is the column that the user wants to see. + + ``TIME`` is the time that the transcript of the menu shows: the sender time, or the + arrival time with a ``?`` after it when the clock of the sender was wrong (refer to + :meth:`~meshterm.core.models.ChatMessage.corrected`). The document gives the facts + instead: ``created_at`` (the time that the message carries), ``received_at`` (our + clock when MeshTerm got it, ``null`` for our own messages and old rows), and + ``clock_corrected``. """ from dataclasses import replace from ..ui import fields peer = fields.plain_only(fields.name("conversation", "PEER")) + when = fields.plain_only(fields.when("shown_at", "TIME")) + age = when.lanes[0] + + def shown(value: tuple[datetime, bool]) -> str: + # A space where there is no mark, so that the right-aligned times stay in a column. + at, corrected = value + return age.render(at) + ("?" if corrected else " ") + return ( - fields.when("created_at", "TIME"), + fields.hidden("created_at"), + fields.hidden("received_at"), + fields.hidden("clock_corrected"), + replace(when, lanes=(replace(age, render=shown),)), fields.word("direction", "DIR"), # One column for the other end of the conversation, whichever type it is. The # document keeps the two types apart, because a parser cannot tell a channel from a @@ -1292,6 +1316,7 @@ def _transcript( """ from ..ui.report import Listing + tolerance = clock_tolerance_s() rows = [] for message in messages: channel = ( @@ -1299,9 +1324,13 @@ def _transcript( if message.is_channel else None ) + corrected = message.corrected(tolerance) rows.append( { "created_at": message.created_at, + "received_at": message.received_at, + "clock_corrected": corrected, + "shown_at": (message.shown_at(tolerance), corrected), "direction": "out" if message.outbound else "in", "conversation": message.peer_name or message.peer or None, "node": None diff --git a/meshterm/ui/chat.py b/meshterm/ui/chat.py index 370e3c0..2bd05fb 100644 --- a/meshterm/ui/chat.py +++ b/meshterm/ui/chat.py @@ -29,7 +29,14 @@ from ..core import links from ..core.channels import MENTION, split_channel_sender from ..core.connection import ContactNotOnDeviceError from ..core.events import EventKind, MeshEvent -from ..core.models import ChatMessage, Contact, Conversation, Message, utcnow +from ..core.models import ( + ChatMessage, + Contact, + Conversation, + Message, + clock_tolerance_s, + utcnow, +) from ..core.regions import WILDCARD from ..platforms import Platform, on_platform from .compose import MentionEditor @@ -48,6 +55,7 @@ from .widgets import ( DESTINATION, ORIGIN, author_chip, + format_offset, hop_chip, mention_hop, mention_mark, @@ -62,6 +70,11 @@ if TYPE_CHECKING: from .node_detail_screen import ContactPages +#: The mark after a corrected time: the clock of the sender was wrong, so the line shows +#: the arrival time instead (JP, 2026-11-02). It is the ``?`` warn mark of the app, and it +#: takes the first of the two cells after the time, so the bodies stay in one column. +_CORRECTED = ("?", "warn") + #: The delivery marks for a *resolved* outbound direct message, at the end of its line: #: acknowledged, or transmitted but not acknowledged (^R can retry it). These are the #: ``✓``/``✗`` status marks of the app in their ok/err styles, not the ✅/❌ emoji, which @@ -367,6 +380,8 @@ class ChatScreen(Screen): # animation step of a spinner) renders again only the rows that changed. Refer to # _render_grouped. self._render_cache: dict | None = None + # The clock tolerance of the last render: which times show as corrected. + self._tolerance_s = clock_tolerance_s() # Strong references to the sends, resends, and spinner tickers that a key handler # starts. The event loop holds only a *weak* reference. Thus the garbage collector # can remove a task that nothing else names while it runs: the result is a message @@ -633,7 +648,8 @@ class ChatScreen(Screen): A change of the locks draws all of the transcript again, because the sender chip of a locked contact has a padlock (refer to :meth:`_group_header`). The locks change - rarely: ^K here, or the lock row of the node page of an author. + rarely: ^K here, or the lock row of the node page of an author. A change of the + clock tolerance does the same, because it decides which times are corrected. Most importantly, this works for each message. It is not a cached prefix that stops at the first such row. Thus a message stays cheap to draw again, however far @@ -647,11 +663,14 @@ class ChatScreen(Screen): cache = self._render_cache spacing = self.section_spacing() locks = self._pages.locked_keys() if self._pages is not None else frozenset() + tolerance = clock_tolerance_s() + self._tolerance_s = tolerance if ( cache is not None and cache["width"] == width and cache["spacing"] == spacing and cache["locks"] == locks + and cache["tolerance"] == tolerance ): cached_ids = cache["ids"] cached_acked = cache["acked"] @@ -683,7 +702,10 @@ class ChatScreen(Screen): for idx in range(total): message = messages[idx] - stamp = message.created_at.astimezone() + # The time that the message shows: its sender time, or its arrival time when + # the clock of the sender is wrong. The day divider follows the same time, so + # a sender with a wrong clock never puts a day from another year in the middle. + stamp = message.shown_at(tolerance).astimezone() day = stamp.date() sender, body = self._sender_and_body(message) # Group by (are-we-the-sender, display name), not by the name alone. Thus our @@ -754,6 +776,7 @@ class ChatScreen(Screen): "width": width, "spacing": spacing, "locks": locks, + "tolerance": tolerance, "ids": ids, "acked": ackeds, "selected": selecteds, @@ -788,7 +811,7 @@ class ChatScreen(Screen): return None def group_of(message: ChatMessage) -> tuple: - day = message.created_at.astimezone().date() + day = message.shown_at(self._tolerance_s).astimezone().date() return (day, message.outbound, self._sender_and_body(message)[0]) group = group_of(messages[selected]) @@ -872,8 +895,15 @@ class ChatScreen(Screen): body gets a QR code under it. The codes are side by side, at the indent of the body. Thus the user can open a link from the handheld on a phone, and does not have to type it. + + When the clock of the sender is wrong (:meth:`~meshterm.core.models.ChatMessage.corrected`), + the line shows the arrival time, with the ``?`` mark after it. On the selected + message, a line under the body then gives the time that the sender's clock wrote, + and how far that clock is off (JP, 2026-11-02). """ - stamp = message.created_at.astimezone() + tolerance = self._tolerance_s + corrected = message.corrected(tolerance) + stamp = message.shown_at(tolerance).astimezone() prefix = Text() prefix.append("❯ " if selected else " ", style="cursor" if selected else None) if self._names_us(body): @@ -881,7 +911,12 @@ class ChatScreen(Screen): prefix.append(" ") else: prefix.append(" ") - prefix.append(f"{stamp:%H:%M} ", style="cursor" if selected else "muted") + prefix.append(f"{stamp:%H:%M}", style="cursor" if selected else "muted") + if corrected: + prefix.append(*_CORRECTED) + else: + prefix.append(" ") + prefix.append(" ") body_text = self._body_text(body, message, selected=selected) whole = [m.span() for m in _WORD.finditer(body_text.plain) if links.urls(m.group())] # A mention is a chip, and a chip never breaks across two lines. @@ -889,6 +924,16 @@ class ChatScreen(Screen): lines = render_hanging( prefix, body_text, width, indent=prefix.cell_len, whole=whole, gutter=_URL_GUTTER ) + if selected and corrected and message.clock_offset_s is not None: + sent = message.created_at.astimezone() + note = Text() + note.append(f"{_CORRECTED[0]} ", style=_CORRECTED[1]) + note.append( + f"its clock: {sent:%Y-%m-%d %H:%M} · {format_offset(message.clock_offset_s)}", + style="muted", + ) + indent = prefix.cell_len + lines += render_hanging(Text(" " * indent), note, width, indent=indent) urls = links.urls(body) if _URL_CODES else [] if urls: lines += render_lines(qr_strip(urls, width, indent=prefix.cell_len), width) @@ -1386,7 +1431,7 @@ class ChatScreen(Screen): starts: list[int] = [] prev_day = None for i, message in enumerate(self._messages): - day = message.created_at.astimezone().date() + day = message.shown_at(self._tolerance_s).astimezone().date() if day != prev_day: starts.append(i) prev_day = day diff --git a/meshterm/ui/message_paths_screen.py b/meshterm/ui/message_paths_screen.py index b69ed56..d2c8e92 100644 --- a/meshterm/ui/message_paths_screen.py +++ b/meshterm/ui/message_paths_screen.py @@ -58,7 +58,7 @@ from typing import TYPE_CHECKING from rich.text import Text -from ..core.models import ChatMessage +from ..core.models import ChatMessage, clock_tolerance_s from ..services.message_paths import Arrival from .pathgraph import PathLayer, render_path_graph, revisited_hops from .pathline import ( @@ -308,7 +308,13 @@ class MessagePathsScreen(Screen): quoted = self._message.text.replace("\n", " ") if len(quoted) > 64: quoted = quoted[:63] + "…" - stamp = Text(self._message.created_at.astimezone().strftime("%b %d %H:%M"), style="muted") + # The time that the transcript shows, with the same ? mark when the clock of the + # sender was wrong (refer to ChatMessage.corrected). + tolerance = clock_tolerance_s() + shown = self._message.shown_at(tolerance).astimezone() + stamp = Text(shown.strftime("%b %d %H:%M"), style="muted") + if self._message.corrected(tolerance): + stamp.append("?", style="warn") stamp.append(" · ", style="muted") stamp.append(self._summary, style="muted" if self._matched else "warn") diff --git a/meshterm/ui/widgets.py b/meshterm/ui/widgets.py index bc9c67f..15c9103 100644 --- a/meshterm/ui/widgets.py +++ b/meshterm/ui/widgets.py @@ -1219,6 +1219,43 @@ def format_age(secs: float | None) -> str: return f"{int(secs // 604800)}w" +#: The units of :func:`format_offset`, largest first: (suffix, seconds). +_OFFSET_UNITS: tuple[tuple[str, int], ...] = ( + ("y", 365 * 86400), + ("w", 604800), + ("d", 86400), + ("h", 3600), + ("m", 60), + ("s", 1), +) + + +def format_offset(secs: float) -> str: + """How far a clock is off, in words: ``3h 12m behind``, ``45s ahead``, ``1y 44w behind``. + + The largest unit, and the next unit when it is not zero, because one unit alone hides + too much: a clock that is 1 h 59 m behind is not "1h" behind. A positive value is a + clock ahead of ours. + + Args: + secs: The sender time minus our time, in seconds. + + Returns: + The size and the direction, or ``in sync`` for less than one second. + """ + left = int(abs(secs)) + if left == 0: + return "in sync" + parts: list[str] = [] + for suffix, size in _OFFSET_UNITS: + count, left = divmod(left, size) + if count: + parts.append(f"{count}{suffix}") + if len(parts) == 2 or (parts and not count): + break + return f"{' '.join(parts)} {'ahead' if secs > 0 else 'behind'}" + + def body_heading(title: str, note: str = "") -> Text: """A section heading in a screen body: an accent title, and an optional muted `` · note``. diff --git a/tests/test_clock_skew.py b/tests/test_clock_skew.py new file mode 100644 index 0000000..3951efb --- /dev/null +++ b/tests/test_clock_skew.py @@ -0,0 +1,297 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A sender with a wrong clock: the two times of a message, and what each one is for. + +A received message carries the sender time, from the clock of the sender. MeshTerm also +keeps the arrival time, from our clock. Each part of the app that is about our side (heard +times, the order of conversations, channel activity, the search of the packet log) uses +the arrival time. The transcript shows the sender time while it is correct, and the arrival +time with a ``?`` mark when it is not. +""" + +from __future__ import annotations + +import sqlite3 +from datetime import timedelta +from pathlib import Path + +from meshterm.core.connection import message_from_event +from meshterm.core.models import ( + CLOCK_TOLERANCE_S, + ChatMessage, + Contact, + Conversation, + Message, + utcnow, +) +from meshterm.persistence import db +from meshterm.persistence.repository import Repository +from meshterm.services.message_paths import direct_arrivals +from meshterm.tools.chat import _transcript +from meshterm.ui.chat import ChatScreen +from meshterm.ui.widgets import format_offset +from tests.conftest import plain as _strip_ansi + +TOLERANCE = CLOCK_TOLERANCE_S +PEER = "d4e5f6a7b8c9" + + +def _received(sender_offset: timedelta, *, waited: bool = False, **kw) -> ChatMessage: + """A received direct message whose sender clock is ``sender_offset`` from ours.""" + now = utcnow() + message = Message( + text="hello", + sender=PEER, + sender_timestamp=now + sender_offset, + received_at=now, + waited=waited, + **kw, + ) + return ChatMessage.from_message(message) + + +class _StubSession: + def invalidate(self) -> None: + pass + + +# -- the rules on ChatMessage --------------------------------------------------------------- + + +def test_a_correct_clock_shows_the_sender_time() -> None: + """A sender time within the tolerance is the time that the transcript shows.""" + message = _received(timedelta(seconds=-40)) + assert not message.corrected(TOLERANCE) + assert message.shown_at(TOLERANCE) == message.created_at + assert message.arrived_at == message.received_at + + +def test_a_clock_behind_or_ahead_shows_the_arrival_time() -> None: + """A sender time past the tolerance, in either direction, is wrong.""" + for offset in (timedelta(days=-700), timedelta(hours=3)): + message = _received(offset) + assert message.corrected(TOLERANCE) + assert message.shown_at(TOLERANCE) == message.received_at + assert message.arrived_at == message.received_at + + +def test_a_message_that_waited_is_wrong_only_in_the_future() -> None: + """A message from the queue of the companion arrived some time before MeshTerm got it. + + An old sender time is then possible and correct, so only a time in the future is proof. + Its arrival is the earlier of the two times. + """ + old = _received(timedelta(hours=-3), waited=True) + assert not old.corrected(TOLERANCE) + assert old.arrived_at == old.created_at + + ahead = _received(timedelta(hours=3), waited=True) + assert ahead.corrected(TOLERANCE) + assert ahead.arrived_at == ahead.received_at + + +def test_a_room_post_from_the_backlog_keeps_its_old_time() -> None: + """A room sends old posts with the times when they were posted. Those times are correct.""" + post = _received(timedelta(days=-3), author="a1b2c3d4") + assert post.is_post + assert not post.corrected(TOLERANCE) + assert _received(timedelta(days=3), author="a1b2c3d4").corrected(TOLERANCE) + + +def test_our_messages_and_old_rows_have_no_arrival_time() -> None: + """Our clock stamps our own messages. A row from before the arrival time keeps its time.""" + now = utcnow() + sent = ChatMessage(text="hi", outbound=True, created_at=now) + old = ChatMessage(text="hi", created_at=now - timedelta(days=900)) + for message in (sent, old): + assert message.clock_offset_s is None + assert not message.corrected(TOLERANCE) + assert message.arrived_at == message.created_at + + +def test_the_offset_reads_in_two_units() -> None: + """The largest unit, and the next one when it is not zero.""" + assert format_offset(0) == "in sync" + assert format_offset(45) == "45s ahead" + assert format_offset(-(3 * 3600 + 12 * 60 + 5)) == "3h 12m behind" + assert format_offset(-(2 * 3600 + 30)) == "2h behind" + assert format_offset(-(365 * 86400 + 44 * 604800)) == "1y 44w behind" + + +# -- the companion queue -------------------------------------------------------------------- + + +class _Event: + def __init__(self, payload: dict) -> None: + self.payload = payload + + +def test_a_message_from_the_queue_is_marked() -> None: + """The pump marks each message of the first drain as one that waited.""" + payload = {"text": "hi", "pubkey_prefix": PEER, "sender_timestamp": 1_700_000_000} + assert message_from_event(_Event(payload), waited=True).waited + assert not message_from_event(_Event(payload)).waited + + +# -- the database --------------------------------------------------------------------------- + + +def test_the_two_times_survive_a_round_trip(tmp_path: Path) -> None: + """The arrival time and the waited flag are stored and read back.""" + repo = Repository(tmp_path / "chat.db") + message = _received(timedelta(days=-700), waited=True) + repo.record_chat_message(message) + (back,) = repo.recent_chat_messages(is_channel=False, peer=PEER) + assert back.received_at == message.received_at + assert back.waited is True + assert back.created_at == message.created_at + repo.close() + + +def test_a_dm_from_a_clock_ahead_does_not_count_as_heard_now(tmp_path: Path) -> None: + """The heard evidence is the arrival time, not the sender time (it was the sender time).""" + repo = Repository(tmp_path / "chat.db") + message = _received(timedelta(days=365)) + repo.record_chat_message(message) + heard = repo.last_message_by_peer()[PEER] + assert abs((heard - message.received_at).total_seconds()) < 1 + repo.close() + + +def test_channel_activity_counts_by_arrival(tmp_path: Path) -> None: + """A channel message from a clock two years behind still counts in the recent activity.""" + repo = Repository(tmp_path / "chat.db") + now = utcnow() + message = ChatMessage.from_message( + Message( + text="Bob: hi", + channel=0, + is_channel=True, + sender_timestamp=now - timedelta(days=730), + received_at=now, + ), + channel_id="chan-1", + ) + repo.record_chat_message(message) + stats = repo.channel_stats()["chan-1"] + assert stats.recent == 1 + assert stats.histogram[0] == 1 + repo.close() + + +def test_the_migration_adds_the_columns_and_leaves_old_rows_alone(tmp_path: Path) -> None: + """A database from before the arrival time gets the columns. Its rows keep NULL and 0.""" + path = tmp_path / "old.db" + conn = sqlite3.connect(path) + conn.executescript( + """ + CREATE TABLE messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, run_id INTEGER, + outbound INTEGER NOT NULL DEFAULT 0, is_channel INTEGER NOT NULL DEFAULT 0, + channel_id TEXT, channel_idx INTEGER, peer TEXT, peer_name TEXT, + text TEXT NOT NULL, snr REAL, acked INTEGER, created_at TEXT NOT NULL, + scope TEXT, author TEXT + ); + INSERT INTO messages (peer, text, created_at) + VALUES ('d4e5f6a7b8c9', 'old', '2026-01-01T00:00:00+00:00'); + """ + ) + conn.commit() + conn.close() + + migrated = db.connect(path) + row = migrated.execute("SELECT received_at, waited FROM messages").fetchone() + assert row["received_at"] is None and row["waited"] == 0 + migrated.close() + + +# -- the packet log ------------------------------------------------------------------------- + + +def test_the_paths_of_a_received_message_are_found_by_its_arrival(tmp_path: Path) -> None: + """A sender clock a year behind does not move the search away from the packets. + + The packets of a received message come just before it arrives, so the correct MAC group + is the last one that started before the arrival, not the next one. + """ + from meshterm.core.models import Observation + + repo = Repository(tmp_path / "paths.db") + run = repo.start_run("monitor", {}, None) + now = utcnow() + for offset, mac in ((-20, "aaaa"), (-1, "beef"), (12, "f00d")): + repo.record_observation( + run, + Observation( + node=None, + kind="packet", + path="3d63", + snr=2.0, + observed_at=now + timedelta(seconds=offset), + raw={ + "payload_typename": "TEXT_MSG", + "src_hash": "d4", + "dest_hash": "a1", + "cipher_mac": mac, + }, + ), + ) + message = ChatMessage( + text="hi", + peer=PEER, + created_at=now - timedelta(days=365), + received_at=now, + ) + got, exact = direct_arrivals(repo, message, self_key="a1" + "0" * 62, peer_key=PEER) + assert exact is True + assert [a.frame["cipher_mac"] for a in got] == ["beef"] + repo.close() + + +# -- the screens ---------------------------------------------------------------------------- + + +def _screen(messages: list[ChatMessage]) -> ChatScreen: + contact = Contact(name="Bob", key_prefix=PEER) + conv = Conversation(label="Bob", is_channel=False, contact=contact) + return ChatScreen(conv, messages, send=None, names={PEER: "Bob"}, session=_StubSession()) + + +def test_the_transcript_marks_a_corrected_time() -> None: + """The corrected time shows the arrival, then ``?``. A correct time has no mark.""" + good = _received(timedelta(seconds=-10)) + bad = _received(timedelta(days=-700)) + screen = _screen([good, bad]) + lines = [_strip_ansi(line) for line in screen._render_grouped(72)] + arrival = f"{bad.received_at.astimezone():%H:%M}" + assert any(f"{arrival}? hello" in line for line in lines) + assert any(f"{good.created_at.astimezone():%H:%M} hello" in line for line in lines) + # No divider for a day two years ago: the day follows the time that the line shows. + assert not any(f"{bad.created_at.astimezone():%b}" in line and "──" in line for line in lines) + + +def test_a_picked_corrected_message_gives_the_sender_time() -> None: + """On the selected message, a line under the body says what the sender's clock wrote.""" + bad = _received(timedelta(days=-700)) + screen = _screen([bad]) + screen._selected = 0 + text = "\n".join(_strip_ansi(line) for line in screen._render_grouped(72)) + sent = bad.created_at.astimezone() + assert f"? its clock: {sent:%Y-%m-%d %H:%M}" in text + assert "behind" in text + + screen._selected = None + assert "its clock:" not in "\n".join( + _strip_ansi(line) for line in screen._render_grouped(72) + ) + + +def test_the_cli_history_marks_a_corrected_time_and_gives_both_times() -> None: + """The plain face puts ``?`` after the time. The document has the two times and the flag.""" + bad = _received(timedelta(days=-700)) + listing = _transcript([bad]) + (row,) = listing.rows + assert row["clock_corrected"] is True + assert row["created_at"] == bad.created_at + assert row["received_at"] == bad.received_at + time_lane = next(c for c in listing.columns if c.lanes and c.lanes[0].header == "TIME") + assert time_lane.lanes[0].render(row["shown_at"]).endswith("?") diff --git a/tests/test_skeleton.py b/tests/test_skeleton.py index af4ab79..ed3def2 100644 --- a/tests/test_skeleton.py +++ b/tests/test_skeleton.py @@ -34,13 +34,13 @@ def test_advert_time_refuses_a_future_sender_clock() -> None: tolerance) stays: its age clamps to "now" for a short time and corrects itself. The usual never-heard forms (zero, absent, or garbage) still become None. """ - from meshterm.core.models import ADVERT_FUTURE_SKEW_S, advert_time, utcnow + from meshterm.core.models import CLOCK_TOLERANCE_S, advert_time, utcnow past = int(utcnow().timestamp()) - 3600 when = advert_time(past) assert when is not None and int(when.timestamp()) == past assert advert_time(int(utcnow().timestamp()) + 60) is not None # normal skew - assert advert_time(int(utcnow().timestamp()) + ADVERT_FUTURE_SKEW_S + 60) is None + assert advert_time(int(utcnow().timestamp()) + CLOCK_TOLERANCE_S + 60) is None assert advert_time(0) is None assert advert_time(None) is None assert advert_time("garbage") is None