diff --git a/docs/development/writing-style.md b/docs/development/writing-style.md index a6c5ac9..8f69113 100644 --- a/docs/development/writing-style.md +++ b/docs/development/writing-style.md @@ -196,7 +196,8 @@ and in quoted text. | message | A text message, direct or on a channel | | | sender time | The time that a message or an advert carries, from the clock of the node that sent it. It is a claim, and a node with a wrong clock makes it false | timestamp (alone) | | arrival time | Our clock when MeshTerm got a received message from the companion (`received_at`) | receive time | -| clock offset | The sender time minus the arrival time. A positive offset is a clock ahead of ours | skew, drift (for this) | +| clock offset | The sender time minus the arrival time. A positive offset is a clock ahead of ours. The offset of a node is the median of its three newest clock samples | skew, drift (for this) | +| clock sample | One packet that gives a clock offset: an advert, a direct message, or a channel message, with its sender time and our time | | | corrected time | The arrival time that the screen shows in place of a wrong sender time, with the `?` mark | | | send | MeshTerm gives a message, a packet, or a command to the device | emit | | transmit | A radio puts a packet on the air | | diff --git a/meshterm/persistence/db.py b/meshterm/persistence/db.py index 6e8b7b2..2bcaf1e 100644 --- a/meshterm/persistence/db.py +++ b/meshterm/persistence/db.py @@ -12,7 +12,7 @@ import json import sqlite3 from pathlib import Path -SCHEMA_VERSION = 21 +SCHEMA_VERSION = 22 _SCHEMA = """ CREATE TABLE IF NOT EXISTS schema_meta ( @@ -143,8 +143,10 @@ CREATE TABLE IF NOT EXISTS observations ( trace_snrs TEXT, -- TRACE rows: comma-separated per-hop SNR readings (dB) route TEXT, -- 'packet' rows: FLOOD/DIRECT/… — what `path` MEANS here transport_code TEXT, -- scoped (TC_*) rows: the 4-byte transport-codes field (hex) - scope_body TEXT -- TC_FLOOD rows: payload type byte + payload (hex), the + scope_body TEXT, -- TC_FLOOD rows: payload type byte + payload (hex), the -- HMAC input a region name is resolved against + sent_at TEXT -- ADVERT packet rows: the advert's own time, by the + -- sender's clock (observed_at is ours) ); -- One chat message, sent or received, on a channel or with a contact. Unlike the other @@ -411,6 +413,22 @@ def _migrate(conn: sqlite3.Connection) -> None: # before (refer to Repository._ARRIVED_AT). conn.execute("ALTER TABLE messages ADD COLUMN received_at TEXT") conn.execute("ALTER TABLE messages ADD COLUMN waited INTEGER NOT NULL DEFAULT 0") + # The report of clock offsets reads the received messages of a time window by their + # arrival time. + conn.execute("CREATE INDEX IF NOT EXISTS idx_messages_received_at ON messages(received_at)") + if "sent_at" not in observation_cols: + # v21 -> v22: the own time of an advert. An advert carries the clock of its sender, + # and ``observed_at`` is our clock when we heard it. The difference is the clock + # offset of that node (refer to meshterm.services.clock_offsets). Only the RX log + # gives the time of an advert, so only ADVERT packet rows have it. Older rows have + # NULL: the raw packet was never stored, so nothing can bring the time back. + conn.execute("ALTER TABLE observations ADD COLUMN sent_at TEXT") + # Only the rows with an advert time, by node and then by time: the offset of one node + # reads its newest few, and the report reads a time window. + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_observations_sent ON observations(node, observed_at) " + "WHERE sent_at IS NOT NULL" + ) stored = conn.execute("SELECT value FROM schema_meta WHERE key = 'version'").fetchone() if stored is not None and int(stored["value"]) < 19: diff --git a/meshterm/persistence/repository.py b/meshterm/persistence/repository.py index eea9d96..e8767a1 100644 --- a/meshterm/persistence/repository.py +++ b/meshterm/persistence/repository.py @@ -12,7 +12,7 @@ import sqlite3 import statistics from collections.abc import Iterable, Sequence from dataclasses import dataclass -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import TYPE_CHECKING, Any @@ -83,8 +83,8 @@ _INSERT_OBSERVATION = ( "INSERT INTO observations " "(run_id, node, public_key, name, kind, node_type, snr, rssi, lat, lon, path, " "observed_at, chan_hash, cipher_mac, crypted, payload_typename, dest, src, tag, " - "trace_snrs, route, transport_code, scope_body) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + "trace_snrs, route, transport_code, scope_body, sent_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" ) #: The SQL form of :attr:`~meshterm.core.models.ChatMessage.arrived_at`: the best time on @@ -179,6 +179,16 @@ def _as_when(iso: Any) -> datetime | None: return None +def _as_int_or_none(value: Any) -> int | None: + """Read a whole number from a raw payload, or return ``None`` when it is not one.""" + if isinstance(value, bool): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + def _row_value(row: sqlite3.Row, key: str) -> Any: """Read ``key`` from a row. If the query did not select ``key``, return ``None``.""" try: @@ -1168,6 +1178,18 @@ class Repository: transport_code = raw.get("transport_code") if route in SCOPED_ROUTES else None body = raw_scope_body(raw) if route == "TC_FLOOD" and transport_code else None scope_body = body.hex() if body is not None else None + # The own time of an advert, by the clock of its sender. With ``observed_at`` (our + # clock), it gives the clock offset of the node (refer to + # :mod:`~meshterm.services.clock_offsets`). Zero is a clock that was never set, + # which is a reading too, but not a time: it stays NULL. + sent_at = None + if typename == "ADVERT": + stamp = _as_int_or_none(raw.get("adv_timestamp")) + if stamp: + try: + sent_at = datetime.fromtimestamp(stamp, tz=timezone.utc).isoformat() + except (OverflowError, OSError, ValueError): + sent_at = None return ( run_id, obs.node, @@ -1192,6 +1214,7 @@ class Repository: route, transport_code, scope_body, + sent_at, ) def observation_count(self) -> int: @@ -2318,6 +2341,83 @@ class Repository: ).fetchall() return {row["peer"]: datetime.fromisoformat(row["last"]) for row in rows} + def advert_clock_samples( + self, *, since: datetime | None = None, node: str | None = None + ) -> list[tuple[str, datetime, datetime]]: + """The adverts that carry their own time: one clock sample for each. + + Args: + since: Only the adverts heard at this time or later. + node: Only the adverts of this node (its 12-hex id). + + Returns: + ``(node, sent_at, observed_at)`` for each advert, oldest first. ``sent_at`` is + the clock of the node, and ``observed_at`` is ours. + """ + sql = "SELECT node, sent_at, observed_at FROM observations WHERE sent_at IS NOT NULL" + args: list[Any] = [] + if node is not None: + sql += " AND node = ?" + args.append(node.lower()[:12]) + if since is not None: + sql += " AND observed_at >= ?" + args.append(since.isoformat()) + samples = [] + for row in self._conn.execute(sql + " ORDER BY observed_at", args): + sent, heard = _as_when(row["sent_at"]), _as_when(row["observed_at"]) + if row["node"] and sent is not None and heard is not None: + samples.append((row["node"], sent, heard)) + return samples + + def message_clock_samples( + self, + *, + since: datetime | None = None, + peer: str | None = None, + sender: str | None = None, + ) -> list[tuple[bool, str | None, str, datetime, datetime]]: + """The received messages with a known arrival time: one clock sample for each. + + A message from the queue of the companion (``waited``) is not a sample, because its + arrival time is not known. A room post is not a sample either: its time is when its + author posted it, not when the room sent it. Our own messages have no sender clock. + + Args: + since: Only the messages that arrived at this time or later. + peer: Only the direct messages of this key prefix (either one can be the + longer one). + sender: Only the channel messages whose ``Name: `` prefix is this name. + + Returns: + ``(is_channel, peer, text, created_at, received_at)`` for each message, oldest + first. ``created_at`` is the sender time, and ``received_at`` is ours. + """ + sql = ( + "SELECT is_channel, peer, text, created_at, received_at FROM messages " + "WHERE outbound = 0 AND received_at IS NOT NULL AND waited = 0 AND author IS NULL" + ) + args: list[Any] = [] + if since is not None: + sql += " AND received_at >= ?" + args.append(since.isoformat()) + if peer is not None: + key = peer.lower() + sql += ( + " AND is_channel = 0 AND peer IS NOT NULL" + " AND (? LIKE peer || '%' OR peer LIKE ? || '%')" + ) + args += [key, key] + if sender is not None: + prefix = f"{sender}:" + sql += " AND is_channel = 1 AND substr(text, 1, ?) = ?" + args += [len(prefix), prefix] + samples = [] + for row in self._conn.execute(sql + " ORDER BY received_at", args): + sent, arrived = _as_when(row["created_at"]), _as_when(row["received_at"]) + if sent is not None and arrived is not None: + samples.append((bool(row["is_channel"]), row["peer"], row["text"], sent, arrived)) + return samples + def direct_message_bounds( self, peer: str | None, when: datetime, *, outbound: bool ) -> tuple[datetime | None, datetime | None]: diff --git a/meshterm/services/clock_offsets.py b/meshterm/services/clock_offsets.py new file mode 100644 index 0000000..7b5cca1 --- /dev/null +++ b/meshterm/services/clock_offsets.py @@ -0,0 +1,190 @@ +# SPDX-License-Identifier: Apache-2.0 +"""How far the clock of each node is from ours, measured from what it sent us. + +Each packet that carries the time of its sender, and that we heard at a known time, is one +**clock sample**: the sender time minus our time. A packet takes only seconds to cross the +mesh, so the sample is the error of the sender's clock. Three kinds of packet give one: + +* **An advert.** It carries the time when its node made it, and the RX log gives our time + (``observations.sent_at`` and ``observed_at``). It has a key, so the sample belongs to a + node with no doubt. Each type of node sends adverts, repeaters too. +* **A direct message**, by its sender time and its arrival time. It comes from a known key + prefix. +* **A channel message.** It has no sender field, only the ``Name: `` prefix of its text. + A name that resolves to exactly one key (refer to + :func:`~meshterm.services.trace_runner.make_name_key_resolver`) gives a sample for that + node. Each other name is a **sender with no key**, and its samples stay under the name. + +A room post is not a sample: its time is when its author posted it. A message that waited +in the companion is not a sample: its arrival time is not known. + +The **offset** of a node is the median of its three newest samples. Thus one wrong sample +has no effect, and when the owner sets the clock, the offset follows after two new samples. +""" + +from __future__ import annotations + +import statistics +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from datetime import datetime +from typing import TYPE_CHECKING + +from ..core.channels import split_channel_sender + +if TYPE_CHECKING: + from ..persistence.repository import Repository + +#: The number of the newest samples whose median is the offset of a node. +NEWEST = 3 + +#: Resolves a channel sender name to the key of its node, or ``None``. +KeyOf = Callable[[str], "str | None"] + + +@dataclass(frozen=True, slots=True) +class ClockSample: + """One packet with the time of its sender and our time of arrival. + + Attributes: + who: The 12-hex id of the node, or the sender name when :attr:`keyed` is false. + keyed: Whether a key identifies the sender. + sent_at: The time by the clock of the sender. + arrived_at: The time by our clock. + """ + + who: str + keyed: bool + sent_at: datetime + arrived_at: datetime + + @property + def offset_s(self) -> float: + """The sender time minus our time, in seconds. Positive is a clock ahead of ours.""" + return (self.sent_at - self.arrived_at).total_seconds() + + +@dataclass(frozen=True, slots=True) +class ClockOffset: + """How far the clock of one node (or one sender with no key) is from ours. + + Attributes: + who: The 12-hex id of the node, or the sender name when :attr:`keyed` is false. + keyed: Whether a key identifies the sender. + offset_s: The median of the newest :data:`NEWEST` samples, in seconds. Positive is + a clock ahead of ours. + samples: The number of samples in all. + last_at: Our time of the newest sample. + """ + + who: str + keyed: bool + offset_s: float + samples: int + last_at: datetime + + def out_of_sync(self, tolerance_s: float) -> bool: + """Whether the clock is farther from ours than the ``clock_tolerance_s`` preference.""" + return abs(self.offset_s) > tolerance_s + + +def offsets(samples: Iterable[ClockSample]) -> list[ClockOffset]: + """Make one offset for each sender from its samples. + + Args: + samples: The samples, in any order. + + Returns: + One :class:`ClockOffset` for each sender, in no specific order. + """ + groups: dict[tuple[str, bool], list[ClockSample]] = {} + for sample in samples: + groups.setdefault((sample.who, sample.keyed), []).append(sample) + result = [] + for (who, keyed), group in groups.items(): + group.sort(key=lambda s: s.arrived_at) + newest = group[-NEWEST:] + result.append( + ClockOffset( + who=who, + keyed=keyed, + offset_s=float(statistics.median(s.offset_s for s in newest)), + samples=len(group), + last_at=newest[-1].arrived_at, + ) + ) + return result + + +def _channel_sample( + text: str, sent: datetime, arrived: datetime, key_of: KeyOf | None +) -> ClockSample | None: + """The sample of one channel message, under its node or under its sender name.""" + name, _body = split_channel_sender(text) + if not name: + return None # no prefix: nobody to give the sample to + key = key_of(name) if key_of is not None else None + if key: + return ClockSample(key.lower().removeprefix("0x")[:12], True, sent, arrived) + return ClockSample(name, False, sent, arrived) + + +def window_samples( + repo: Repository, *, since: datetime | None, key_of: KeyOf | None +) -> list[ClockSample]: + """All the clock samples that arrived since a time, from adverts and messages. + + Args: + repo: The repository that holds the packet log and the messages. + since: The start of the window, or ``None`` for all the history. + key_of: Resolves a channel sender name to a key. + + Returns: + The samples, oldest first within each source. + """ + samples = [ + ClockSample(node, True, sent, heard) + for node, sent, heard in repo.advert_clock_samples(since=since) + ] + for is_channel, peer, text, sent, arrived in repo.message_clock_samples(since=since): + if is_channel: + sample = _channel_sample(text, sent, arrived, key_of) + if sample is not None: + samples.append(sample) + elif peer: + samples.append(ClockSample(peer.lower()[:12], True, sent, arrived)) + return samples + + +def node_offset( + repo: Repository, node: str, *, name: str | None = None, key_of: KeyOf | None = None +) -> ClockOffset | None: + """The clock offset of one node, from all its history. + + Args: + repo: The repository that holds the packet log and the messages. + node: The key of the node, or its 12-hex id. + name: The name of the node. Its channel messages count only when ``key_of`` + resolves this name to this node, because another node can use the same name. + key_of: Resolves a channel sender name to a key. + + Returns: + The offset, or ``None`` when no packet of the node carried a usable time. + """ + node_id = node.lower().removeprefix("0x")[:12] + samples = [ + ClockSample(node_id, True, sent, heard) + for _node, sent, heard in repo.advert_clock_samples(node=node_id) + ] + samples += [ + ClockSample(node_id, True, sent, arrived) + for _chan, _peer, _text, sent, arrived in repo.message_clock_samples(peer=node_id) + ] + resolved = key_of(name) if (name and key_of is not None) else None + if name and resolved and resolved.lower().removeprefix("0x")[:12] == node_id: + samples += [ + ClockSample(node_id, True, sent, arrived) + for _chan, _peer, _text, sent, arrived in repo.message_clock_samples(sender=name) + ] + found = offsets(samples) + return found[0] if found else None diff --git a/meshterm/ui/node_detail_screen.py b/meshterm/ui/node_detail_screen.py index 8e9ef29..6414198 100644 --- a/meshterm/ui/node_detail_screen.py +++ b/meshterm/ui/node_detail_screen.py @@ -113,6 +113,7 @@ from ..core.connection import ContactNotOnDeviceError, DeviceCommandError from ..core.geo import haversine_km, usable_fix from ..core.models import NODE_TYPE_LABELS, NODE_TYPE_REPEATER, Contact, utcnow from ..platforms import Platform, on_platform +from ..services.clock_offsets import node_offset from . import menus from .mapcanvas import RGB from .marks import SELF_MARK @@ -147,6 +148,7 @@ from .widgets import ( _recency_style, age_seconds, format_ago, + format_offset, highlighted_hash, node_type_legend, short_frame, @@ -1555,6 +1557,14 @@ async def open_node_detail( signal = _signal_row(hn) if signal is not None: info_rows.append(("signal", signal)) + clock = _clock_row( + ctx, + node_id, + name, + make_name_key_resolver(contacts, stored_names, heard=named_at), + ) + if clock is not None: + info_rows.append(("clock", clock)) if lat is not None and lon is not None: info_rows.append(("where", _range_text(lat, lon, self_lat, self_lon))) # The regions of a repeater: what it was last heard to relay, from the region store (its @@ -2173,6 +2183,31 @@ def _signal_row(hn) -> Text | None: # noqa: ANN001 - Optional[HeardNode] return text if text.plain else None +def _clock_row(ctx, node_id: str, name: str | None, key_of) -> Text | None: # noqa: ANN001 + """How far the clock of the node is from ours, or ``None`` when no packet told it. + + The offset is in ``warn`` when it is past the ``clock_tolerance_s`` preference, so that + a node whose times are wrong is visible on its own page. The muted aside says how old + the newest sample is (refer to :mod:`~meshterm.services.clock_offsets`). + """ + if not node_id: + return None + try: + offset = node_offset(ctx.repo, node_id, name=name, key_of=key_of) + except Exception: # noqa: BLE001 - the page is useful without the clock line + return None + if offset is None: + return None + tolerance = ctx.preferences.clock_tolerance_s + text = Text() + if offset.out_of_sync(tolerance): + text.append(format_offset(offset.offset_s), style="warn") + else: + text.append("in sync") + text.append(f" · {format_ago(age_seconds(offset.last_at))}", style="muted") + return text + + def _hop_hash(key: str | None, fallback: str, hash_bytes: int) -> str: """The hash of a key at ``hash_bytes`` width, or ``fallback`` when there is no usable key.""" hex_key = (key or "").lower().removeprefix("0x") diff --git a/tests/test_clock_skew.py b/tests/test_clock_skew.py index 16d2a81..e1f4ea1 100644 --- a/tests/test_clock_skew.py +++ b/tests/test_clock_skew.py @@ -201,9 +201,143 @@ def test_the_migration_adds_the_columns_and_leaves_old_rows_alone(tmp_path: Path migrated = db.connect(path) row = migrated.execute("SELECT received_at, waited FROM messages").fetchone() assert row["received_at"] is None and row["waited"] == 0 + observation_cols = {r["name"] for r in migrated.execute("PRAGMA table_info(observations)")} + assert "sent_at" in observation_cols migrated.close() +# -- the clock offset of a node ------------------------------------------------------------- + +NODE = "c0ffee123456" + + +def _advert(repo: Repository, run: int, node_key: str, sent, heard) -> None: # noqa: ANN001 + """An ADVERT packet in the RX log: the time of its node, and our time of reception.""" + from meshterm.core.models import Observation + + raw = { + "payload_typename": "ADVERT", + "adv_key": node_key, + "adv_timestamp": int(sent.timestamp()), + } + repo.record_observation( + run, + Observation(node=node_key[:12], kind="packet", path="", observed_at=heard, raw=raw), + ) + + +def test_an_advert_keeps_its_own_time(tmp_path: Path) -> None: + """The RX log gives the advert time. Other packets and a time of zero store nothing.""" + from meshterm.core.models import Observation + + repo = Repository(tmp_path / "obs.db") + run = repo.start_run("monitor", {}, None) + now = utcnow().replace(microsecond=0) + _advert(repo, run, NODE + "00" * 26, now - timedelta(hours=2), now) + never_set = {"payload_typename": "ADVERT", "adv_key": "ab" * 32, "adv_timestamp": 0} + repo.record_observation( + run, Observation(node="ab" * 6, kind="packet", path="", observed_at=now, raw=never_set) + ) + (sample,) = repo.advert_clock_samples() + assert sample == (NODE, now - timedelta(hours=2), now) + repo.close() + + +def test_the_offset_is_the_median_of_the_newest_three() -> None: + """One wrong sample has no effect, and the old samples do not count.""" + from meshterm.services.clock_offsets import ClockSample, offsets + + now = utcnow() + + def sample(minutes_ago: int, offset_s: float) -> ClockSample: + arrived = now - timedelta(minutes=minutes_ago) + return ClockSample(NODE, True, arrived + timedelta(seconds=offset_s), arrived) + + # Old samples a day behind, then the clock was set: two good samples and one glitch. + found = offsets( + [sample(90, -86400), sample(80, -86400), sample(30, 4), sample(20, 9000), sample(10, 6)] + ) + (offset,) = found + assert offset.offset_s == 6 + assert offset.samples == 5 + assert offset.last_at == now - timedelta(minutes=10) + assert not offset.out_of_sync(TOLERANCE) + + +def test_a_channel_sender_counts_for_its_node_or_under_its_name(tmp_path: Path) -> None: + """A name with one key gives samples to that node. Another name keeps its own samples.""" + from meshterm.services.clock_offsets import offsets, window_samples + + repo = Repository(tmp_path / "chat.db") + run = repo.start_run("monitor", {}, None) + now = utcnow().replace(microsecond=0) + _advert( + repo, run, NODE + "00" * 26, now - timedelta(hours=3, minutes=5), now - timedelta(minutes=5) + ) + for text in ("Bob: hi", "Stranger: hello"): + repo.record_chat_message( + ChatMessage.from_message( + Message( + text=text, + channel=0, + is_channel=True, + sender_timestamp=now - timedelta(hours=3), + received_at=now, + ), + channel_id="c", + ) + ) + keys = {"bob": NODE + "00" * 26} + samples = window_samples(repo, since=None, key_of=lambda name: keys.get(name.casefold())) + found = {(o.who, o.keyed): o for o in offsets(samples)} + assert found[(NODE, True)].samples == 2 # the advert and the channel message + assert found[(NODE, True)].offset_s == -3 * 3600 + assert found[("Stranger", False)].samples == 1 + repo.close() + + +def test_the_node_offset_joins_adverts_dms_and_its_channel_messages(tmp_path: Path) -> None: + """The node page reads all the history of one node. + + Its channel messages count only when its name resolves to it: another node can use + the same name. + """ + from meshterm.services.clock_offsets import node_offset + + repo = Repository(tmp_path / "chat.db") + run = repo.start_run("monitor", {}, None) + now = utcnow().replace(microsecond=0) + _advert(repo, run, NODE + "00" * 26, now - timedelta(days=2), now - timedelta(hours=1)) + repo.record_chat_message( + ChatMessage.from_message( + Message( + text="hi", sender=NODE, sender_timestamp=now - timedelta(days=2), received_at=now + ) + ) + ) + repo.record_chat_message( + ChatMessage.from_message( + Message( + text="Bob: on the channel", + channel=0, + is_channel=True, + sender_timestamp=now - timedelta(days=2), + received_at=now, + ), + channel_id="c", + ) + ) + mine = node_offset(repo, NODE, name="Bob", key_of=lambda _name: NODE + "00" * 26) + assert mine is not None and mine.samples == 3 + assert mine.out_of_sync(TOLERANCE) + assert format_offset(mine.offset_s) == "2d behind" + + other = node_offset(repo, NODE, name="Bob", key_of=lambda _name: "ab" * 32) + assert other is not None and other.samples == 2 # the name belongs to another key + assert node_offset(repo, "ab" * 6) is None + repo.close() + + # -- the packet log -------------------------------------------------------------------------