Each node gets a clock offset, and its node page shows it

ADVERT packets now keep their own time (observations.sent_at, schema 22).
With the arrival times of messages, each advert, DM, and channel message
is a clock sample: the sender time minus ours. A channel sender counts
for the node its name resolves to, or under its name. A node's offset is
the median of its three newest samples. The node page's Info tab shows
"clock 3h 12m behind" in warn past the tolerance, or "in sync".

Co-Authored-By: Claude Haiku 5.5 <noreply@anthropic.com>
This commit is contained in:
Jean-Pierre MartineauandClaude Haiku 5.5 committed 2026-11-03 00:12:45 -05:00
1 parent 2cf0e09eb4
commit adb79d051a
6 files changed
+484 -6

No files matched your search

+2 -1
View File
@@ -196,7 +196,8 @@ and in quoted text.
| message | A text message, direct or on a channel | |
| sender time | The time that a message or an advert carries, from the clock of the node that sent it. It is a claim, and a node with a wrong clock makes it false | timestamp (alone) |
| arrival time | Our clock when MeshTerm got a received message from the companion (`received_at`) | receive time |
| clock offset | The sender time minus the arrival time. A positive offset is a clock ahead of ours | skew, drift (for this) |
| clock offset | The sender time minus the arrival time. A positive offset is a clock ahead of ours. The offset of a node is the median of its three newest clock samples | skew, drift (for this) |
| clock sample | One packet that gives a clock offset: an advert, a direct message, or a channel message, with its sender time and our time | |
| corrected time | The arrival time that the screen shows in place of a wrong sender time, with the `?` mark | |
| send | MeshTerm gives a message, a packet, or a command to the device | emit |
| transmit | A radio puts a packet on the air | |
+20 -2
View File
@@ -12,7 +12,7 @@ import json
import sqlite3
from pathlib import Path
SCHEMA_VERSION = 21
SCHEMA_VERSION = 22
_SCHEMA = """
CREATE TABLE IF NOT EXISTS schema_meta (
@@ -143,8 +143,10 @@ CREATE TABLE IF NOT EXISTS observations (
trace_snrs TEXT, -- TRACE rows: comma-separated per-hop SNR readings (dB)
route TEXT, -- 'packet' rows: FLOOD/DIRECT/… — what `path` MEANS here
transport_code TEXT, -- scoped (TC_*) rows: the 4-byte transport-codes field (hex)
scope_body TEXT -- TC_FLOOD rows: payload type byte + payload (hex), the
scope_body TEXT, -- TC_FLOOD rows: payload type byte + payload (hex), the
-- HMAC input a region name is resolved against
sent_at TEXT -- ADVERT packet rows: the advert's own time, by the
-- sender's clock (observed_at is ours)
);
-- One chat message, sent or received, on a channel or with a contact. Unlike the other
@@ -411,6 +413,22 @@ def _migrate(conn: sqlite3.Connection) -> None:
# before (refer to Repository._ARRIVED_AT).
conn.execute("ALTER TABLE messages ADD COLUMN received_at TEXT")
conn.execute("ALTER TABLE messages ADD COLUMN waited INTEGER NOT NULL DEFAULT 0")
# The report of clock offsets reads the received messages of a time window by their
# arrival time.
conn.execute("CREATE INDEX IF NOT EXISTS idx_messages_received_at ON messages(received_at)")
if "sent_at" not in observation_cols:
# v21 -> v22: the own time of an advert. An advert carries the clock of its sender,
# and ``observed_at`` is our clock when we heard it. The difference is the clock
# offset of that node (refer to meshterm.services.clock_offsets). Only the RX log
# gives the time of an advert, so only ADVERT packet rows have it. Older rows have
# NULL: the raw packet was never stored, so nothing can bring the time back.
conn.execute("ALTER TABLE observations ADD COLUMN sent_at TEXT")
# Only the rows with an advert time, by node and then by time: the offset of one node
# reads its newest few, and the report reads a time window.
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_observations_sent ON observations(node, observed_at) "
"WHERE sent_at IS NOT NULL"
)
stored = conn.execute("SELECT value FROM schema_meta WHERE key = 'version'").fetchone()
if stored is not None and int(stored["value"]) < 19:
+103 -3
View File
@@ -12,7 +12,7 @@ import sqlite3
import statistics
from collections.abc import Iterable, Sequence
from dataclasses import dataclass
from datetime import datetime, timedelta
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import TYPE_CHECKING, Any
@@ -83,8 +83,8 @@ _INSERT_OBSERVATION = (
"INSERT INTO observations "
"(run_id, node, public_key, name, kind, node_type, snr, rssi, lat, lon, path, "
"observed_at, chan_hash, cipher_mac, crypted, payload_typename, dest, src, tag, "
"trace_snrs, route, transport_code, scope_body) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
"trace_snrs, route, transport_code, scope_body, sent_at) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
)
#: The SQL form of :attr:`~meshterm.core.models.ChatMessage.arrived_at`: the best time on
@@ -179,6 +179,16 @@ def _as_when(iso: Any) -> datetime | None:
return None
def _as_int_or_none(value: Any) -> int | None:
"""Read a whole number from a raw payload, or return ``None`` when it is not one."""
if isinstance(value, bool):
return None
try:
return int(value)
except (TypeError, ValueError):
return None
def _row_value(row: sqlite3.Row, key: str) -> Any:
"""Read ``key`` from a row. If the query did not select ``key``, return ``None``."""
try:
@@ -1168,6 +1178,18 @@ class Repository:
transport_code = raw.get("transport_code") if route in SCOPED_ROUTES else None
body = raw_scope_body(raw) if route == "TC_FLOOD" and transport_code else None
scope_body = body.hex() if body is not None else None
# The own time of an advert, by the clock of its sender. With ``observed_at`` (our
# clock), it gives the clock offset of the node (refer to
# :mod:`~meshterm.services.clock_offsets`). Zero is a clock that was never set,
# which is a reading too, but not a time: it stays NULL.
sent_at = None
if typename == "ADVERT":
stamp = _as_int_or_none(raw.get("adv_timestamp"))
if stamp:
try:
sent_at = datetime.fromtimestamp(stamp, tz=timezone.utc).isoformat()
except (OverflowError, OSError, ValueError):
sent_at = None
return (
run_id,
obs.node,
@@ -1192,6 +1214,7 @@ class Repository:
route,
transport_code,
scope_body,
sent_at,
)
def observation_count(self) -> int:
@@ -2318,6 +2341,83 @@ class Repository:
).fetchall()
return {row["peer"]: datetime.fromisoformat(row["last"]) for row in rows}
def advert_clock_samples(
self, *, since: datetime | None = None, node: str | None = None
) -> list[tuple[str, datetime, datetime]]:
"""The adverts that carry their own time: one clock sample for each.
Args:
since: Only the adverts heard at this time or later.
node: Only the adverts of this node (its 12-hex id).
Returns:
``(node, sent_at, observed_at)`` for each advert, oldest first. ``sent_at`` is
the clock of the node, and ``observed_at`` is ours.
"""
sql = "SELECT node, sent_at, observed_at FROM observations WHERE sent_at IS NOT NULL"
args: list[Any] = []
if node is not None:
sql += " AND node = ?"
args.append(node.lower()[:12])
if since is not None:
sql += " AND observed_at >= ?"
args.append(since.isoformat())
samples = []
for row in self._conn.execute(sql + " ORDER BY observed_at", args):
sent, heard = _as_when(row["sent_at"]), _as_when(row["observed_at"])
if row["node"] and sent is not None and heard is not None:
samples.append((row["node"], sent, heard))
return samples
def message_clock_samples(
self,
*,
since: datetime | None = None,
peer: str | None = None,
sender: str | None = None,
) -> list[tuple[bool, str | None, str, datetime, datetime]]:
"""The received messages with a known arrival time: one clock sample for each.
A message from the queue of the companion (``waited``) is not a sample, because its
arrival time is not known. A room post is not a sample either: its time is when its
author posted it, not when the room sent it. Our own messages have no sender clock.
Args:
since: Only the messages that arrived at this time or later.
peer: Only the direct messages of this key prefix (either one can be the
longer one).
sender: Only the channel messages whose ``Name: `` prefix is this name.
Returns:
``(is_channel, peer, text, created_at, received_at)`` for each message, oldest
first. ``created_at`` is the sender time, and ``received_at`` is ours.
"""
sql = (
"SELECT is_channel, peer, text, created_at, received_at FROM messages "
"WHERE outbound = 0 AND received_at IS NOT NULL AND waited = 0 AND author IS NULL"
)
args: list[Any] = []
if since is not None:
sql += " AND received_at >= ?"
args.append(since.isoformat())
if peer is not None:
key = peer.lower()
sql += (
" AND is_channel = 0 AND peer IS NOT NULL"
" AND (? LIKE peer || '%' OR peer LIKE ? || '%')"
)
args += [key, key]
if sender is not None:
prefix = f"{sender}:"
sql += " AND is_channel = 1 AND substr(text, 1, ?) = ?"
args += [len(prefix), prefix]
samples = []
for row in self._conn.execute(sql + " ORDER BY received_at", args):
sent, arrived = _as_when(row["created_at"]), _as_when(row["received_at"])
if sent is not None and arrived is not None:
samples.append((bool(row["is_channel"]), row["peer"], row["text"], sent, arrived))
return samples
def direct_message_bounds(
self, peer: str | None, when: datetime, *, outbound: bool
) -> tuple[datetime | None, datetime | None]:
+190
View File
@@ -0,0 +1,190 @@
# SPDX-License-Identifier: Apache-2.0
"""How far the clock of each node is from ours, measured from what it sent us.
Each packet that carries the time of its sender, and that we heard at a known time, is one
**clock sample**: the sender time minus our time. A packet takes only seconds to cross the
mesh, so the sample is the error of the sender's clock. Three kinds of packet give one:
* **An advert.** It carries the time when its node made it, and the RX log gives our time
(``observations.sent_at`` and ``observed_at``). It has a key, so the sample belongs to a
node with no doubt. Each type of node sends adverts, repeaters too.
* **A direct message**, by its sender time and its arrival time. It comes from a known key
prefix.
* **A channel message.** It has no sender field, only the ``Name: `` prefix of its text.
A name that resolves to exactly one key (refer to
:func:`~meshterm.services.trace_runner.make_name_key_resolver`) gives a sample for that
node. Each other name is a **sender with no key**, and its samples stay under the name.
A room post is not a sample: its time is when its author posted it. A message that waited
in the companion is not a sample: its arrival time is not known.
The **offset** of a node is the median of its three newest samples. Thus one wrong sample
has no effect, and when the owner sets the clock, the offset follows after two new samples.
"""
from __future__ import annotations
import statistics
from collections.abc import Callable, Iterable
from dataclasses import dataclass
from datetime import datetime
from typing import TYPE_CHECKING
from ..core.channels import split_channel_sender
if TYPE_CHECKING:
from ..persistence.repository import Repository
#: The number of the newest samples whose median is the offset of a node.
NEWEST = 3
#: Resolves a channel sender name to the key of its node, or ``None``.
KeyOf = Callable[[str], "str | None"]
@dataclass(frozen=True, slots=True)
class ClockSample:
"""One packet with the time of its sender and our time of arrival.
Attributes:
who: The 12-hex id of the node, or the sender name when :attr:`keyed` is false.
keyed: Whether a key identifies the sender.
sent_at: The time by the clock of the sender.
arrived_at: The time by our clock.
"""
who: str
keyed: bool
sent_at: datetime
arrived_at: datetime
@property
def offset_s(self) -> float:
"""The sender time minus our time, in seconds. Positive is a clock ahead of ours."""
return (self.sent_at - self.arrived_at).total_seconds()
@dataclass(frozen=True, slots=True)
class ClockOffset:
"""How far the clock of one node (or one sender with no key) is from ours.
Attributes:
who: The 12-hex id of the node, or the sender name when :attr:`keyed` is false.
keyed: Whether a key identifies the sender.
offset_s: The median of the newest :data:`NEWEST` samples, in seconds. Positive is
a clock ahead of ours.
samples: The number of samples in all.
last_at: Our time of the newest sample.
"""
who: str
keyed: bool
offset_s: float
samples: int
last_at: datetime
def out_of_sync(self, tolerance_s: float) -> bool:
"""Whether the clock is farther from ours than the ``clock_tolerance_s`` preference."""
return abs(self.offset_s) > tolerance_s
def offsets(samples: Iterable[ClockSample]) -> list[ClockOffset]:
"""Make one offset for each sender from its samples.
Args:
samples: The samples, in any order.
Returns:
One :class:`ClockOffset` for each sender, in no specific order.
"""
groups: dict[tuple[str, bool], list[ClockSample]] = {}
for sample in samples:
groups.setdefault((sample.who, sample.keyed), []).append(sample)
result = []
for (who, keyed), group in groups.items():
group.sort(key=lambda s: s.arrived_at)
newest = group[-NEWEST:]
result.append(
ClockOffset(
who=who,
keyed=keyed,
offset_s=float(statistics.median(s.offset_s for s in newest)),
samples=len(group),
last_at=newest[-1].arrived_at,
)
)
return result
def _channel_sample(
text: str, sent: datetime, arrived: datetime, key_of: KeyOf | None
) -> ClockSample | None:
"""The sample of one channel message, under its node or under its sender name."""
name, _body = split_channel_sender(text)
if not name:
return None # no prefix: nobody to give the sample to
key = key_of(name) if key_of is not None else None
if key:
return ClockSample(key.lower().removeprefix("0x")[:12], True, sent, arrived)
return ClockSample(name, False, sent, arrived)
def window_samples(
repo: Repository, *, since: datetime | None, key_of: KeyOf | None
) -> list[ClockSample]:
"""All the clock samples that arrived since a time, from adverts and messages.
Args:
repo: The repository that holds the packet log and the messages.
since: The start of the window, or ``None`` for all the history.
key_of: Resolves a channel sender name to a key.
Returns:
The samples, oldest first within each source.
"""
samples = [
ClockSample(node, True, sent, heard)
for node, sent, heard in repo.advert_clock_samples(since=since)
]
for is_channel, peer, text, sent, arrived in repo.message_clock_samples(since=since):
if is_channel:
sample = _channel_sample(text, sent, arrived, key_of)
if sample is not None:
samples.append(sample)
elif peer:
samples.append(ClockSample(peer.lower()[:12], True, sent, arrived))
return samples
def node_offset(
repo: Repository, node: str, *, name: str | None = None, key_of: KeyOf | None = None
) -> ClockOffset | None:
"""The clock offset of one node, from all its history.
Args:
repo: The repository that holds the packet log and the messages.
node: The key of the node, or its 12-hex id.
name: The name of the node. Its channel messages count only when ``key_of``
resolves this name to this node, because another node can use the same name.
key_of: Resolves a channel sender name to a key.
Returns:
The offset, or ``None`` when no packet of the node carried a usable time.
"""
node_id = node.lower().removeprefix("0x")[:12]
samples = [
ClockSample(node_id, True, sent, heard)
for _node, sent, heard in repo.advert_clock_samples(node=node_id)
]
samples += [
ClockSample(node_id, True, sent, arrived)
for _chan, _peer, _text, sent, arrived in repo.message_clock_samples(peer=node_id)
]
resolved = key_of(name) if (name and key_of is not None) else None
if name and resolved and resolved.lower().removeprefix("0x")[:12] == node_id:
samples += [
ClockSample(node_id, True, sent, arrived)
for _chan, _peer, _text, sent, arrived in repo.message_clock_samples(sender=name)
]
found = offsets(samples)
return found[0] if found else None
+35
View File
@@ -113,6 +113,7 @@ from ..core.connection import ContactNotOnDeviceError, DeviceCommandError
from ..core.geo import haversine_km, usable_fix
from ..core.models import NODE_TYPE_LABELS, NODE_TYPE_REPEATER, Contact, utcnow
from ..platforms import Platform, on_platform
from ..services.clock_offsets import node_offset
from . import menus
from .mapcanvas import RGB
from .marks import SELF_MARK
@@ -147,6 +148,7 @@ from .widgets import (
_recency_style,
age_seconds,
format_ago,
format_offset,
highlighted_hash,
node_type_legend,
short_frame,
@@ -1555,6 +1557,14 @@ async def open_node_detail(
signal = _signal_row(hn)
if signal is not None:
info_rows.append(("signal", signal))
clock = _clock_row(
ctx,
node_id,
name,
make_name_key_resolver(contacts, stored_names, heard=named_at),
)
if clock is not None:
info_rows.append(("clock", clock))
if lat is not None and lon is not None:
info_rows.append(("where", _range_text(lat, lon, self_lat, self_lon)))
# The regions of a repeater: what it was last heard to relay, from the region store (its
@@ -2173,6 +2183,31 @@ def _signal_row(hn) -> Text | None: # noqa: ANN001 - Optional[HeardNode]
return text if text.plain else None
def _clock_row(ctx, node_id: str, name: str | None, key_of) -> Text | None: # noqa: ANN001
"""How far the clock of the node is from ours, or ``None`` when no packet told it.
The offset is in ``warn`` when it is past the ``clock_tolerance_s`` preference, so that
a node whose times are wrong is visible on its own page. The muted aside says how old
the newest sample is (refer to :mod:`~meshterm.services.clock_offsets`).
"""
if not node_id:
return None
try:
offset = node_offset(ctx.repo, node_id, name=name, key_of=key_of)
except Exception: # noqa: BLE001 - the page is useful without the clock line
return None
if offset is None:
return None
tolerance = ctx.preferences.clock_tolerance_s
text = Text()
if offset.out_of_sync(tolerance):
text.append(format_offset(offset.offset_s), style="warn")
else:
text.append("in sync")
text.append(f" · {format_ago(age_seconds(offset.last_at))}", style="muted")
return text
def _hop_hash(key: str | None, fallback: str, hash_bytes: int) -> str:
"""The hash of a key at ``hash_bytes`` width, or ``fallback`` when there is no usable key."""
hex_key = (key or "").lower().removeprefix("0x")
+134
View File
@@ -201,9 +201,143 @@ def test_the_migration_adds_the_columns_and_leaves_old_rows_alone(tmp_path: Path
migrated = db.connect(path)
row = migrated.execute("SELECT received_at, waited FROM messages").fetchone()
assert row["received_at"] is None and row["waited"] == 0
observation_cols = {r["name"] for r in migrated.execute("PRAGMA table_info(observations)")}
assert "sent_at" in observation_cols
migrated.close()
# -- the clock offset of a node -------------------------------------------------------------
NODE = "c0ffee123456"
def _advert(repo: Repository, run: int, node_key: str, sent, heard) -> None: # noqa: ANN001
"""An ADVERT packet in the RX log: the time of its node, and our time of reception."""
from meshterm.core.models import Observation
raw = {
"payload_typename": "ADVERT",
"adv_key": node_key,
"adv_timestamp": int(sent.timestamp()),
}
repo.record_observation(
run,
Observation(node=node_key[:12], kind="packet", path="", observed_at=heard, raw=raw),
)
def test_an_advert_keeps_its_own_time(tmp_path: Path) -> None:
"""The RX log gives the advert time. Other packets and a time of zero store nothing."""
from meshterm.core.models import Observation
repo = Repository(tmp_path / "obs.db")
run = repo.start_run("monitor", {}, None)
now = utcnow().replace(microsecond=0)
_advert(repo, run, NODE + "00" * 26, now - timedelta(hours=2), now)
never_set = {"payload_typename": "ADVERT", "adv_key": "ab" * 32, "adv_timestamp": 0}
repo.record_observation(
run, Observation(node="ab" * 6, kind="packet", path="", observed_at=now, raw=never_set)
)
(sample,) = repo.advert_clock_samples()
assert sample == (NODE, now - timedelta(hours=2), now)
repo.close()
def test_the_offset_is_the_median_of_the_newest_three() -> None:
"""One wrong sample has no effect, and the old samples do not count."""
from meshterm.services.clock_offsets import ClockSample, offsets
now = utcnow()
def sample(minutes_ago: int, offset_s: float) -> ClockSample:
arrived = now - timedelta(minutes=minutes_ago)
return ClockSample(NODE, True, arrived + timedelta(seconds=offset_s), arrived)
# Old samples a day behind, then the clock was set: two good samples and one glitch.
found = offsets(
[sample(90, -86400), sample(80, -86400), sample(30, 4), sample(20, 9000), sample(10, 6)]
)
(offset,) = found
assert offset.offset_s == 6
assert offset.samples == 5
assert offset.last_at == now - timedelta(minutes=10)
assert not offset.out_of_sync(TOLERANCE)
def test_a_channel_sender_counts_for_its_node_or_under_its_name(tmp_path: Path) -> None:
"""A name with one key gives samples to that node. Another name keeps its own samples."""
from meshterm.services.clock_offsets import offsets, window_samples
repo = Repository(tmp_path / "chat.db")
run = repo.start_run("monitor", {}, None)
now = utcnow().replace(microsecond=0)
_advert(
repo, run, NODE + "00" * 26, now - timedelta(hours=3, minutes=5), now - timedelta(minutes=5)
)
for text in ("Bob: hi", "Stranger: hello"):
repo.record_chat_message(
ChatMessage.from_message(
Message(
text=text,
channel=0,
is_channel=True,
sender_timestamp=now - timedelta(hours=3),
received_at=now,
),
channel_id="c",
)
)
keys = {"bob": NODE + "00" * 26}
samples = window_samples(repo, since=None, key_of=lambda name: keys.get(name.casefold()))
found = {(o.who, o.keyed): o for o in offsets(samples)}
assert found[(NODE, True)].samples == 2 # the advert and the channel message
assert found[(NODE, True)].offset_s == -3 * 3600
assert found[("Stranger", False)].samples == 1
repo.close()
def test_the_node_offset_joins_adverts_dms_and_its_channel_messages(tmp_path: Path) -> None:
"""The node page reads all the history of one node.
Its channel messages count only when its name resolves to it: another node can use
the same name.
"""
from meshterm.services.clock_offsets import node_offset
repo = Repository(tmp_path / "chat.db")
run = repo.start_run("monitor", {}, None)
now = utcnow().replace(microsecond=0)
_advert(repo, run, NODE + "00" * 26, now - timedelta(days=2), now - timedelta(hours=1))
repo.record_chat_message(
ChatMessage.from_message(
Message(
text="hi", sender=NODE, sender_timestamp=now - timedelta(days=2), received_at=now
)
)
)
repo.record_chat_message(
ChatMessage.from_message(
Message(
text="Bob: on the channel",
channel=0,
is_channel=True,
sender_timestamp=now - timedelta(days=2),
received_at=now,
),
channel_id="c",
)
)
mine = node_offset(repo, NODE, name="Bob", key_of=lambda _name: NODE + "00" * 26)
assert mine is not None and mine.samples == 3
assert mine.out_of_sync(TOLERANCE)
assert format_offset(mine.offset_s) == "2d behind"
other = node_offset(repo, NODE, name="Bob", key_of=lambda _name: "ab" * 32)
assert other is not None and other.samples == 2 # the name belongs to another key
assert node_offset(repo, "ab" * 6) is None
repo.close()
# -- the packet log -------------------------------------------------------------------------