diff --git a/meshterm/persistence/repository.py b/meshterm/persistence/repository.py index 3a88adc..a910d8d 100644 --- a/meshterm/persistence/repository.py +++ b/meshterm/persistence/repository.py @@ -1511,6 +1511,35 @@ class Repository: ).fetchall() return {row["node"]: row["name"] for row in rows} + def last_heard_by_node(self) -> dict[str, datetime]: + """The most recent reception per node, stamped by *our* clock. + + The evidence half of a contact's heard time. The device's contact table reports + ``last_advert``, which the advertising node stamped with its own clock — hearsay a + wrong RTC can hold days in the past forever — whereas every row here was written + when *we* received something (see + :meth:`~meshterm.services.device_state.DeviceState.contacts`, which takes the later + of the two). ``packet`` rows are excluded for the same reason + :meth:`heard_nodes` excludes them: a relayed frame tells us we heard its last + *relay*, not its originator. + + This is :meth:`heard_nodes` reduced to the one column that answers "when last?" — + the aggregate happens in SQLite and yields one row per node, rather than streaming + the whole history into Python to build per-node stat objects. That matters because + every contacts fetch calls this, and the row-building is the expensive half. + + Returns: + Latest reception time keyed by stored node id (the 12-hex key prefix), aware UTC. + """ + # ``NOT INDEXED`` for the same reason as :meth:`node_names`: the planner otherwise + # walks ``idx_observations_node`` with a random-access fetch per row, slower than + # the plain scan this aggregate wants. + rows = self._conn.execute( + "SELECT node, MAX(observed_at) AS last FROM observations NOT INDEXED " + "WHERE node IS NOT NULL AND kind != 'packet' GROUP BY node" + ).fetchall() + return {row["node"]: datetime.fromisoformat(row["last"]) for row in rows} + def heard_nodes(self, *, since: Optional[datetime] = None) -> list[HeardNode]: """Aggregate stored observations into per-node reception statistics. @@ -1703,6 +1732,30 @@ class Repository: ).fetchall() return {msg.key: msg for msg in (self._row_to_chat(r) for r in rows)} + def last_message_by_peer(self) -> dict[str, datetime]: + """The most recent *inbound* direct message per peer, stamped by our clock. + + The other half of the heard-time evidence (see :meth:`last_heard_by_node`). A direct + message received from a node is, in the app's own lexicon, hearing that node — but + it arrives as a ``CONTACT_MSG_RECV`` event and is stored here, never as an + observation, so nothing in the reception history knows about it. It is deliberately + *not* recorded as an observation instead: a routed message's SNR describes the link + to its last relay, which is exactly why :meth:`heard_nodes` excludes ``packet`` + rows, and counting it would inflate a lane that means overheard traffic. + + Outbound messages are excluded — sending to a node is not hearing from it. + + Returns: + Latest inbound-message time keyed by stored peer prefix (lowercased, as + :meth:`record_chat_message` writes it), aware UTC. The prefix width is whatever + the wire addressed, so callers match it as a prefix, not by equality. + """ + rows = self._conn.execute( + "SELECT peer, MAX(created_at) AS last FROM messages " + "WHERE is_channel = 0 AND outbound = 0 AND peer IS NOT NULL GROUP BY peer" + ).fetchall() + return {row["peer"]: datetime.fromisoformat(row["last"]) for row in rows} + def channel_stats(self) -> dict[str, ChannelStats]: """Aggregate stored channel messages into per-channel statistics. diff --git a/meshterm/services/device_state.py b/meshterm/services/device_state.py index 90ecec6..5549482 100644 --- a/meshterm/services/device_state.py +++ b/meshterm/services/device_state.py @@ -40,6 +40,7 @@ from __future__ import annotations import asyncio import time from dataclasses import replace +from datetime import datetime, timezone from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: @@ -55,6 +56,18 @@ if TYPE_CHECKING: _CONTACTS_TTL_S = 90.0 +def _aware(when: "Optional[datetime]") -> "Optional[datetime]": + """Read a stored timestamp as UTC, so two of them can be compared. + + Timestamps are stored as UTC (rendered local only at the very edge), but a row written + by an older build may come back without a tzinfo — and comparing one of those against an + aware one raises. Since the storage contract says UTC, a naive stamp simply *is* UTC. + """ + if when is None or when.tzinfo is not None: + return when + return when.replace(tzinfo=timezone.utc) + + class DeviceState: """Session-scoped cache of the stable facts screens read from the companion on open. @@ -132,7 +145,7 @@ class DeviceState: return self._contacts device = await self._ctx.device() merged = await self._remember_and_merge(await device.get_contacts()) - self._contacts = self._fill_heard(merged) + self._contacts = self._merge_heard(merged) self._contacts_at = time.monotonic() return self._contacts @@ -160,37 +173,55 @@ class DeviceState: store.remember_all(pubkey, fetched) return merge_contacts(store, pubkey, fetched) - def _fill_heard(self, contacts: list["Contact"]) -> list["Contact"]: - """Fill each contact's missing last-heard time from our own recorded receptions. + def _merge_heard(self, contacts: list["Contact"]) -> list["Contact"]: + """Give each contact the *later* of the device's advert time and our own receptions. - A contact arrives with no ``last_seen`` when the firmware never caught an advert — - or when it reported one stamped implausibly far in the future by the *sender's* - clock, which :func:`~meshterm.core.models.advert_time` refuses rather than letting - the contact read "heard now" forever. Either way our observation history may still - hold first-hand evidence (stamped by *our* clock at reception), so it fills the gap - here — once, at the one point every screen fetches contacts through — and the heard - lanes, sorts, and the purge ladder all see the same honest value. A contact we have - truly never heard stays ``None`` and reads ``never``. Best-effort: a history read - failure just returns the list unfilled, never blocking the fetch. + A contact's ``last_seen`` arrives as the firmware's ``last_advert``, which the + advertising node stamped with its own clock — hearsay + (:func:`~meshterm.core.models.advert_time` can only refuse a *future* stamp; a node + whose clock runs days behind reports a plausible-looking time that never catches + up). Our own history holds first-hand evidence instead, stamped when we received + something: overheard adverts and telemetry + (:meth:`~meshterm.persistence.repository.Repository.last_heard_by_node`) and direct + messages the node sent us + (:meth:`~meshterm.persistence.repository.Repository.last_message_by_peer`) — which + count, because in this app's lexicon "heard" means received from, and a message is + received from its sender. + + So the three are merged by taking the latest, here — once, at the one point every + screen fetches contacts through, so heard lanes, recency sorts, silence alerts and + the purge ladder all see the same honest value. Taking the *latest* rather than + preferring either side means a device time still stands whenever the firmware caught + an advert we did not record (monitoring off, app not running); we only override it + with proof of a later reception. A contact we have truly never heard stays ``None`` + and reads ``never``. Best-effort: a history read failure just returns the list + unmerged, never blocking the fetch. """ - if all(c.last_seen is not None for c in contacts): - return contacts try: - heard = { - n.node: n.last_seen for n in self._ctx.repo.heard_nodes() if n.node - } + heard = self._ctx.repo.last_heard_by_node() + messaged = self._ctx.repo.last_message_by_peer() except Exception as exc: # noqa: BLE001 - never block a contacts read on history - self._ctx.log.debug("devstate: last-heard fill skipped: %s", exc) + self._ctx.log.debug("devstate: last-heard merge skipped: %s", exc) return contacts - filled: list["Contact"] = [] + merged: list["Contact"] = [] for contact in contacts: - if contact.last_seen is None: - ident = (contact.public_key or contact.key_prefix or "").lower() - when = heard.get(ident.removeprefix("0x")[:12]) - if when is not None: - contact = replace(contact, last_seen=when) - filled.append(contact) - return filled + ident = (contact.public_key or contact.key_prefix or "").lower() + ident = ident.removeprefix("0x") + stamps = [contact.last_seen, heard.get(ident[:12])] + # A message's peer is whatever width the wire addressed, which need not match + # the contact table's — so either may be the shorter, exactly as the live chat + # matches an inbound sender to its thread. + stamps += [ + when + for peer, when in messaged.items() + if ident and peer and (ident.startswith(peer) or peer.startswith(ident)) + ] + known = [a for a in (_aware(s) for s in stamps) if a is not None] + latest = max(known) if known else None + if latest is not None and latest != contact.last_seen: + contact = replace(contact, last_seen=latest) + merged.append(contact) + return merged async def _refresh_contacts_quietly(self) -> None: """Background contacts refresh: update the cache, swallow a failure (keep the old list).""" diff --git a/meshterm/ui/node_detail_screen.py b/meshterm/ui/node_detail_screen.py index 189a61c..c430ebe 100644 --- a/meshterm/ui/node_detail_screen.py +++ b/meshterm/ui/node_detail_screen.py @@ -1030,7 +1030,13 @@ async def open_node_detail(ctx: "AppContext", contact: Optional["Contact"]) -> N else: info_rows.append(("key", Text("?", style="muted"))) if not you: - secs = _age_seconds(hn.last_seen if hn else (contact.last_seen if contact else None)) + # A contact's heard time already arrives merged with our reception history (see + # ``DeviceState._merge_heard``), so it is the later of the two and is what the + # contact list's lane shows — take it, and the two surfaces can never disagree about + # one node. Only a node that is no contact, or one whose merge could not read our + # history, falls back to the raw reception stat. + heard_at = contact.last_seen if contact else None + secs = _age_seconds(heard_at or (hn.last_seen if hn else None)) heard_val = Text(format_ago(secs), style=_recency_style(secs)) if first_heard is not None: heard_val.append( diff --git a/tests/test_device_state.py b/tests/test_device_state.py index ee2b979..0886256 100644 --- a/tests/test_device_state.py +++ b/tests/test_device_state.py @@ -56,8 +56,17 @@ class FakeDevice: return 8 -def _devstate(device: FakeDevice, heard: tuple = ()) -> DeviceState: - """A DeviceState over a fake ctx: device(), a silent logger, and a heard-node history.""" +def _devstate( + device: FakeDevice, + heard: "dict | None" = None, + messaged: "dict | None" = None, +) -> DeviceState: + """A DeviceState over a fake ctx: device(), a silent logger, and our reception history. + + ``heard`` is node-id → when we last overheard it; ``messaged`` is peer prefix → when it + last sent us a direct message. Together they are the first-hand evidence the contacts + merge weighs against the device's advert times. + """ async def device_getter(): return device @@ -65,7 +74,10 @@ def _devstate(device: FakeDevice, heard: tuple = ()) -> DeviceState: ctx = SimpleNamespace( device=device_getter, log=SimpleNamespace(debug=lambda *a, **k: None), - repo=SimpleNamespace(heard_nodes=lambda: list(heard)), + repo=SimpleNamespace( + last_heard_by_node=lambda: dict(heard or {}), + last_message_by_peer=lambda: dict(messaged or {}), + ), ) return DeviceState(ctx) # type: ignore[arg-type] @@ -123,33 +135,101 @@ def test_contacts_refresh_in_background_past_ttl() -> None: asyncio.run(run()) -def test_missing_heard_times_fill_from_recorded_receptions() -> None: - """A contact with no plausible advert time gets the time *we* last heard it, if any. +def test_heard_time_takes_the_later_of_the_device_and_our_own_receptions() -> None: + """A contact's heard time is the latest of the device's advert time and our evidence. - The device's ``last_advert`` is stamped by the sender's clock and refused upstream when - implausible (see ``models.advert_time``), so a contact can arrive with no heard time - even though our own observation history holds first-hand receptions. The cache fills - that gap once, at the fetch — and never overrides a plausible device-reported time. + The device's ``last_advert`` is stamped by the *sender's* clock, so it is hearsay: it can + be absent (refused upstream by ``models.advert_time``), or plausible-looking yet days + stale because the node's RTC runs behind. Our own history is first-hand — stamped when we + received something — so the merge takes whichever is later. A device time still wins + whenever the firmware caught an advert we never recorded. """ dev = FakeDevice() - device_says = datetime(2026, 7, 28, 9, 0, tzinfo=timezone.utc) - we_heard = datetime(2026, 7, 29, 12, 0, tzinfo=timezone.utc) + stale = datetime(2026, 7, 25, 9, 0, tzinfo=timezone.utc) + fresh = datetime(2026, 7, 29, 12, 0, tzinfo=timezone.utc) dev.contact_rows = [ Contact(name="Bogus-Clock", public_key="aa" * 32, key_prefix="aa" * 6), - Contact(name="Honest", public_key="bb" * 32, key_prefix="bb" * 6, last_seen=device_says), - Contact(name="Quiet", public_key="cc" * 32, key_prefix="cc" * 6), + Contact(name="Behind-Clock", public_key="bb" * 32, key_prefix="bb" * 6, last_seen=stale), + Contact(name="Ahead", public_key="cc" * 32, key_prefix="cc" * 6, last_seen=fresh), + Contact(name="Quiet", public_key="dd" * 32, key_prefix="dd" * 6), ] - ds = _devstate(dev, heard=(SimpleNamespace(node="aa" * 6, last_seen=we_heard),)) + ds = _devstate(dev, heard={"aa" * 6: fresh, "bb" * 6: fresh, "cc" * 6: stale}) async def run() -> None: by_name = {c.name: c for c in await ds.contacts()} - assert by_name["Bogus-Clock"].last_seen == we_heard # filled from our own history - assert by_name["Honest"].last_seen == device_says # a plausible device time stands + assert by_name["Bogus-Clock"].last_seen == fresh # filled from our own history + assert by_name["Behind-Clock"].last_seen == fresh # our proof beats a stale stamp + assert by_name["Ahead"].last_seen == fresh # a newer device time still stands assert by_name["Quiet"].last_seen is None # truly never heard stays never asyncio.run(run()) +def test_a_direct_message_counts_as_hearing_its_sender() -> None: + """An inbound DM updates the heard time — "heard" means received from. + + A direct message never touches the firmware's ``last_advert`` and is stored as a message + rather than an observation, so a node we actively chat with could read days stale (and be + swept by the purge ladder's quiet rungs) while talking to us. The peer prefix the wire + addressed need not match the contact table's width, so either may be the shorter. + """ + dev = FakeDevice() + stale = datetime(2026, 7, 25, 9, 0, tzinfo=timezone.utc) + messaged_at = datetime(2026, 7, 29, 12, 0, tzinfo=timezone.utc) + dev.contact_rows = [ + Contact(name="Chatty", public_key="ab" * 32, key_prefix="ab" * 6, last_seen=stale), + Contact(name="Silent", public_key="cd" * 32, key_prefix="cd" * 6, last_seen=stale), + ] + # A six-hex peer against a twelve-hex contact id: the shorter one is the wire's. + ds = _devstate(dev, messaged={"ababab": messaged_at}) + + async def run() -> None: + by_name = {c.name: c for c in await ds.contacts()} + assert by_name["Chatty"].last_seen == messaged_at + assert by_name["Silent"].last_seen == stale # nobody else is credited + + asyncio.run(run()) + + +def test_a_naive_stored_stamp_still_compares() -> None: + """A tz-less row from an older build reads as the UTC the storage contract says it is. + + Comparing a naive datetime against an aware one raises, which would take the whole + contacts fetch down rather than one lane. + """ + dev = FakeDevice() + stale = datetime(2026, 7, 25, 9, 0, tzinfo=timezone.utc) + naive = datetime(2026, 7, 29, 12, 0) # written before timestamps carried a zone + dev.contact_rows = [ + Contact(name="Legacy", public_key="aa" * 32, key_prefix="aa" * 6, last_seen=stale) + ] + ds = _devstate(dev, heard={"aa" * 6: naive}) + + async def run() -> None: + assert (await ds.contacts())[0].last_seen == naive.replace(tzinfo=timezone.utc) + + asyncio.run(run()) + + +def test_a_history_read_failure_leaves_the_contacts_untouched() -> None: + """The merge is best-effort: a broken history read never blocks a contacts fetch.""" + dev = FakeDevice() + device_says = datetime(2026, 7, 28, 9, 0, tzinfo=timezone.utc) + dev.contact_rows = [Contact(name="Solo", public_key="ee" * 32, key_prefix="ee" * 6, + last_seen=device_says)] + ds = _devstate(dev) + + def boom() -> dict: + raise RuntimeError("history unavailable") + + ds._ctx.repo.last_heard_by_node = boom # type: ignore[attr-defined] + + async def run() -> None: + assert (await ds.contacts())[0].last_seen == device_says + + asyncio.run(run()) + + def test_force_bypasses_the_contacts_cache() -> None: """A forced read always re-fetches, for the rare caller that needs it fresh.""" dev = FakeDevice() diff --git a/tests/test_monitor.py b/tests/test_monitor.py index 1f82a3a..35d7a57 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -20,7 +20,7 @@ from meshterm.core.connection import ( message_from_event, observation_from_event, ) -from meshterm.core.models import HeardNode, Observation, utcnow +from meshterm.core.models import ChatMessage, HeardNode, Observation, utcnow from meshterm.persistence.repository import Repository from meshterm.services.monitor_service import MonitorService @@ -298,6 +298,44 @@ def test_observation_full_key_round_trips_into_heard_nodes(tmp_path: Path) -> No repo.close() +def test_last_heard_by_node_is_the_latest_non_packet_reception(tmp_path: Path) -> None: + """The heard-time evidence: one stamp per node, packet rows excluded. + + Backs the contacts merge, which weighs it against the device's sender-stamped advert + time. ``packet`` rows are excluded exactly as ``heard_nodes`` excludes them — hearing a + relayed frame means hearing its last relay, not its originator. + """ + repo = Repository(tmp_path / "heard.db") + run_id = repo.start_run("monitor", {}) + early = utcnow() - timedelta(days=4) + late = utcnow() - timedelta(minutes=5) + repo.record_observation(run_id, Observation(node="aa" * 6, observed_at=early)) + repo.record_observation(run_id, Observation(node="aa" * 6, observed_at=late)) + repo.record_observation( + run_id, Observation(node="bb" * 6, kind="packet", path="c1", observed_at=late) + ) + heard = repo.last_heard_by_node() + assert heard == {"aa" * 6: late} # latest wins; the relayed packet credits nobody + repo.close() + + +def test_last_message_by_peer_credits_only_inbound_direct_messages(tmp_path: Path) -> None: + """Receiving a DM is hearing its sender; sending one, and channel traffic, are not.""" + repo = Repository(tmp_path / "msgs.db") + early = utcnow() - timedelta(days=4) + late = utcnow() - timedelta(minutes=5) + repo.record_chat_message(ChatMessage(text="hi", peer="AbAbAb", created_at=early)) + repo.record_chat_message(ChatMessage(text="again", peer="ababab", created_at=late)) + repo.record_chat_message( + ChatMessage(text="mine", peer="cdcdcd", outbound=True, created_at=late) + ) + repo.record_chat_message( + ChatMessage(text="all", is_channel=True, channel_id="pub", created_at=late) + ) + assert repo.last_message_by_peer() == {"ababab": late} # peers store lowercased + repo.close() + + async def test_monitor_service_records_while_hub_pumps(tmp_path: Path) -> None: """start() subscribes before the hub opens, then logs observations once it pumps.""" repo = Repository(tmp_path / "svc.db")