Observations: capture the full public key, keep node as the 12-hex id

Adverts carry the transmitting node's whole public key, but only its 12-hex
prefix was ever stored (observation_from_event truncated it with [:12]). That
prefix is the app's identity/join key — heard_nodes groups on it, and the map,
nodes, and watchtower screens correlate it against contacts' 12-hex key_prefix
— so it must stay short. Instead, record the whole key alongside it in a new
observations.public_key column (schema v12) and keep the prefix as node.

The Time Machine picker prefers the captured full key, then the live contact
resolver, then the bare prefix — so a heard node's key now shows as fully as
the hash lane fits, offline included. HeardNode surfaces the key from whichever
row captured it; the simulator's observations carry it too.

No backfill: the full keys past adverts arrived in were never persisted (the
raw payload wasn't stored) and no local source has them, so old rows can't be
completed. The database was wiped to start fresh under the new schema.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

meshterm/core/connection.py

@@ -2543,6 +2543,7 @@ class MockDevice(Device):
         node_type = NODE_TYPE_REPEATER if contact.name in self._MOCK_LOCATIONS else NODE_TYPE_CHAT
         return Observation(
             node=contact.key_prefix or contact.public_key[:12],
+            public_key=contact.public_key or None,
             name=contact.name,
             kind="telemetry" if seq % 4 == 3 else "advert",
             node_type=node_type,
@@ -2569,6 +2570,7 @@ class MockDevice(Device):
         contact = routed[seq % len(routed)]
         return Observation(
             node=contact.key_prefix or contact.public_key[:12],
+            public_key=contact.public_key or None,
             kind="packet",
             snr=round(self._rng.gauss(6.0, 3.0), 1),
             rssi=round(self._rng.gauss(-95.0, 8.0), 1),
@@ -2713,19 +2715,24 @@ def observation_from_event(event, kind: str) -> Optional[Observation]:  # noqa:
         The parsed :class:`Observation`, or ``None`` if the payload carried no node id.
     """
     payload = dict(getattr(event, "payload", {}) or {})
-    node = (
+    ident = (
         payload.get("public_key")
         or payload.get("pubkey")
         or payload.get("hash")
         or payload.get("key_prefix")
     )
-    if not node:
+    if not ident:
         return None
-    node = str(node).lower().removeprefix("0x")[:12]
+    ident = str(ident).lower().removeprefix("0x")
+    node = ident[:12]  # the stored 12-hex canonical id (what everything groups/joins on)
+    # Keep the whole key when the advert carried one (public_key/pubkey), so a hash lane can
+    # later show more than the twelve stored digits; a short-hash-only advert leaves it None.
+    public_key = ident if len(ident) > len(node) else None
     lat = payload.get("adv_lat", payload.get("lat"))
     lon = payload.get("adv_lon", payload.get("lon"))
     return Observation(
         node=node,
+        public_key=public_key,
         name=payload.get("adv_name") or payload.get("name"),
         kind=kind,
         node_type=_as_int(payload.get("adv_type", payload.get("type"))),
@@ -2772,11 +2779,14 @@ def packet_observation_from_event(event) -> Optional[Observation]:  # noqa: ANN0
     origin = payload.get("adv_key")
     if not origin and not hops:
         return None  # neither endpoint nor relays: no topology content
-    node = str(origin).lower().removeprefix("0x")[:12] if origin else None
+    ident = str(origin).lower().removeprefix("0x") if origin else None
+    node = ident[:12] if ident else None
+    public_key = ident if ident and len(ident) > 12 else None  # keep the whole adv_key
     lat = payload.get("adv_lat")
     lon = payload.get("adv_lon")
     return Observation(
         node=node,
+        public_key=public_key,
         name=payload.get("adv_name"),
         kind="packet",
         node_type=_as_int(payload.get("adv_type")),

meshterm/core/models.py

@@ -119,7 +119,13 @@ class Observation:
     time rather than only at the instant a command runs.
 
     Attributes:
-        node: Key prefix / hash identifying the transmitting node, if known.
+        node: Key prefix / hash identifying the transmitting node, if known — the stored
+            12-hex canonical id everything groups and joins on. Kept short on purpose so it
+            matches the key prefixes traces, messages, and the topology graph carry.
+        public_key: The node's *full* public key, when the advert carried one (it arrives
+            in the payload and is truncated to :attr:`node` for the id). Display-only —
+            lets a hash lane show more than the twelve stored digits — and ``None`` for a
+            packet class or firmware that named only a short hash.
         name: Friendly name advertised by the node, if carried.
         kind: Packet class, e.g. ``advert`` or ``telemetry``.
         node_type: Advert type of the transmitting node (see the ``NODE_TYPE_*`` constants),
@@ -141,6 +147,7 @@ class Observation:
     """
 
     node: Optional[str]
+    public_key: Optional[str] = None
     name: Optional[str] = None
     kind: str = "advert"
     node_type: Optional[int] = None
@@ -359,6 +366,8 @@ class HeardNode:
         lon: Most recent advertised longitude, if the node shared one.
         node_type: Most recent advert type seen for the node (see the ``NODE_TYPE_*``
             constants), if any observation carried it.
+        public_key: The node's full public key, when any observation captured one (see
+            :attr:`Observation.public_key`); ``None`` leaves only the short :attr:`node` id.
     """
 
     node: Optional[str]
@@ -371,6 +380,7 @@ class HeardNode:
     lat: Optional[float] = None
     lon: Optional[float] = None
     node_type: Optional[int] = None
+    public_key: Optional[str] = None
 
     @property
     def has_location(self) -> bool:
@@ -404,6 +414,7 @@ class HeardNode:
         )
         name = next((o.name for o in reversed(ordered) if o.name), None)
         node_type = next((o.node_type for o in reversed(ordered) if o.node_type is not None), None)
+        public_key = next((o.public_key for o in reversed(ordered) if o.public_key), None)
         return cls(
             node=node,
             name=name,
@@ -415,6 +426,7 @@ class HeardNode:
             lat=located.lat if located else None,
             lon=located.lon if located else None,
             node_type=node_type,
+            public_key=public_key,
         )
 
 

meshterm/persistence/db.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 import sqlite3
 from pathlib import Path
 
-SCHEMA_VERSION = 11
+SCHEMA_VERSION = 12
 
 _SCHEMA = """
 CREATE TABLE IF NOT EXISTS schema_meta (
@@ -119,7 +119,8 @@ CREATE TABLE IF NOT EXISTS discovered_paths (
 CREATE TABLE IF NOT EXISTS observations (
     id          INTEGER PRIMARY KEY AUTOINCREMENT,
     run_id      INTEGER NOT NULL REFERENCES runs(id) ON DELETE CASCADE,
-    node        TEXT,
+    node        TEXT,             -- 12-hex canonical id (grouped/joined on)
+    public_key  TEXT,             -- the node's full public key, when the advert carried one
     name        TEXT,
     kind        TEXT    NOT NULL DEFAULT 'advert',
     node_type   INTEGER,          -- advert type (repeater/chat/room/...), when carried
@@ -230,3 +231,11 @@ def _migrate(conn: sqlite3.Connection) -> None:
         # origin node — so the dashboard feed can read "channel text" / "trace" instead of
         # a bare "?" even when seeded from history. Older rows carry NULL (class unknown).
         conn.execute("ALTER TABLE observations ADD COLUMN payload_typename TEXT")
+    if "public_key" not in observation_cols:
+        # v11 -> v12: adverts carry the transmitting node's full public key, but only its
+        # 12-hex prefix was ever stored as the node id. Keep that prefix as the canonical id
+        # (traces, contacts, and the topology graph all match on it) and record the whole key
+        # alongside it, so a hash lane can show more than the twelve stored digits. Older rows
+        # carry NULL until re-heard — the raw payloads the past keys arrived in were never
+        # persisted, so history can't be completed from itself.
+        conn.execute("ALTER TABLE observations ADD COLUMN public_key TEXT")

meshterm/persistence/repository.py

@@ -907,12 +907,13 @@ class Repository:
             crypted = raw.get("crypted")
         self._conn.execute(
             "INSERT INTO observations "
-            "(run_id, node, name, kind, node_type, snr, rssi, lat, lon, path, observed_at, "
-            "chan_hash, cipher_mac, crypted, payload_typename) "
-            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+            "(run_id, node, public_key, name, kind, node_type, snr, rssi, lat, lon, path, "
+            "observed_at, chan_hash, cipher_mac, crypted, payload_typename) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
             (
                 run_id,
                 obs.node,
+                obs.public_key,
                 obs.name,
                 obs.kind,
                 obs.node_type,
@@ -1431,7 +1432,7 @@ class Repository:
             One :class:`HeardNode` per distinct node, ordered by most-recently heard.
         """
         sql = (
-            "SELECT node, name, node_type, snr, rssi, lat, lon, observed_at "
+            "SELECT node, public_key, name, node_type, snr, rssi, lat, lon, observed_at "
             "FROM observations WHERE kind != 'packet'"
         )
         params: list[Any] = []
@@ -1444,6 +1445,7 @@ class Repository:
         for row in rows:
             obs = Observation(
                 node=row["node"],
+                public_key=row["public_key"],
                 name=row["name"],
                 node_type=row["node_type"],
                 snr=row["snr"],

meshterm/ui/timemachine_screen.py

@@ -1134,10 +1134,9 @@ def _picker_row(
             stored :attr:`~meshterm.core.models.HeardNode.node_type` is absent, so a node
             heard only via non-advert packets still shows its true kind (see
             :func:`~meshterm.services.trace_runner.make_node_type_resolver`).
-        key: The hex to render in the hash lane — the node's full public key when the device
-            holds it as a contact (see
-            :func:`~meshterm.services.trace_runner.make_key_resolver`), so more than the
-            stored 12-hex prefix shows; ``None`` falls back to the stored prefix.
+        key: The hex to render in the hash lane — the node's full public key when we have
+            one (captured with the observation, or resolved from the device's contacts), so
+            more than the stored 12-hex prefix shows; ``None`` falls back to the stored prefix.
     """
     kind = node.node_type if node.node_type is not None else node_type
     glyph, glyph_style = _NODE_GLYPHS.get(kind, _DEFAULT_GLYPH)
@@ -1355,7 +1354,9 @@ class TimeMachinePickerScreen(SelectScreen):
         ]
         for node, name in _ordered_heard(self._listed, self._sort):
             node_type = node.node_type if node.node_type is not None else self._type_of(node.node)
-            key = self._resolve_key(node.node) or node.node
+            # The full key, most-durable source first: the one captured with the observation
+            # (shows offline), else the device's live contact list, else the stored prefix.
+            key = node.public_key or self._resolve_key(node.node) or node.node
             items.append(
                 Choice(
                     _picker_row(

tests/test_monitor.py

@@ -160,6 +160,29 @@ def test_observation_from_event_parses_advert() -> None:
     assert (obs.lat, obs.lon) == (45.5, -73.6)
 
 
+def test_observation_from_event_keeps_full_public_key() -> None:
+    """An advert carrying the whole key keeps it (public_key) while node stays the 12-hex id."""
+    full = "aabbccddee11223344556677" + "00" * 20  # 64 hex
+
+    class _Event:
+        payload = {"public_key": full.upper(), "adv_name": "Repeater", "snr": 7.5}
+
+    obs = observation_from_event(_Event(), "advert")
+    assert obs is not None
+    assert obs.node == full[:12]  # the stored 12-hex canonical id everything joins on
+    assert obs.public_key == full  # the whole key, kept for a fuller hash display
+
+
+def test_observation_from_event_short_hash_leaves_no_full_key() -> None:
+    """A short-hash-only advert stores just the id, no (fabricated) full key."""
+
+    class _Event:
+        payload = {"hash": "aabbcc0011", "adv_name": "Rep"}
+
+    obs = observation_from_event(_Event(), "advert")
+    assert obs is not None and obs.node == "aabbcc0011" and obs.public_key is None
+
+
 def test_observation_from_event_without_node_is_skipped() -> None:
     """A payload carrying no node identifier yields no observation."""
 
@@ -260,6 +283,21 @@ def test_observation_count_totals_every_run(tmp_path: Path) -> None:
     repo.close()
 
 
+def test_observation_full_key_round_trips_into_heard_nodes(tmp_path: Path) -> None:
+    """A captured full public key persists and surfaces on the aggregated HeardNode."""
+    full = "3d63c6429436" + "ab" * 26  # 64 hex; node is its first 12
+    repo = Repository(tmp_path / "keys.db")
+    run_id = repo.start_run("monitor", {})
+    # One advert with the whole key, a later one for the same node carrying only the prefix.
+    repo.record_observation(run_id, Observation(node=full[:12], public_key=full, snr=5.0))
+    repo.record_observation(run_id, Observation(node=full[:12], snr=6.0))
+    (node,) = repo.heard_nodes()
+    assert node.node == full[:12]  # the id stays the 12-hex prefix
+    assert node.public_key == full  # the full key is surfaced from the row that had it
+    assert node.count == 2
+    repo.close()
+
+
 async def test_monitor_service_records_while_hub_pumps(tmp_path: Path) -> None:
     """start() subscribes before the hub opens, then logs observations once it pumps."""
     repo = Repository(tmp_path / "svc.db")

tests/test_timemachine.py

@@ -962,3 +962,19 @@ def test_picker_hash_lane_shows_a_heard_nodes_full_key_when_a_contact_holds_it()
     assert stored + "ab" * 10 in row.label.plain  # 32 hex shown — far more than the stored 12
     # Its value still carries the stored 12-hex id, so the page query it opens is unchanged.
     assert row.value == (stored, "Rep")
+
+
+def test_picker_hash_lane_shows_a_captured_full_key_without_a_device() -> None:
+    """A full key captured with the observation shows offline — no contact resolver needed."""
+    from meshterm.core.models import HeardNode
+
+    stored = "3d63c6429436"
+    full = stored + "cd" * 26
+    node = HeardNode(
+        node=stored, name="Rep", count=7, median_snr=None, best_snr=None,
+        last_rssi=None, last_seen=utcnow(), public_key=full,
+    )
+    # No resolve_key passed: the stored public_key alone drives the hash lane.
+    screen = _picker([(node, "Rep")], width=100)
+    row = next(c for c in screen._choices() if c.value == (stored, "Rep"))
+    assert stored + "cd" * 10 in row.label.plain