Persistence: a trace's link readings get a column, not a borrowed one

The readings a trace collects arrive where every other class keeps its
relay hashes, so `path` was exactly the wrong place to keep them — and
keeping them there fed the topology graph links nobody observed. They get
`observations.trace_snrs` instead, written and read like the crypto trio
and the addressing columns before them, so a replayed frame shows the same
links a live one does rather than a blank lane.

The 1606 rows already stored were migrated in place: 2454 readings
recovered, every one of them at the 1-byte width the wire uses, and every
fabricated path cleared. The pairs read as what they always were — one
trace heard twice, once a hop out (+13.25 dB) and once on the way home
(+13.25, -5.0, -4.25), the return legs weaker exactly as a longer walk
should be.
This commit is contained in:
Jean-Pierre Martineau committed 2026-08-22 14:00:48 -04:00
1 parent 6217a7dc58
commit c246fb8080
3 files changed
+68 -8

No files matched your search

+12 -2
View File
@@ -10,7 +10,7 @@ from __future__ import annotations
import sqlite3
from pathlib import Path
SCHEMA_VERSION = 13
SCHEMA_VERSION = 14
_SCHEMA = """
CREATE TABLE IF NOT EXISTS schema_meta (
@@ -129,6 +129,7 @@ CREATE TABLE IF NOT EXISTS observations (
lat REAL,
lon REAL,
path TEXT, -- 'packet' rows: comma-separated relay-hop hex hashes
-- (never a TRACE's: see trace_snrs)
observed_at TEXT NOT NULL,
chan_hash TEXT, -- overheard channel frames: channel-hash fingerprint (hex)
cipher_mac TEXT, -- …its 2-byte MAC (hex)
@@ -136,7 +137,8 @@ CREATE TABLE IF NOT EXISTS observations (
payload_typename TEXT, -- 'packet' rows: the frame's payload class (GRP_TXT/TRACE/…)
dest TEXT, -- …the recipient's key hash, for an addressed class (hex)
src TEXT, -- …the sender's key hash, or its whole key (anon request)
tag TEXT -- …the frame's own token: an ack's checksum, a trace's tag
tag TEXT, -- …the frame's own token: an ack's checksum, a trace's tag
trace_snrs TEXT -- TRACE rows: comma-separated per-hop SNR readings (dB)
);
-- One chat message, sent or received, on a channel or with a contact. Unlike the other
@@ -306,3 +308,11 @@ def _migrate(conn: sqlite3.Connection) -> None:
# carry NULL until re-heard — the raw payloads the past keys arrived in were never
# persisted, so history can't be completed from itself.
conn.execute("ALTER TABLE observations ADD COLUMN public_key TEXT")
if "trace_snrs" not in observation_cols:
# v13 -> v14: a trace's header path field is not relay hashes but one signed SNR
# byte per hop it traversed (see meshterm.core.frames.trace_link_snrs), so it gets
# a column of its own rather than sitting in `path` pretending to be adjacency.
# Live frames carried the readings in their raw payload; without somewhere to keep
# them a replayed row would show a trace's links blank, the same gap the crypto
# trio and the addressing columns above were added to close.
conn.execute("ALTER TABLE observations ADD COLUMN trace_snrs TEXT")
+18 -6
View File
@@ -67,14 +67,18 @@ def _packet_raw(row: sqlite3.Row) -> Optional[dict]:
sender, the token — restored to the key each was decoded under. The stored sender is
a hash or a whole public key, and the stored token an ack's checksum or a trace's tag;
the value's own width and the frame's class say which, so neither needed a second
column to be told apart. A row that stored none of it carries no raw (``None``).
column to be told apart. A trace's per-hop link readings are the one thing that did
need a column of its own — they arrive where every other class keeps its relay hashes,
so ``path`` was exactly the wrong place for them. A row that stored none of it carries
no raw (``None``).
"""
typename = _row_value(row, "payload_typename")
chan_hash = row["chan_hash"]
dest = _row_value(row, "dest")
src = _row_value(row, "src")
tag = _row_value(row, "tag")
if not any((typename, chan_hash, dest, src, tag)):
trace_snrs = _row_value(row, "trace_snrs")
if not any((typename, chan_hash, dest, src, tag, trace_snrs)):
return None
raw: dict = {}
if typename:
@@ -91,6 +95,8 @@ def _packet_raw(row: sqlite3.Row) -> Optional[dict]:
raw["src_key" if len(src) > 2 * ENDPOINT_HASH_BYTES else "src_hash"] = src
if tag:
raw["trace_tag" if typename == "TRACE" else "ack_crc"] = tag
if trace_snrs:
raw["trace_snrs"] = [float(v) for v in str(trace_snrs).split(",") if v]
return raw
@@ -978,11 +984,16 @@ class Repository:
dest = raw.get("dest_hash") if typename else None
src = (raw.get("src_key") or raw.get("src_hash")) if typename else None
tag = (raw.get("ack_crc") or raw.get("trace_tag")) if typename else None
# A trace's per-hop readings, which live in its header path field where every other
# class keeps relay hashes — so they get their own column rather than `path`.
readings = raw.get("trace_snrs") if typename == "TRACE" else None
trace_snrs = ",".join(f"{v:g}" for v in readings) if readings else None
self._conn.execute(
"INSERT INTO observations "
"(run_id, node, public_key, name, kind, node_type, snr, rssi, lat, lon, path, "
"observed_at, chan_hash, cipher_mac, crypted, payload_typename, dest, src, tag) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"observed_at, chan_hash, cipher_mac, crypted, payload_typename, dest, src, tag, "
"trace_snrs) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
run_id,
obs.node,
@@ -1003,6 +1014,7 @@ class Repository:
dest,
src,
tag,
trace_snrs,
),
)
self._conn.commit()
@@ -1040,7 +1052,7 @@ class Repository:
"""
rows = self._conn.execute(
"SELECT node, name, kind, node_type, snr, rssi, lat, lon, path, observed_at, "
"chan_hash, cipher_mac, crypted, payload_typename, dest, src, tag "
"chan_hash, cipher_mac, crypted, payload_typename, dest, src, tag, trace_snrs "
"FROM observations WHERE observed_at >= ? ORDER BY observed_at DESC LIMIT ?",
(since.isoformat(), limit),
).fetchall()
@@ -1066,7 +1078,7 @@ class Repository:
"""
rows = self._conn.execute(
"SELECT node, name, kind, node_type, snr, rssi, lat, lon, path, observed_at, "
"chan_hash, cipher_mac, crypted, payload_typename, dest, src, tag "
"chan_hash, cipher_mac, crypted, payload_typename, dest, src, tag, trace_snrs "
"FROM observations WHERE kind = 'packet' AND observed_at >= ? "
"AND observed_at <= ? ORDER BY observed_at ASC LIMIT ?",
(start.isoformat(), end.isoformat(), limit),
+38
View File
@@ -203,3 +203,41 @@ def test_a_frame_with_no_class_at_all_is_still_dropped() -> None:
_Event(payload_typename="UNK", pkt_payload=b"", path_len=0, path="")
) is None
assert packet_observation_from_event(_Event(snr=6.0)) is None
def test_a_traces_readings_survive_being_stored(tmp_path: Path) -> None:
"""A replayed trace reads back with the same links as the live one, not a blank lane.
The feed opens seeded from history, so readings that lived only in a live event's raw
payload would vanish the moment the screen was drawn from storage — the same gap the
crypto and addressing columns were added to close.
"""
repo = Repository(tmp_path / "trace.db")
run = repo.start_run("monitor", {}, None)
repo.record_observation(
run,
Observation(
node=None, kind="packet", path="",
raw={"payload_typename": "TRACE", "trace_tag": "102a3c5f",
"trace_snrs": [13.25, -4.5, -4.25]},
),
)
(stored,) = repo.recent_observations(since=utcnow() - timedelta(hours=2))
assert stored.path == ""
assert (stored.raw or {})["trace_snrs"] == [13.25, -4.5, -4.25]
assert (stored.raw or {})["trace_tag"] == "102a3c5f"
repo.close()
def test_only_a_trace_stores_readings(tmp_path: Path) -> None:
"""No other class has any, so a stray value on one is not written down as though it did."""
repo = Repository(tmp_path / "notrace.db")
run = repo.start_run("monitor", {}, None)
repo.record_observation(
run,
Observation(node=None, kind="packet", path="3d",
raw={"payload_typename": "TEXT_MSG", "trace_snrs": [1.0]}),
)
(stored,) = repo.recent_observations(since=utcnow() - timedelta(hours=2))
assert "trace_snrs" not in (stored.raw or {})
repo.close()