MeshTerm/meshterm/persistence/repository.py

"""Repository: the single gateway between the app and the database.

Tools and services never issue SQL directly; they call typed methods here. This keeps
persistence concerns in one place and makes the storage backend swappable.
"""

from __future__ import annotations

import json
import sqlite3
from dataclasses import dataclass
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, Optional

from ..core.models import (
    PATH_TRACE_TARGET,
    ChatMessage,
    HeardNode,
    Hop,
    NeighbourInfo,
    Observation,
    TraceResult,
    TxLevelResult,
    utcnow,
)
from . import db


#: The trailing window a channel's recent-activity view is computed over. Two hours is
#: "what's happening right now" territory: the activity sparkline reads as a live pulse of
#: the mesh rather than a long-term traffic log (the MSGS lane covers all-time volume).
ACTIVITY_WINDOW = timedelta(hours=2)

#: How many equal time buckets the activity window is split into — one per column of the
#: channel manager's braille sparkline (two columns per braille cell, so 24 buckets fill
#: its twelve characters). At a two-hour window each bucket spans five minutes.
ACTIVITY_BUCKETS = 24


def _packet_raw(row: sqlite3.Row) -> Optional[dict]:
    """Rebuild the minimal raw payload a stored ``packet`` observation is read back with.

    Two things a live RX-log event carried in its raw payload are kept per row: the
    frame's payload class (``payload_typename``), so a list can name what the packet is,
    and — for an overheard ``GRP_TXT`` frame — the three crypto fields it decrypts from
    (fingerprint, MAC, ciphertext), so a channel we hold the key for stays readable
    straight from history via the same code path a fresh frame takes. A row that stored
    neither carries no raw (``None``), exactly as before.
    """
    typename = _row_value(row, "payload_typename")
    chan_hash = row["chan_hash"]
    if not typename and not chan_hash:
        return None
    raw: dict = {}
    if typename:
        raw["payload_typename"] = typename
    if chan_hash:
        raw.setdefault("payload_typename", "GRP_TXT")
        raw["chan_hash"] = chan_hash
        raw["cipher_mac"] = row["cipher_mac"]
        raw["crypted"] = row["crypted"]
    return raw


def _row_value(row: sqlite3.Row, key: str) -> Any:
    """Read ``key`` from a row, tolerating a query that didn't select it (returns ``None``)."""
    try:
        return row[key]
    except (IndexError, KeyError):
        return None


def _hops_hash_bytes(hops: list[Hop]) -> Optional[int]:
    """Recover a stored trace's per-hop path-hash width from its hop hashes.

    The ``traces`` table predates :attr:`~meshterm.core.models.TraceResult.
    path_hash_bytes`, so the width isn't a column — but it doesn't need to be: a trace
    reply names every hop at exactly the command's width, so the (uniform) length of
    the stored hashes *is* the width. Without it, rehydrated traces would render node
    hashes — and our own device's full public key — at absurd lengths.

    Args:
        hops: The rehydrated hops (the final hash-less hop, our own device, is skipped).

    Returns:
        The width in bytes, or ``None`` when there are no hashes or they disagree
        (which stored trace replies never do; ``None`` falls back to full-width display).
    """
    widths = {len(h.node) for h in hops if h.node}
    if len(widths) != 1:
        return None
    width = widths.pop()
    return width // 2 if width and width % 2 == 0 else None


@dataclass(slots=True)
class ChannelStats:
    """Aggregated message history for one channel conversation.

    Attributes:
        total: Messages ever stored for the channel, sent and received alike.
        recent: Messages within the trailing :data:`ACTIVITY_WINDOW` window.
        last_at: When the channel's most recent message was stored, or ``None`` if the
            stored timestamp can't be parsed.
        histogram: The window's messages split into :data:`ACTIVITY_BUCKETS` equal time
            buckets, *newest first* — bucket 0 is the current five minutes, the order
            :func:`~meshterm.ui.braillechart.activity_sparkline` expects (it flips the
            window so "now" draws at the right edge). ``recent`` is always its sum.
    """

    total: int
    recent: int
    last_at: Optional[datetime]
    histogram: tuple[int, ...]


@dataclass(slots=True)
class TracedPath:
    """One successful trace's walked path, as evidence for the topology graph.

    Attributes:
        when: When the trace completed.
        hops: The per-hop readings in path order, each ``(node, snr)`` — ``node`` is the
            hop's raw hex hash (``None`` for the final hash-less hop, our own device) and
            ``snr`` the reception measured *arriving at* that hop. Because trace replies
            retrace the path, the sequence covers the outbound and return legs alike.
    """

    when: datetime
    hops: list[tuple[Optional[str], float]]


@dataclass(slots=True)
class PacketPath:
    """One RX-logged packet's relay path, as evidence for the topology graph.

    Attributes:
        when: When the packet was overheard.
        origin: The originating node's hex hash, when the packet class reveals it
            (adverts do); ``None`` otherwise.
        snr: Our reception SNR (dB) — a reading on the link from the *last relay*
            (or, with no relays, the origin) to us.
        hops: The relay hashes in propagation order, nearest the origin first and the
            repeater we actually heard last; empty for a direct (zero-hop) packet.
    """

    when: datetime
    origin: Optional[str]
    snr: Optional[float]
    hops: list[str]


@dataclass(slots=True)
class NeighbourLink:
    """One repeater-reported direct link, as evidence for the topology graph.

    The fetched counterpart of :class:`TracedPath`/:class:`PacketPath`: instead of being
    inferred from what *we* received, this link was asserted by a remote repeater about
    its own reception (see :meth:`Repository.neighbour_links`).

    Attributes:
        when: The link's recency — when the repeater last heard the neighbour, falling
            back to when we fetched the table.
        repeater: Canonical id of the repeater that reported the link.
        neighbour: The neighbour's hex hash as the repeater replied it (any width; the
            topology layer canonicalizes).
        snr: SNR (dB) measured at the repeater, if reported.
    """

    when: datetime
    repeater: str
    neighbour: str
    snr: Optional[float]


@dataclass(slots=True)
class RunRecord:
    """A summary row from the ``runs`` table.

    Attributes:
        id: Primary key of the run.
        tool: Name of the tool that executed.
        profile: Device profile used, if any.
        status: ``running``, ``ok``, or ``error``.
        started_at: ISO-8601 start timestamp.
        finished_at: ISO-8601 finish timestamp, or ``None`` if still running.
        summary: Decoded JSON summary, or ``None``.
    """

    id: int
    tool: str
    profile: Optional[str]
    status: str
    started_at: str
    finished_at: Optional[str]
    summary: Optional[dict]


class Repository:
    """Typed data-access layer over the SQLite database."""

    def __init__(self, db_path: Path) -> None:
        """Open the repository against a database file.

        Args:
            db_path: Location of the SQLite database (created if absent).
        """
        self._conn = db.connect(db_path)

    def close(self) -> None:
        """Close the underlying database connection."""
        self._conn.close()

    # -- runs -------------------------------------------------------------------

    def start_run(
        self, tool: str, args: dict[str, Any], profile: Optional[str] = None
    ) -> int:
        """Record the start of a tool execution.

        Args:
            tool: Tool name.
            args: Arguments the tool was invoked with (must be JSON-serializable).
            profile: Active device profile name, if any.

        Returns:
            The new run's primary key.
        """
        cur = self._conn.execute(
            "INSERT INTO runs (tool, profile, args_json, status, started_at) "
            "VALUES (?, ?, ?, 'running', ?)",
            (tool, profile, json.dumps(args, default=str), utcnow().isoformat()),
        )
        self._conn.commit()
        return int(cur.lastrowid)

    def finish_run(self, run_id: int, status: str, summary: Optional[dict] = None) -> None:
        """Mark a run as finished.

        Args:
            run_id: The run to update.
            status: Terminal status (``ok`` or ``error``).
            summary: Optional JSON-serializable result summary.
        """
        self._conn.execute(
            "UPDATE runs SET status = ?, summary_json = ?, finished_at = ? WHERE id = ?",
            (
                status,
                json.dumps(summary, default=str) if summary is not None else None,
                utcnow().isoformat(),
                run_id,
            ),
        )
        self._conn.commit()

    def list_runs(self, limit: int = 50) -> list[RunRecord]:
        """Return the most recent runs, newest first.

        Args:
            limit: Maximum number of rows to return.

        Returns:
            A list of :class:`RunRecord`.
        """
        rows = self._conn.execute(
            "SELECT * FROM runs ORDER BY id DESC LIMIT ?", (limit,)
        ).fetchall()
        return [self._row_to_run(r) for r in rows]

    @staticmethod
    def _row_to_run(row: sqlite3.Row) -> RunRecord:
        """Map a ``runs`` row to a :class:`RunRecord`."""
        return RunRecord(
            id=row["id"],
            tool=row["tool"],
            profile=row["profile"],
            status=row["status"],
            started_at=row["started_at"],
            finished_at=row["finished_at"],
            summary=json.loads(row["summary_json"]) if row["summary_json"] else None,
        )

    # -- traces -----------------------------------------------------------------

    def record_trace(self, run_id: int, trace: TraceResult) -> int:
        """Persist a single trace and its per-hop SNR readings.

        Args:
            run_id: The owning run.
            trace: The trace result to store.

        Returns:
            The new trace's primary key.
        """
        cur = self._conn.execute(
            "INSERT INTO traces "
            "(run_id, target, success, hop_count, min_snr, round_trip_ms, tx_power, created_at) "
            "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
            (
                run_id,
                trace.target,
                int(trace.success),
                trace.hop_count,
                trace.min_snr,
                trace.round_trip_ms,
                trace.tx_power,
                trace.timestamp.isoformat(),
            ),
        )
        trace_id = int(cur.lastrowid)
        self._conn.executemany(
            "INSERT INTO trace_hops (trace_id, hop_index, node, snr) VALUES (?, ?, ?, ?)",
            [(trace_id, h.index, h.node, h.snr) for h in trace.hops],
        )
        self._conn.commit()
        return trace_id

    def latest_trace(
        self,
        target: str,
        *,
        exclude_run_id: Optional[int] = None,
        success_only: bool = True,
    ) -> Optional[TraceResult]:
        """Return the most recently recorded trace to ``target``, rehydrated with hops.

        Used to show the previous run's result before a new trace starts.

        Args:
            target: The trace destination to look up.
            exclude_run_id: A run to skip (typically the in-progress one) so we surface
                a genuinely prior result.
            success_only: When ``True``, ignore traces that timed out.

        Returns:
            The latest matching :class:`TraceResult`, or ``None`` if none is stored.
        """
        sql = "SELECT * FROM traces WHERE target = ?"
        params: list[Any] = [target]
        if success_only:
            sql += " AND success = 1"
        if exclude_run_id is not None:
            sql += " AND run_id != ?"
            params.append(exclude_run_id)
        sql += " ORDER BY id DESC LIMIT 1"
        row = self._conn.execute(sql, params).fetchone()
        if row is None:
            return None

        hop_rows = self._conn.execute(
            "SELECT hop_index, node, snr FROM trace_hops WHERE trace_id = ? "
            "ORDER BY hop_index",
            (row["id"],),
        ).fetchall()
        hops = [Hop(index=h["hop_index"], node=h["node"], snr=h["snr"]) for h in hop_rows]
        return TraceResult(
            target=row["target"],
            success=bool(row["success"]),
            hops=hops,
            round_trip_ms=row["round_trip_ms"],
            tx_power=row["tx_power"],
            path_hash_bytes=_hops_hash_bytes(hops),
            timestamp=datetime.fromisoformat(row["created_at"]),
        )

    def recent_traces(self, target: str, *, limit: int = 200) -> list[TraceResult]:
        """Return recent traces to ``target``, newest first, rehydrated with hops.

        This is the history the link-quality baseline is computed over, so both timed-out
        and successful traces are included (a rise in failures is itself a regression).

        Args:
            target: The trace destination to load.
            limit: Maximum number of traces to return.

        Returns:
            The matching :class:`TraceResult` objects, newest first.
        """
        rows = self._conn.execute(
            "SELECT * FROM traces WHERE target = ? ORDER BY id DESC LIMIT ?",
            (target, limit),
        ).fetchall()
        return [self._hydrate_trace(row) for row in rows]

    def _hydrate_trace(self, row: sqlite3.Row) -> TraceResult:
        """Rebuild a :class:`TraceResult` (with hops) from a ``traces`` row.

        Args:
            row: A ``traces`` table row.

        Returns:
            The rehydrated trace.
        """
        hop_rows = self._conn.execute(
            "SELECT hop_index, node, snr FROM trace_hops WHERE trace_id = ? ORDER BY hop_index",
            (row["id"],),
        ).fetchall()
        hops = [Hop(index=h["hop_index"], node=h["node"], snr=h["snr"]) for h in hop_rows]
        return TraceResult(
            target=row["target"],
            success=bool(row["success"]),
            hops=hops,
            round_trip_ms=row["round_trip_ms"],
            tx_power=row["tx_power"],
            path_hash_bytes=_hops_hash_bytes(hops),
            timestamp=datetime.fromisoformat(row["created_at"]),
        )

    def traced_targets(self, *, limit: int = 5) -> list[str]:
        """Return recent trace destinations, most recently traced first.

        This feeds the target picker's *Recently traced* section, so the list is
        deliberately short and recency-ordered: an all-time tally only ever grows,
        burying current work under stale names (``--mock`` targets included).
        Target-less path walks (recorded under :data:`~meshterm.core.models.
        PATH_TRACE_TARGET`) are excluded — they aren't destinations one can pick.

        Args:
            limit: Maximum number of distinct targets to return.

        Returns:
            Distinct target names, most recently traced first.
        """
        rows = self._conn.execute(
            "SELECT target, MAX(id) AS latest FROM traces WHERE target != ? "
            "GROUP BY target ORDER BY latest DESC LIMIT ?",
            (PATH_TRACE_TARGET, limit),
        ).fetchall()
        return [row["target"] for row in rows]

    def trace_paths(self, *, limit: int = 2000) -> list[TracedPath]:
        """Return the walked paths of recent successful traces, newest first.

        This is the trace side of the topology evidence: every successful trace is a
        packet that demonstrably crossed each link in its path (out and back), with an
        SNR reading at every hop. Hops are returned raw — hex hashes at whatever width
        the original command addressed them — for the topology layer to canonicalize.

        Args:
            limit: Maximum number of traces to load.

        Returns:
            One :class:`TracedPath` per successful trace that recorded hops.
        """
        rows = self._conn.execute(
            "SELECT t.id, t.created_at FROM traces t "
            "WHERE t.success = 1 ORDER BY t.id DESC LIMIT ?",
            (limit,),
        ).fetchall()
        paths: list[TracedPath] = []
        for row in rows:
            hop_rows = self._conn.execute(
                "SELECT node, snr FROM trace_hops WHERE trace_id = ? ORDER BY hop_index",
                (row["id"],),
            ).fetchall()
            if not hop_rows:
                continue
            try:
                when = datetime.fromisoformat(row["created_at"])
            except (TypeError, ValueError):
                continue  # a malformed stray contributes no evidence
            paths.append(
                TracedPath(when=when, hops=[(h["node"], h["snr"]) for h in hop_rows])
            )
        return paths

    def packet_paths(self, *, limit: int = 5000) -> list[PacketPath]:
        """Return the relay paths of recent RX-logged packets, newest first.

        The passive side of the topology evidence: each row is a packet the companion
        overheard whose header carried the repeater path it had traversed so far. Only
        ``packet``-kind observations carry one (see :meth:`record_observation`); rows
        whose path is NULL are skipped, and an empty path (a direct packet) is returned
        with no hops so a known origin still yields a direct-link reading.

        Args:
            limit: Maximum number of packet observations to load.

        Returns:
            One :class:`PacketPath` per stored packet observation.
        """
        rows = self._conn.execute(
            "SELECT node, snr, path, observed_at FROM observations "
            "WHERE kind = 'packet' AND path IS NOT NULL ORDER BY id DESC LIMIT ?",
            (limit,),
        ).fetchall()
        paths: list[PacketPath] = []
        for row in rows:
            try:
                when = datetime.fromisoformat(row["observed_at"])
            except (TypeError, ValueError):
                continue  # a malformed stray contributes no evidence
            hops = [h for h in (row["path"] or "").split(",") if h]
            paths.append(
                PacketPath(when=when, origin=row["node"], snr=row["snr"], hops=hops)
            )
        return paths

    def record_neighbours(
        self, run_id: int, repeater: str, neighbours: list[NeighbourInfo]
    ) -> None:
        """Persist one fetched neighbour-table snapshot from a remote repeater.

        Every entry becomes a row; refetching the same repeater later appends a fresh
        snapshot rather than overwriting, and :meth:`neighbour_links` reads back only
        the latest row per ``(repeater, neighbour)`` pair — a neighbour table is the
        repeater's *current* state, so a new snapshot supersedes the old one instead
        of stacking as extra evidence.

        Args:
            run_id: The owning run.
            repeater: Canonical id of the repeater the table came from.
            neighbours: The fetched entries (may be empty — recorded as no rows).
        """
        fetched_at = utcnow().isoformat()
        self._conn.executemany(
            "INSERT INTO neighbour_reports "
            "(run_id, repeater, neighbour, snr, heard_at, fetched_at) "
            "VALUES (?, ?, ?, ?, ?, ?)",
            [
                (
                    run_id,
                    repeater,
                    n.node,
                    n.snr,
                    n.heard_at.isoformat() if n.heard_at is not None else None,
                    fetched_at,
                )
                for n in neighbours
            ],
        )
        self._conn.commit()

    def neighbour_links(self, *, limit: int = 2000) -> list[NeighbourLink]:
        """Return the current repeater-reported links, newest report first.

        The fetched side of the topology evidence. Only the most recent row per
        ``(repeater, neighbour)`` pair is returned (see :meth:`record_neighbours`), so
        repeatedly refreshing a table never inflates a link's sample count.

        Args:
            limit: Maximum number of links to load.

        Returns:
            One :class:`NeighbourLink` per currently-reported link.
        """
        rows = self._conn.execute(
            "SELECT r.repeater, r.neighbour, r.snr, r.heard_at, r.fetched_at "
            "FROM neighbour_reports r "
            "JOIN (SELECT MAX(id) AS id FROM neighbour_reports "
            "      GROUP BY repeater, neighbour) latest ON latest.id = r.id "
            "ORDER BY r.id DESC LIMIT ?",
            (limit,),
        ).fetchall()
        links: list[NeighbourLink] = []
        for row in rows:
            when: Optional[datetime] = None
            for stamp in (row["heard_at"], row["fetched_at"]):
                try:
                    when = datetime.fromisoformat(stamp)
                    break
                except (TypeError, ValueError):
                    continue
            if when is None:
                continue  # a malformed stray contributes no evidence
            links.append(
                NeighbourLink(
                    when=when,
                    repeater=row["repeater"],
                    neighbour=row["neighbour"],
                    snr=row["snr"],
                )
            )
        return links

    def record_tx_sample(self, run_id: int, level: TxLevelResult) -> None:
        """Persist one robust TX-power level from an optimization sweep.

        The ``median_min_snr`` column stores this optimizer's headline metric — the
        median SNR measured *at the target* — rather than a path bottleneck.

        Args:
            run_id: The owning run.
            level: Aggregated result for a single TX power level.
        """
        self._conn.execute(
            "INSERT INTO tx_samples "
            "(run_id, target, tx_power, median_min_snr, success_rate, samples, created_at) "
            "VALUES (?, ?, ?, ?, ?, ?, ?)",
            (
                run_id,
                level.stats.target,
                level.tx_power,
                level.target_snr,
                level.success_rate,
                level.samples,
                utcnow().isoformat(),
            ),
        )
        self._conn.commit()

    def record_path_candidate(
        self,
        run_id: int,
        target: str,
        path: str,
        *,
        bottleneck_snr: Optional[float],
        success_rate: float,
        median_rtt_ms: Optional[float],
    ) -> None:
        """Persist one measured candidate path from a path-probe sweep.

        Args:
            run_id: The owning probe run.
            target: The node the candidate paths lead to.
            path: The forced outbound path measured (comma-separated hex hashes).
            bottleneck_snr: Median of the candidate's per-trace bottleneck SNRs (dB),
                or ``None`` when no trace over it succeeded.
            success_rate: Fraction of traces over this path that replied, ``[0, 1]``.
            median_rtt_ms: Median round-trip time over this path, if measured.
        """
        self._conn.execute(
            "INSERT INTO path_candidates "
            "(run_id, target, path_json, bottleneck_snr, success_rate, median_rtt_ms, "
            "created_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
            (
                run_id,
                target,
                json.dumps(path.split(",")),
                bottleneck_snr,
                success_rate,
                median_rtt_ms,
                utcnow().isoformat(),
            ),
        )
        self._conn.commit()

    # -- observations (passive monitoring) --------------------------------------

    def record_observation(self, run_id: int, obs: Observation) -> None:
        """Persist one overheard packet from a monitoring run.

        ``packet``-kind observations (the companion's RX packet log) also carry the relay
        path the packet traversed — the raw material of the topology graph (see
        :meth:`packet_paths`).

        An overheard channel-text (``GRP_TXT``) frame also keeps the three fields the
        packet viewer decrypts from — the channel-hash fingerprint, the 2-byte MAC, and
        the ciphertext — lifted out of the raw payload so a channel we hold the key for
        is still readable when the feed is later seeded from stored history.

        Args:
            run_id: The owning run.
            obs: The observation to store.
        """
        chan_hash = cipher_mac = crypted = None
        raw = obs.raw if isinstance(obs.raw, dict) else {}
        # The payload class identifies a relayed 'packet' that names no origin node — kept
        # for every packet, not only channel text; the crypto trio is kept for GRP_TXT alone.
        typename = raw.get("payload_typename") if obs.kind == "packet" else None
        if raw.get("payload_typename") == "GRP_TXT":
            chan_hash = raw.get("chan_hash")
            cipher_mac = raw.get("cipher_mac")
            crypted = raw.get("crypted")
        self._conn.execute(
            "INSERT INTO observations "
            "(run_id, node, name, kind, node_type, snr, rssi, lat, lon, path, observed_at, "
            "chan_hash, cipher_mac, crypted, payload_typename) "
            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
            (
                run_id,
                obs.node,
                obs.name,
                obs.kind,
                obs.node_type,
                obs.snr,
                obs.rssi,
                obs.lat,
                obs.lon,
                obs.path,
                obs.observed_at.isoformat(),
                chan_hash,
                cipher_mac,
                crypted,
                typename,
            ),
        )
        self._conn.commit()

    def observation_count(self) -> int:
        """Return the total number of overheard packets stored across every run.

        Backs the monitor's "total" counter, so it counts all observations ever logged,
        not just the current session's.

        Returns:
            The row count of the ``observations`` table.
        """
        row = self._conn.execute("SELECT COUNT(*) AS n FROM observations").fetchone()
        return int(row["n"]) if row else 0

    def recent_observations(
        self, *, since: datetime, limit: int = 4000
    ) -> list[Observation]:
        """Return raw stored observations in a recent window, oldest first.

        The dashboard's seed: everything overheard in the window — adverts, telemetry,
        and ``packet`` RX-log rows alike — hydrated back into
        :class:`~meshterm.core.models.Observation` values so the live screen can treat
        stored history and fresh hub events identically. Timestamps compare as strings
        (every ``observed_at`` is written by ``datetime.isoformat`` in UTC), the same
        trick :meth:`channel_stats` relies on.

        Args:
            since: Only observations at or after this time.
            limit: Hard cap on rows (newest kept), so a very busy window stays cheap.

        Returns:
            The window's observations, oldest first (ready to append live events to).
        """
        rows = self._conn.execute(
            "SELECT node, name, kind, node_type, snr, rssi, lat, lon, path, observed_at, "
            "chan_hash, cipher_mac, crypted, payload_typename "
            "FROM observations WHERE observed_at >= ? ORDER BY observed_at DESC LIMIT ?",
            (since.isoformat(), limit),
        ).fetchall()
        return self._hydrate_observations(reversed(rows))

    def packet_frames_between(
        self, start: datetime, end: datetime, *, limit: int = 2000
    ) -> list[Observation]:
        """RX-logged ``packet`` frames inside a closed time window, oldest first.

        The message-paths view's feed: every relayed frame the radio reported in the
        window around a chat message, raw crypto fields included, so a channel frame
        can be decrypted and matched to the message and a direct frame correlated by
        time. Timestamps compare as ISO strings, like every observation query.

        Args:
            start: The window's inclusive start.
            end: The window's inclusive end.
            limit: Hard cap on rows (oldest kept — the window centres on the message).

        Returns:
            The window's ``packet`` observations, oldest first.
        """
        rows = self._conn.execute(
            "SELECT node, name, kind, node_type, snr, rssi, lat, lon, path, observed_at, "
            "chan_hash, cipher_mac, crypted, payload_typename "
            "FROM observations WHERE kind = 'packet' AND observed_at >= ? "
            "AND observed_at <= ? ORDER BY observed_at ASC LIMIT ?",
            (start.isoformat(), end.isoformat(), limit),
        ).fetchall()
        return self._hydrate_observations(rows)

    def _hydrate_observations(self, rows) -> list[Observation]:  # noqa: ANN001 - sqlite rows
        """Rebuild :class:`Observation` values from full observation rows, in order."""
        observations: list[Observation] = []
        for row in rows:
            try:
                observed_at = datetime.fromisoformat(row["observed_at"])
            except (TypeError, ValueError):
                continue  # a malformed stray simply doesn't make the window
            observations.append(
                Observation(
                    node=row["node"],
                    name=row["name"],
                    kind=row["kind"] or "advert",
                    node_type=row["node_type"],
                    snr=row["snr"],
                    rssi=row["rssi"],
                    lat=row["lat"],
                    lon=row["lon"],
                    path=row["path"],
                    observed_at=observed_at,
                    raw=_packet_raw(row),
                )
            )
        return observations

    def node_observations(
        self, node: str, *, since: Optional[datetime] = None, limit: int = 50000
    ) -> list[Observation]:
        """Every stored reception of one node, oldest first — its longitudinal record.

        The Time Machine's per-node feed. ``packet``-kind rows are excluded for the
        same reason :meth:`heard_nodes` drops them: their SNR describes the last relay,
        not the node itself, so they would poison a reception timeline.

        Args:
            node: The node's stored id (the 12-hex key prefix observations carry).
            since: Only observations at or after this time, if given.
            limit: Hard cap on rows (newest kept) so an ancient, chatty node stays cheap.

        Returns:
            The node's observations, oldest first.
        """
        sql = (
            "SELECT node, name, kind, node_type, snr, rssi, lat, lon, path, observed_at "
            "FROM observations WHERE node = ? AND kind != 'packet'"
        )
        params: list[Any] = [node]
        if since is not None:
            sql += " AND observed_at >= ?"
            params.append(since.isoformat())
        sql += " ORDER BY observed_at DESC LIMIT ?"
        params.append(limit)
        rows = self._conn.execute(sql, params).fetchall()
        observations: list[Observation] = []
        for row in reversed(rows):
            try:
                observed_at = datetime.fromisoformat(row["observed_at"])
            except (TypeError, ValueError):
                continue  # a malformed stray simply doesn't make the record
            observations.append(
                Observation(
                    node=row["node"],
                    name=row["name"],
                    kind=row["kind"] or "advert",
                    node_type=row["node_type"],
                    snr=row["snr"],
                    rssi=row["rssi"],
                    lat=row["lat"],
                    lon=row["lon"],
                    path=row["path"],
                    observed_at=observed_at,
                )
            )
        return observations

    def daily_activity(self) -> list[tuple[str, int, int]]:
        """Per-day activity totals across the whole stored history, oldest first.

        Days are **local** calendar days: ``observed_at`` is stored as UTC ISO-8601,
        but SQLite's ``datetime(…, 'localtime')`` rotates each timestamp into the
        machine's zone (per-instant, so DST-correct) before the day prefix is sliced,
        so a bar breaks at local midnight — not at the UTC-offset hour. Read the same
        database in another zone and the days re-bucket to wherever you are, which is
        the point: history is shown in the viewer's local time.

        Returns:
            ``(day, packets, nodes)`` per day with any activity: the day as a local
            ``YYYY-MM-DD``, every stored observation counted, and the distinct
            identified nodes heard (``packet`` rows excluded — no reliable identity).
        """
        rows = self._conn.execute(
            "SELECT substr(datetime(observed_at, 'localtime'), 1, 10) AS day, "
            "COUNT(*) AS pkts, "
            "COUNT(DISTINCT CASE WHEN kind != 'packet' THEN node END) AS nodes "
            "FROM observations GROUP BY day ORDER BY day"
        ).fetchall()
        return [(row["day"], int(row["pkts"]), int(row["nodes"])) for row in rows]

    def hourly_series(self, since: datetime) -> list[tuple[str, int, int]]:
        """Per-clock-hour activity totals since a time, oldest first (the 24 h window feed).

        The hour-resolution sibling of :meth:`daily_activity`: the same packet and
        distinct-node counts, grouped down to the hour. ``observed_at`` is stored as
        UTC, so SQLite's ``datetime(…, 'localtime')`` rotates each timestamp into the
        machine's zone before the ``YYYY-MM-DDTHH`` key is sliced — the buckets break
        on local hour boundaries in every zone, half-hour offsets included, not on UTC
        edges. A day of history folds into ~24 rows however dense it is. Only hours
        with traffic come back; the caller fills the quiet ones (see
        :func:`~meshterm.ui.timemachine_screen._fill_hours`) so the axis is real
        clock time. The ``since`` filter stays on the raw UTC column — the window is
        an absolute span; only the labelling is local.

        Args:
            since: Only observations at or after this time.

        Returns:
            ``(hour, packets, nodes)`` per active hour: ``hour`` as a local
            ``YYYY-MM-DDTHH``, every observation counted, and the distinct identified
            nodes heard (``packet`` rows excluded from the node count — no reliable
            identity), oldest first.
        """
        rows = self._conn.execute(
            "SELECT replace(substr(datetime(observed_at, 'localtime'), 1, 13), ' ', 'T') "
            "AS hour, COUNT(*) AS pkts, "
            "COUNT(DISTINCT CASE WHEN kind != 'packet' THEN node END) AS nodes "
            "FROM observations WHERE observed_at >= ? GROUP BY hour ORDER BY hour",
            (since.isoformat(),),
        ).fetchall()
        return [(row["hour"], int(row["pkts"]), int(row["nodes"])) for row in rows]

    def hourly_activity(self, *, since: Optional[datetime] = None) -> list[int]:
        """Observation counts by local hour of day (0–23) across the stored history.

        The whole-mesh Rhythm chart's feed: every stored observation counted into
        the hour-of-day it arrived. ``observed_at`` is stored as UTC, so SQLite's
        ``datetime(…, 'localtime')`` rotates each timestamp into the machine's zone
        (per-instant, so DST-correct) before the ``HH`` slice, giving a histogram
        already in local hours — no caller rotation needed. The cost stays 24 rows
        however deep the history grows.

        Args:
            since: Only observations at or after this time, if given.

        Returns:
            24 counts, index = local hour.
        """
        sql = (
            "SELECT substr(datetime(observed_at, 'localtime'), 12, 2) AS hh, "
            "COUNT(*) AS n FROM observations"
        )
        params: list[Any] = []
        if since is not None:
            sql += " WHERE observed_at >= ?"
            params.append(since.isoformat())
        sql += " GROUP BY hh"
        counts = [0] * 24
        for row in self._conn.execute(sql, params).fetchall():
            try:
                counts[int(row["hh"])] += int(row["n"])
            except (TypeError, ValueError, IndexError):
                continue  # a malformed stray timestamp simply isn't counted
        return counts

    def quarter_hour_activity(self, *, since: Optional[datetime] = None) -> list[int]:
        """Observation counts by local quarter-hour of day (0–95) across the history.

        The finer sibling of :meth:`hourly_activity` — four slices an hour instead of
        one — feeding the whole-mesh Rhythm chart at its full 15-minute resolution. The
        slice index is ``HH * 4 + MM // 15``, computed in SQL off the ``HH``/``MM``
        substrings of ``datetime(…, 'localtime')`` (``observed_at`` is stored as UTC,
        rotated per-instant into the machine's zone before slicing, so DST-correct), so
        the histogram lands in local time with no caller rotation. The cost stays 96
        rows however deep the history grows.

        Args:
            since: Only observations at or after this time, if given.

        Returns:
            96 counts, index = local quarter-hour slice of the day.
        """
        sql = (
            "SELECT CAST(substr(datetime(observed_at, 'localtime'), 12, 2) AS INTEGER) "
            "* 4 + CAST(substr(datetime(observed_at, 'localtime'), 15, 2) AS INTEGER) "
            "/ 15 AS slot, COUNT(*) AS n FROM observations"
        )
        params: list[Any] = []
        if since is not None:
            sql += " WHERE observed_at >= ?"
            params.append(since.isoformat())
        sql += " GROUP BY slot"
        counts = [0] * 96
        for row in self._conn.execute(sql, params).fetchall():
            try:
                counts[int(row["slot"])] += int(row["n"])
            except (TypeError, ValueError, IndexError):
                continue  # a malformed stray timestamp simply isn't counted
        return counts

    def first_seen(
        self, *, since: Optional[datetime] = None
    ) -> list[tuple[str, Optional[str], datetime]]:
        """When each node first ever appeared in the history, newest arrivals first.

        The Time Machine's "new arrivals" feed: a single ordered scan folds out each
        node's earliest observation and its most recent advertised name (``packet``
        rows excluded — no reliable identity).

        Args:
            since: Only nodes whose *first* appearance is at or after this time.

        Returns:
            ``(node, latest_name, first_heard)`` triples, most recent arrival first.
        """
        rows = self._conn.execute(
            "SELECT node, name, observed_at FROM observations "
            "WHERE node IS NOT NULL AND kind != 'packet' ORDER BY observed_at"
        ).fetchall()
        firsts: dict[str, datetime] = {}
        names: dict[str, str] = {}
        for row in rows:
            node = row["node"]
            if node not in firsts:
                try:
                    firsts[node] = datetime.fromisoformat(row["observed_at"])
                except (TypeError, ValueError):
                    continue
            if row["name"]:
                names[node] = row["name"]
        arrivals = [
            (node, names.get(node), first)
            for node, first in firsts.items()
            if since is None or first >= since
        ]
        arrivals.sort(key=lambda t: t[2], reverse=True)
        return arrivals

    def kind_counts(self, *, since: Optional[datetime] = None) -> dict[str, int]:
        """Stored observation tallies by packet class, optionally windowed.

        The persistent seed of the dashboard's traffic panel: what the recorder has
        heard across sessions, by kind, so the panel opens populated instead of
        counting from zero every launch. Messages and acks are separate event
        families (not observations) and are counted live on top of this.

        Args:
            since: Only observations at or after this time, if given.

        Returns:
            ``kind → count`` for every packet class ever stored (in the window).
        """
        sql = "SELECT kind, COUNT(*) AS n FROM observations"
        params: list[Any] = []
        if since is not None:
            sql += " WHERE observed_at >= ?"
            params.append(since.isoformat())
        sql += " GROUP BY kind"
        rows = self._conn.execute(sql, params).fetchall()
        return {row["kind"]: row["n"] for row in rows}

    def prune_observations(self, older_than: datetime) -> int:
        """Delete observations that aged past the retention window (housekeeping).

        The once-per-session sweep that keeps a permanently-recording database
        bounded: everything the dashboard and Time Machine read lives inside the
        retention window, so rows beyond it are pure weight. Timestamps compare as
        strings, like every other ``observed_at`` filter here.

        Args:
            older_than: Observations strictly before this time are deleted.

        Returns:
            How many rows were removed (0 when the window has no stale rows).
        """
        cur = self._conn.execute(
            "DELETE FROM observations WHERE observed_at < ?", (older_than.isoformat(),)
        )
        self._conn.commit()
        return cur.rowcount if cur.rowcount is not None and cur.rowcount > 0 else 0

    def node_names(self) -> dict[str, str]:
        """The most recent advertised name per node, across everything ever recorded.

        The fill-in-the-blanks source for screens that meet a node id without a name
        (a telemetry-only node in the Time Machine, a relay hash in the dashboard
        feed): whatever name that node *ever* put on the air. One indexed scan;
        ``packet`` rows are excluded because they carry no reliable identity.

        Returns:
            Latest non-empty name keyed by stored node id (the 12-hex key prefix).
        """
        rows = self._conn.execute(
            "SELECT node, name FROM observations "
            "WHERE node IS NOT NULL AND name IS NOT NULL AND name != '' "
            "AND kind != 'packet' ORDER BY observed_at"
        ).fetchall()
        return {row["node"]: row["name"] for row in rows}

    def heard_nodes(self, *, since: Optional[datetime] = None) -> list[HeardNode]:
        """Aggregate stored observations into per-node reception statistics.

        Spans every monitoring run (optionally limited to recent history), so the result
        is a longitudinal view of which nodes have been heard, how strongly, and where —
        the substrate for the monitor summary and the coverage map. ``packet``-kind rows
        are excluded: their SNR describes our link to the packet's *last relay*, not to
        the originating node, so folding them in would misattribute reception quality
        (they feed the topology graph instead — see :meth:`packet_paths`).

        Args:
            since: Only include observations at or after this time, if given.

        Returns:
            One :class:`HeardNode` per distinct node, ordered by most-recently heard.
        """
        sql = (
            "SELECT node, name, node_type, snr, rssi, lat, lon, observed_at "
            "FROM observations WHERE kind != 'packet'"
        )
        params: list[Any] = []
        if since is not None:
            sql += " AND observed_at >= ?"
            params.append(since.isoformat())
        rows = self._conn.execute(sql, params).fetchall()

        grouped: dict[Optional[str], list[Observation]] = {}
        for row in rows:
            obs = Observation(
                node=row["node"],
                name=row["name"],
                node_type=row["node_type"],
                snr=row["snr"],
                rssi=row["rssi"],
                lat=row["lat"],
                lon=row["lon"],
                observed_at=datetime.fromisoformat(row["observed_at"]),
            )
            grouped.setdefault(row["node"], []).append(obs)

        nodes = [HeardNode.from_observations(node, obs) for node, obs in grouped.items()]
        return sorted(nodes, key=lambda n: n.last_seen, reverse=True)

    # -- chat messages ----------------------------------------------------------

    def record_chat_message(
        self, msg: ChatMessage, *, run_id: Optional[int] = None
    ) -> int:
        """Persist one chat message (sent or received).

        Args:
            msg: The message to store. Its ``peer`` is normalized to lowercase so a
                direct conversation queries back consistently.
            run_id: The owning background ``chat`` run, if any (inbound messages log to
                one; outbound sends may not).

        Returns:
            The new message's primary key.
        """
        peer = msg.peer.lower() if msg.peer else None
        cur = self._conn.execute(
            "INSERT INTO messages "
            "(run_id, outbound, is_channel, channel_id, channel_idx, peer, peer_name, text, "
            "snr, acked, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
            (
                run_id,
                int(msg.outbound),
                int(msg.is_channel),
                msg.channel_id,
                msg.channel_idx,
                peer,
                msg.peer_name,
                msg.text,
                msg.snr,
                None if msg.acked is None else int(msg.acked),
                msg.created_at.isoformat(),
            ),
        )
        self._conn.commit()
        return int(cur.lastrowid)

    def update_chat_ack(self, message_id: int, acked: bool) -> None:
        """Update one outbound message's delivery acknowledgement (used on retry).

        Args:
            message_id: The ``messages`` row to update.
            acked: The new delivery state — ``True`` acknowledged, ``False`` not.
        """
        self._conn.execute(
            "UPDATE messages SET acked = ? WHERE id = ?", (int(acked), message_id)
        )
        self._conn.commit()

    def recent_chat_messages(
        self,
        *,
        is_channel: bool,
        channel_id: Optional[str] = None,
        peer: Optional[str] = None,
        limit: int = 200,
    ) -> list[ChatMessage]:
        """Return a conversation's most recent messages, oldest-first.

        Args:
            is_channel: Whether to load a channel conversation.
            channel_id: The channel's slot-independent identity (channel conversations).
            peer: The contact key prefix (direct conversations).
            limit: Maximum number of messages to return.

        Returns:
            The messages in chronological order (ready to render as a transcript).
        """
        if is_channel:
            where, params = "is_channel = 1 AND channel_id = ?", [channel_id]
        else:
            where, params = "is_channel = 0 AND peer = ?", [(peer or "").lower()]
        rows = self._conn.execute(
            f"SELECT * FROM messages WHERE {where} ORDER BY id DESC LIMIT ?",
            [*params, limit],
        ).fetchall()
        return [self._row_to_chat(row) for row in reversed(rows)]

    def last_chat_messages(self) -> dict[str, ChatMessage]:
        """Return the latest message per conversation, keyed by conversation key.

        Backs the conversation picker's preview snippets. One row per distinct
        conversation, taken as the highest-id (most recent) message in each.

        Returns:
            A mapping of :func:`~meshterm.core.models.conversation_key` to its latest
            :class:`ChatMessage`.
        """
        rows = self._conn.execute(
            "SELECT * FROM messages WHERE id IN ("
            "  SELECT MAX(id) FROM messages GROUP BY "
            "  CASE WHEN is_channel = 1 THEN 'chan:' || channel_id "
            "       ELSE 'dm:' || peer END)"
        ).fetchall()
        return {msg.key: msg for msg in (self._row_to_chat(r) for r in rows)}

    def channel_stats(self) -> dict[str, ChannelStats]:
        """Aggregate stored channel messages into per-channel statistics.

        Backs the channel manager's list lanes: each configured channel's row shows its
        total message count, the age of its last message, and an activity sparkline over
        the trailing :data:`ACTIVITY_WINDOW` — whose :data:`ACTIVITY_BUCKETS`-column
        histogram is built here. Two passes, each grouped/filtered in SQL so the cost
        tracks message volume, not channel count: an aggregate for the all-time totals,
        then the window's individual timestamps, bucketed in Python (the window holds at
        most two hours of chatter, so the row set stays small).

        Timestamps are compared as strings: every ``created_at`` is written by
        ``utcnow().isoformat()`` (a fixed-width UTC ISO-8601 form), so lexicographic order
        *is* chronological order and the window cutoff needs no per-row parsing.
        Messages predating identity-keyed history (a ``NULL`` ``channel_id``; see
        :meth:`backfill_channel_ids`) have no channel to be counted under and are skipped.

        Returns:
            A mapping of channel identity to its :class:`ChannelStats`. Channels with no
            stored messages simply have no entry.
        """
        now = utcnow()
        cutoff = now - ACTIVITY_WINDOW
        bucket_span = ACTIVITY_WINDOW / ACTIVITY_BUCKETS
        rows = self._conn.execute(
            "SELECT channel_id, COUNT(*) AS total, MAX(created_at) AS last_at "
            "FROM messages WHERE is_channel = 1 AND channel_id IS NOT NULL "
            "GROUP BY channel_id"
        ).fetchall()

        histograms: dict[str, list[int]] = {}
        recent_rows = self._conn.execute(
            "SELECT channel_id, created_at FROM messages "
            "WHERE is_channel = 1 AND channel_id IS NOT NULL AND created_at >= ?",
            (cutoff.isoformat(),),
        ).fetchall()
        for row in recent_rows:
            try:
                created_at = datetime.fromisoformat(row["created_at"])
                # Bucket by *age* so the histogram comes out newest-first (bucket 0 holds
                # the current five minutes) — the order the sparkline widget expects.
                idx = min(ACTIVITY_BUCKETS - 1, int((now - created_at) / bucket_span))
            except (TypeError, ValueError):
                continue  # a malformed/naive stray simply doesn't land in a bucket
            histogram = histograms.setdefault(row["channel_id"], [0] * ACTIVITY_BUCKETS)
            histogram[max(0, idx)] += 1

        stats: dict[str, ChannelStats] = {}
        for row in rows:
            try:
                last_at = datetime.fromisoformat(row["last_at"])
            except (TypeError, ValueError):
                last_at = None  # a malformed stray must not hide the channel's counts
            histogram = tuple(histograms.get(row["channel_id"], [0] * ACTIVITY_BUCKETS))
            stats[row["channel_id"]] = ChannelStats(
                total=int(row["total"]),
                recent=sum(histogram),
                last_at=last_at,
                histogram=histogram,
            )
        return stats

    def backfill_channel_ids(self, mapping: dict[int, str]) -> int:
        """Give legacy channel messages an identity, keyed by the slot they were stored on.

        Messages written before channel history was keyed by identity have a ``NULL``
        ``channel_id``. There is no record of which channel occupied each slot back then, so
        the best available guess is the channel *currently* at that slot. ``mapping`` maps a
        slot index to the identity of the channel now there; only rows still missing an
        identity are touched, so this is safe to run on every startup.

        Args:
            mapping: Slot index to the current channel identity at that slot.

        Returns:
            The number of legacy rows given an identity.
        """
        changed = 0
        for idx, channel_id in mapping.items():
            cur = self._conn.execute(
                "UPDATE messages SET channel_id = ? "
                "WHERE is_channel = 1 AND channel_id IS NULL AND channel_idx = ?",
                (channel_id, idx),
            )
            changed += cur.rowcount
        if changed:
            self._conn.commit()
        return changed

    # -- persisted UI state -----------------------------------------------------

    def get_map_view(self) -> Optional[tuple[float, float, int]]:
        """Return the last saved map viewport as ``(center_lat, center_lon, zoom)``.

        Lets the interactive map reopen exactly where the user left it. Returns ``None``
        when no view has been saved yet, or a stored value can't be parsed (treated as
        absent rather than an error, so a corrupt row just refits to the nodes).
        """
        row = self._conn.execute(
            "SELECT value FROM app_state WHERE key = 'map_view'"
        ).fetchone()
        if row is None:
            return None
        try:
            data = json.loads(row["value"])
            return float(data["lat"]), float(data["lon"]), int(data["zoom"])
        except (ValueError, KeyError, TypeError):
            return None

    def set_map_view(self, lat: float, lon: float, zoom: int) -> None:
        """Persist the map viewport so the next session reopens on the same spot.

        Args:
            lat: Latitude at the centre of the view.
            lon: Longitude at the centre of the view.
            zoom: Display zoom level.
        """
        self._conn.execute(
            "INSERT OR REPLACE INTO app_state(key, value) VALUES ('map_view', ?)",
            (json.dumps({"lat": lat, "lon": lon, "zoom": zoom}),),
        )
        self._conn.commit()

    @staticmethod
    def _row_to_chat(row: sqlite3.Row) -> ChatMessage:
        """Rebuild a :class:`ChatMessage` from a ``messages`` row."""
        return ChatMessage(
            text=row["text"],
            outbound=bool(row["outbound"]),
            is_channel=bool(row["is_channel"]),
            channel_id=row["channel_id"],
            channel_idx=row["channel_idx"],
            peer=row["peer"],
            peer_name=row["peer_name"],
            snr=row["snr"],
            acked=None if row["acked"] is None else bool(row["acked"]),
            created_at=datetime.fromisoformat(row["created_at"]),
            row_id=row["id"],
        )