"""The Watchtower: passive rules over what the hub already hears, raising alerts. A sentinel, not a prober — it transmits nothing and asks the radio for nothing. It rides the always-on :class:`~meshterm.services.event_hub.EventHub` like the monitor does, and watches for the things an operator actually loses sleep over: * **silence** — a starred node hasn't been heard for its configured threshold. Fires once per quiet spell (latched in the store) and re-arms when the node returns, which also raises a friendly **recovered** note. * **SNR sag** — a starred node's receptions are degrading: the median of its last few readings sits well below the median of the readings before them. A cooldown keeps a slowly dying link from paging every minute. * **new node** — something never before seen (not in the observation history at start, not in the store's memory) just appeared on the mesh. Announced at most once per id, ever. Alerts land in the :class:`~meshterm.core.watch_store.WatchStore` (which persists them), feed the header's alert badge, and are read and acknowledged in the Watchtower screen (:mod:`meshterm.ui.watchtower_screen`). The service is session-scoped state on the :class:`~meshterm.context.AppContext` (``ctx.watchtower``), started alongside the other always-on services; scripted CLI runs never start it. """ from __future__ import annotations import asyncio import statistics from collections import deque from datetime import datetime from typing import TYPE_CHECKING from ..core.connection import Unsubscribe from ..core.events import EventKind, MeshEvent from ..core.models import Observation, utcnow from ..core.watch_store import OFF if TYPE_CHECKING: from ..context import AppContext #: Seconds between rule sweeps (the silence rule is time-driven, not packet-driven). SWEEP_S = 30.0 #: How many recent SNR readings form the "now" side of the sag comparison. SNR_WINDOW = 4 #: How far (dB) the recent median must sit below the prior median to alert. SNR_SAG_DB = 6.0 #: Seconds before the sag rule may fire again for the same node. SNR_COOLDOWN_S = 6 * 3600.0 #: How many SNR readings are kept per node (the comparison uses at most this many). _SNR_KEEP = 12 class WatchtowerService: """Watches starred nodes and the mesh's cast of characters; raises alerts. Attributes are private; interact through the properties and the async lifecycle methods (:meth:`start`, :meth:`stop`, :meth:`aclose`). Rule evaluation lives in the synchronous :meth:`note` / :meth:`evaluate` so tests can drive it directly. """ def __init__(self, ctx: AppContext) -> None: """Initialize the (idle) service. Args: ctx: The shared application context (store, repository, event hub). """ self._ctx = ctx self._unsubscribe: Unsubscribe | None = None self._task: asyncio.Task | None = None #: Every node id ever seen (DB history + store memory + this session), the #: baseline the new-node rule compares against. ``None`` until started. self._known: set[str] | None = None #: Rolling SNR readings per watched node, session-scoped. self._snr: dict[str, deque[float]] = {} #: When the sag rule last fired per node (the cooldown clock). self._snr_fired: dict[str, datetime] = {} @property def active(self) -> bool: """Whether the sentinel is currently watching.""" return self._unsubscribe is not None def unacked_count(self) -> int: """Alerts awaiting acknowledgement — the header badge's number.""" return self._ctx.watch_store.unacked_count() # --- lifecycle --------------------------------------------------------------------- async def start(self) -> None: """Begin watching. Idempotent, device-free, and safe before any connection. Seeds the new-node baseline from the observation history (every id the DB has ever heard) plus the store's memory of past announcements, then subscribes to the hub and starts the sweep. Like the monitor, packets flow in whenever the hub is pumping — including a hub that opens lazily later. """ if self.active: return store = self._ctx.watch_store baseline = {h.node for h in self._ctx.repo.heard_nodes() if h.node} baseline |= set(store.state.known) baseline |= set(store.watched()) self._known = baseline def on_event(event: MeshEvent) -> None: obs = event.observation if obs is not None: try: self.note(obs) except Exception as exc: # noqa: BLE001 - a rule bug must not kill the hub self._ctx.log.debug("watchtower: note failed: %s", exc) self._unsubscribe = self._ctx.events.subscribe(on_event, EventKind.OBSERVATION) self._task = asyncio.ensure_future(self._sweep()) self._ctx.log.info("watchtower watching (%d nodes starred)", len(store.watched())) async def stop(self) -> None: """Stop watching and flush pending store writes. Idempotent.""" if self._unsubscribe is not None: self._unsubscribe() self._unsubscribe = None if self._task is not None: self._task.cancel() try: await self._task except (asyncio.CancelledError, Exception): # noqa: BLE001 - teardown pass self._task = None self._ctx.watch_store.flush() async def aclose(self) -> None: """Stop watching at session end.""" await self.stop() async def _sweep(self) -> None: """Run the time-driven rules on a slow heartbeat.""" while True: await asyncio.sleep(SWEEP_S) try: self.evaluate() except Exception as exc: # noqa: BLE001 - keep the sentinel alive self._ctx.log.debug("watchtower: sweep failed: %s", exc) # --- packet-driven rules ------------------------------------------------------------- def note(self, obs: Observation) -> None: """Fold one observation into the rules (called per hub packet; keep cheap). Args: obs: The overheard observation. """ node = obs.node if not node: return store = self._ctx.watch_store entry = store.watched().get(node) # New node: never in the DB, the store's memory, or this session before now. # Starring a node is itself an introduction, so watched nodes are never "new". if self._known is not None and node not in self._known: self._known.add(node) store.remember_known(node) if entry is None and store.new_node_alerts: label = obs.name or node store.add_alert( "new-node", label, "first appearance — never heard on this mesh before", when=obs.observed_at, ) if entry is None: return was_silent = entry.silent_since is not None store.note_heard(node, when=obs.observed_at, name=obs.name) if was_silent: store.clear_silent(node) store.add_alert( "recovered", entry.name, "back on the air — heard again after a silence alarm", when=obs.observed_at, ) # Packet-kind rows measure our link to the last relay, not to the node, so # they prove liveness (above) but say nothing about the node's own signal. if entry.snr_watch and obs.snr is not None and obs.kind != "packet": self._track_snr(node, entry.name, obs) def _track_snr(self, node: str, label: str, obs: Observation) -> None: """Fold one SNR reading in and fire the sag rule when the trend warrants.""" readings = self._snr.setdefault(node, deque(maxlen=_SNR_KEEP)) readings.append(float(obs.snr)) # type: ignore[arg-type] if len(readings) < 2 * SNR_WINDOW: return recent = statistics.median(list(readings)[-SNR_WINDOW:]) prior = statistics.median(list(readings)[:-SNR_WINDOW]) if prior - recent < SNR_SAG_DB: return fired = self._snr_fired.get(node) now = obs.observed_at or utcnow() if fired is not None and (now - fired).total_seconds() < SNR_COOLDOWN_S: return self._snr_fired[node] = now self._ctx.watch_store.add_alert( "snr", label, f"reception sagging — median {prior:+.1f} dB → {recent:+.1f} dB", when=now, ) # --- time-driven rules --------------------------------------------------------------- def evaluate(self, now: datetime | None = None) -> None: """Run the silence rule over every watched node and flush the store. Args: now: The evaluation time (defaults to the current time; tests inject). """ now = now or utcnow() store = self._ctx.watch_store for key, entry in store.watched().items(): hours = entry.silence_hours if hours == OFF or entry.last_heard is None or entry.silent_since is not None: continue quiet_s = (now - entry.last_heard).total_seconds() if quiet_s >= hours * 3600: store.mark_silent(key, now) store.add_alert( "silence", entry.name, f"nothing heard for {hours} h", when=now, ) store.flush()