From 6c63697e216dfb398a13fcc272046ae5d42ff0c9 Mon Sep 17 00:00:00 2001 From: Jean-Pierre Martineau Date: Tue, 3 Nov 2026 02:47:05 -0500 Subject: [PATCH] Nodes get a place: the named area nearest their location MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A background job finds the place of each node that advertises a location (Villeray, Montréal; a village; a peak near its summit). The map tiles give the first answer offline; Nominatim corrects it when online (place_source, default tiles then Nominatim; nominatim_url). Places are stored in the new node_places table, a typed place in places.json wins over a found one. Co-Authored-By: Claude Haiku 5.5 --- CLAUDE.md | 2 + docs/development/writing-style.md | 2 + meshterm/context.py | 25 ++ meshterm/core/place_store.py | 137 +++++++++ meshterm/core/places.py | 385 ++++++++++++++++++++++++ meshterm/core/preferences.py | 26 ++ meshterm/persistence/db.py | 17 +- meshterm/persistence/repository.py | 76 +++++ meshterm/services/basemap.py | 28 ++ meshterm/services/places.py | 454 +++++++++++++++++++++++++++++ meshterm/ui/menu.py | 3 + tests/test_places.py | 342 ++++++++++++++++++++++ 12 files changed, 1496 insertions(+), 1 deletion(-) create mode 100644 meshterm/core/place_store.py create mode 100644 meshterm/core/places.py create mode 100644 meshterm/services/places.py create mode 100644 tests/test_places.py diff --git a/CLAUDE.md b/CLAUDE.md index be8b53d..cf6d79f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,6 +82,8 @@ them instead of hand-rolling: | path | an ordered hop spec you compose or force (`a1,3d,…`) | | route | the concrete node sequence a trace walked or will walk | | via | prefix for a packet/message's relay chain | +| location | the coordinates a node advertises (latitude, longitude) — `Set location`, the location preview, ^L locate | +| place | the named area nearest a node's location: a neighbourhood, a village, a peak (`core/places.py`, `services/places.py`). Read as *fine, coarse* (`Villeray, Montréal`); a village, town, city or peak stands alone; a path chip shows the fine part. Found from the map tiles, then Nominatim (`place_source`); a place the user typed wins. Never "location name" | | region | a named area a repeater relays floods for (`region put yul`); shown bare, never `#yul` — the `#` belongs to key derivation only | | scope | the region a flood is limited to — a packet's, a channel's send scope, the device's default scope. A direct packet has none. MeshCore's sense and the only one: the menu's This node / Managed nodes / This app sections name *whose* a setting is, never a "scope" | | unscoped | a plain flood, relayed by any repeater that allows the wildcard `*` | diff --git a/docs/development/writing-style.md b/docs/development/writing-style.md index 1509fe9..80e6250 100644 --- a/docs/development/writing-style.md +++ b/docs/development/writing-style.md @@ -184,6 +184,8 @@ and in quoted text. | route | The sequence of nodes that a trace went through or will go through | | | hop | One node in a path or a route | | | via | The prefix for the relay chain of a packet or a message | | +| location | The coordinates (latitude and longitude) that a node advertises | coordinates (alone) | +| place | The named area nearest the location of a node: a neighbourhood, a village, a town, or a peak. It has a fine part and a coarse part (`Villeray, Montréal`) | location name, locality | | region | A named area for which a repeater relays floods (`region put yul`). Shown without `#` | | | scope | The region to which a flood is limited | | | unscoped | A flood that each repeater relays when it permits the wildcard `*` | | diff --git a/meshterm/context.py b/meshterm/context.py index ff46270..d42f240 100644 --- a/meshterm/context.py +++ b/meshterm/context.py @@ -24,6 +24,7 @@ from .core.courier_store import CourierStore from .core.device_store import DeviceStore from .core.discovery import DiscoveredDevice, discover_devices from .core.mute_store import MuteStore +from .core.place_store import PlaceStore from .core.preferences import PREFERENCES_FILENAME, Preferences, report_dropped from .core.preferences import install as install_preferences from .core.region_store import RegionStore @@ -46,6 +47,7 @@ if TYPE_CHECKING: from .services.device_state import DeviceState from .services.event_hub import EventHub from .services.monitor_service import MonitorService + from .services.places import PlaceService from .services.rooms import RoomService from .services.watchtower import WatchtowerService from .ui.surface import Ui @@ -101,6 +103,9 @@ class AppContext: room_store: The store for the rooms that this machine joined: the password of each room and the access that it granted (if the caller does not inject it, the default is ``/rooms.json``). + place_store: The store for the places that the user typed for nodes. A typed place + wins over a found place (if the caller does not inject it, the default is + ``/places.json``). mock: Whether the context uses the simulator device. demo: Whether the simulator is the demo world of ``--demo`` (:mod:`meshterm.demo`), which implies ``mock``. @@ -145,6 +150,7 @@ class AppContext: settings_store: SettingsStore | None = None contact_store: ContactStore | None = None room_store: RoomStore | None = None + place_store: PlaceStore | None = None profile: DeviceProfile | None = None mock: bool = False demo: bool = False @@ -189,6 +195,7 @@ class AppContext: _clock_sync: ClockSync | None = field(default=None, init=False, repr=False) _courier: CourierService | None = field(default=None, init=False, repr=False) _battery: BatteryService | None = field(default=None, init=False, repr=False) + _places: PlaceService | None = field(default=None, init=False, repr=False) _devstate: DeviceState | None = field(default=None, init=False, repr=False) _basemap_source: BasemapSource | None = field(default=None, init=False, repr=False) _ui: Ui | None = field(default=None, init=False, repr=False) @@ -225,6 +232,8 @@ class AppContext: self.contact_store = ContactStore(self.settings.config_dir / "contacts.json") if self.room_store is None: self.room_store = RoomStore(self.settings.config_dir / "rooms.json") + if self.place_store is None: + self.place_store = PlaceStore(self.settings.config_dir / "places.json") @property def profile_name(self) -> str | None: @@ -481,6 +490,20 @@ class AppContext: self._courier = CourierService(self) return self._courier + @property + def places(self) -> PlaceService: + """Return the places of the nodes, and create the service at first use. + + A screen reads a place from it at each paint (:meth:`PlaceService.get`). The + interactive session starts its background job with the other always-on services. + A scripted CLI run only reads the stored places, and never starts the job. + """ + if self._places is None: + from .services.places import PlaceService + + self._places = PlaceService(self) + return self._places + @property def battery(self) -> BatteryService: """Return the battery poller of the session, and create it at first use. @@ -1047,6 +1070,8 @@ class AppContext: await self._courier.aclose() if self._battery is not None: await self._battery.aclose() + if self._places is not None: + await self._places.aclose() if self._watchtower is not None: await self._watchtower.aclose() if self._clock_sync is not None: diff --git a/meshterm/core/place_store.py b/meshterm/core/place_store.py new file mode 100644 index 0000000..6e201f1 --- /dev/null +++ b/meshterm/core/place_store.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The places that the user typed for nodes. + +MeshTerm finds the place of a node from its location (refer to :mod:`meshterm.core.places`). +But the user can know better: a repeater on the roof of a known building, or a node whose +location is far from where it really is. The node page lets the user type a place for the +node, and this store keeps it. A typed place wins over each found place, on each screen. + +The store finds a place by the key prefix of the node (12 hex digits, the same id as +``observations.node``). A place belongs to a node and not to a contact: a repeater in a path +is often no contact of the device. The file is ``/places.json``, global machine +state like the other choices of the user. After the first read, the store answers reads from +memory, because the Contacts list and each path line read it at each paint. A write occurs +only when the user changes a place, and the write is atomic. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import ClassVar + +from .atomicwrite import write_atomically + + +def _norm(node: str) -> str: + """The 12-hex id of a node, from a key prefix or a full public key.""" + return node.lower().removeprefix("0x")[:12] + + +@dataclass(frozen=True, slots=True) +class TypedPlace: + """The place that the user typed for one node. + + Attributes: + name: The place, as the user typed it. + set_at: When the user typed it (seconds since the epoch). + """ + + name: str + set_at: int + + #: The names that left this record. A retired name is never used again with a new + #: meaning (refer to :data:`meshterm.core.preferences.RETIRED` for the reason). + RETIRED: ClassVar[frozenset[str]] = frozenset() + + +class PlaceStore: + """Reads and writes the places that the user typed, from memory first.""" + + def __init__(self, path: Path) -> None: + """Open the store on the location of a JSON file. + + Args: + path: The path to the JSON state file. The store makes the file at the first + typed place. + """ + self._path = path + self._places: dict[str, TypedPlace] | None = None + + @property + def _state(self) -> dict[str, TypedPlace]: + """The typed places by node id, read from disk at the first access.""" + if self._places is None: + self._places = self._load() + return self._places + + def _load(self) -> dict[str, TypedPlace]: + """Parse the file into typed records (empty for a bad or missing file).""" + try: + data = json.loads(self._path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + entries = data.get("places") if isinstance(data, dict) else None + if not isinstance(entries, dict): + return {} + places: dict[str, TypedPlace] = {} + for node, entry in entries.items(): + if not isinstance(node, str) or not isinstance(entry, dict): + continue + name = entry.get("name") + if not isinstance(name, str) or not name.strip(): + continue + try: + set_at = int(entry.get("set_at") or 0) + except (TypeError, ValueError): + set_at = 0 + places[_norm(node)] = TypedPlace(name.strip(), set_at) + return places + + def get(self, node: str | None) -> str | None: + """The place that the user typed for a node, or ``None``. + + Args: + node: The key prefix or the full public key of the node. + """ + if not node: + return None + typed = self._state.get(_norm(node)) + return typed.name if typed else None + + def all(self) -> dict[str, str]: + """Each typed place by node id, as a copy that the caller can keep.""" + return {node: typed.name for node, typed in self._state.items()} + + def set(self, node: str, name: str | None, *, when: int) -> None: + """Type a place for a node, or clear it with ``None`` or an empty name. + + Args: + node: The key prefix or the full public key of the node. + name: The place. ``None`` or blank text clears the typed place, and the found + place shows again. + when: The time of the change (seconds since the epoch). + """ + key = _norm(node) + name = (name or "").strip() + current = self._state.get(key) + if not name: + if current is None: + return + del self._state[key] + else: + if current is not None and current.name == name: + return + self._state[key] = TypedPlace(name, when) + self._save() + + def _save(self) -> None: + """Store all the typed places atomically, from the records.""" + data = { + "places": { + node: {"name": typed.name, "set_at": typed.set_at} + for node, typed in sorted(self._state.items()) + } + } + write_atomically(self._path, json.dumps(data, indent=2, ensure_ascii=False)) diff --git a/meshterm/core/places.py b/meshterm/core/places.py new file mode 100644 index 0000000..eca0255 --- /dev/null +++ b/meshterm/core/places.py @@ -0,0 +1,385 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The place of a node: the named area nearest its location, in pure functions. + +Many nodes have a name that says nothing about where they are: a call sign (``VE2XYZ``) or +the name of the hardware (``RAK-1W``). A node that advertises a location has a better answer +near it on the map. This module finds that answer, the **place** of the node: a +neighbourhood, a village, a town, or a peak. It has no I/O. A caller gives it the decoded +tiles (:func:`place_from_tiles`) or the document that Nominatim returned +(:func:`place_from_nominatim`), and it returns a :class:`Place`. + +A place has two parts. The **fine** part is the most precise name: a neighbourhood in a +city, or the village itself in the country. The **coarse** part is the town or the city +that holds a fine part that is not itself a town: ``Villeray, Montréal``. A village, a town, +a city, and a peak stand alone. A path chip shows only the fine part, because a chip must be +short (:attr:`Place.short`). + +**The tiles give points, not areas.** The OpenMapTiles ``place`` layer has one point for +each place, where its label goes. It has no boundary. Thus the tiles can tell the nearest +place, but not the place that contains a node. Near a border, the answer can be the +neighbouring place. :func:`place_from_tiles` limits this error: each class has a radius +past which its point no longer names the node, and the candidate that is nearest relative +to its own radius wins. Nominatim knows the boundaries, and when the preference permits, +its answer replaces the answer of the tiles (refer to :mod:`meshterm.services.places`). + +**A peak wins near its summit.** Many repeaters are on a summit, and there the name of the +peak tells more than the name of the town below it. A named peak within +:data:`PEAK_RADIUS_KM` of the node is the place. Nominatim does not replace such a place. +""" + +from __future__ import annotations + +import math +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from .geo import TILE_PX, haversine_km, lonlat_to_world, usable_fix, world_to_lonlat + +if TYPE_CHECKING: + from .mvt import Layer + + #: Loads the decoded layers of one tile ``(z, x, y)``, or ``None`` when it has no answer. + TileLoader = Callable[[int, int, int], list[Layer] | None] + +#: Where a place came from. ``tiles`` and ``nominatim`` are found. ``user`` is the place +#: that the user typed for a node, and it wins over a found place. +SOURCE_TILES = "tiles" +SOURCE_NOMINATIM = "nominatim" +SOURCE_USER = "user" + +#: The layers of a tile that :func:`place_from_tiles` reads. +PLACE_LAYERS = frozenset({"place", "mountain_peak"}) + +#: The distance (km) within which a named peak is the place of a node. +PEAK_RADIUS_KM = 0.3 + +#: The distance (km) that a node must move before MeshTerm finds its place again. +MOVE_KM = 0.5 + +#: The classes that are a part of a town or a city, from the most specific, with the radius +#: (km) past which their point no longer names a node. The most specific class that has a +#: point within its radius gives the fine part: ``Villeray``, not the borough around it. A +#: hamlet is here and not with the settlements. Its name is often known only near it, so it +#: gets the name of its town as the coarse part. An island is last, because a large island +#: holds a full city. +_AREA_RADIUS_KM: dict[str, float] = { + "neighbourhood": 1.2, + "quarter": 1.8, + "hamlet": 1.5, + "suburb": 2.5, + "borough": 4.0, + "island": 1.5, +} + +#: The settlements, with the radius (km) within which one is the fine part, alone. +_SETTLEMENT_RADIUS_KM: dict[str, float] = { + "village": 2.5, + "town": 4.0, + "city": 6.0, +} + +#: The weight of a settlement when MeshTerm chooses the coarse part. The distance is divided +#: by the weight, so a city a little farther away wins over a small town nearer. A node in a +#: district of a city is usually nearer to the point of a neighbouring town than to the +#: centre of its own city. +_SETTLEMENT_WEIGHT: dict[str, float] = {"village": 1.0, "town": 2.0, "city": 4.0} + +#: The ``rank`` that MeshTerm uses for a settlement whose tile gives none. OpenMapTiles ranks +#: the labels of a tile from 1, the most important. A large city has a low rank. For a node +#: in Ahuntsic, Montréal (rank 3, 9.4 km) must win over Laval (rank 11, 4.4 km), and the +#: class weight alone cannot see that difference. Thus the rank multiplies the distance +#: (refer to :func:`_coarse`). +_DEFAULT_RANK = 10 + +#: The classes of ``mountain_peak`` that have a summit. +_PEAK_CLASSES = frozenset({"peak", "volcano"}) + +#: The zoom of the tiles for the settlements and the peaks, and how far (km) MeshTerm looks +#: in them. At z11, a tile holds cities, towns, villages, hamlets, suburbs, islands, and the +#: important peaks, at approximately 40 KB a tile. +_COARSE_ZOOM = 11 +_COARSE_REACH_KM = 15.0 + +#: The zoom of the tiles for the parts of a town, and how far (km) MeshTerm looks in them. +#: Only z14 has the neighbourhoods, the quarters, and the boroughs. MeshTerm loads these +#: tiles only for a node that is in a town (refer to :func:`place_from_tiles`). +_FINE_ZOOM = 14 +_FINE_REACH_KM = 2.0 + +#: The zoom and the reach (km) for a node far from all settlements at :data:`_COARSE_ZOOM`. +_FAR_ZOOM = 9 +_FAR_REACH_KM = 50.0 + + +@dataclass(frozen=True, slots=True) +class Place: + """The place of a node. + + Attributes: + fine: The most precise name (a neighbourhood, a village, a peak), or ``None``. + coarse: The town or the city that holds :attr:`fine`, or ``None`` when the fine + part stands alone. + source: :data:`SOURCE_TILES`, :data:`SOURCE_NOMINATIM`, or :data:`SOURCE_USER`. + peak: Whether :attr:`fine` is a peak near its summit. Nominatim does not replace + such a place. + """ + + fine: str | None + coarse: str | None + source: str + peak: bool = False + + @property + def empty(self) -> bool: + """Whether the source answered, but found no place. MeshTerm stores this answer too.""" + return not self.fine and not self.coarse + + @property + def label(self) -> str: + """The full name: ``fine, coarse``, or the one part that there is (``""`` if empty).""" + parts = [p for p in (self.fine, self.coarse) if p] + if len(parts) == 2 and parts[0] == parts[1]: + parts = parts[:1] + return ", ".join(parts) + + @property + def short(self) -> str: + """The name for a path chip: the fine part, else the coarse part (``""`` if empty).""" + return self.fine or self.coarse or "" + + +def moved(lat1: float, lon1: float, lat2: float, lon2: float) -> bool: + """Whether a node moved far enough from where its place was found to find it again.""" + return haversine_km(lat1, lon1, lat2, lon2) > MOVE_KM + + +@dataclass(frozen=True, slots=True) +class _Candidate: + """One named point from a tile, with its class and its distance from the node.""" + + name: str + cls: str + km: float + rank: int = _DEFAULT_RANK + + +def _tiles_near(lat: float, lon: float, zoom: int, reach_km: float) -> list[tuple[int, int, int]]: + """The tiles at ``zoom`` that a square of ``reach_km`` around the point touches. + + The tile that holds the point comes first. If it has no answer, the caller has no answer. + """ + n = 2**zoom + cx, cy = lonlat_to_world(lat, lon, zoom) + # The length of one world pixel at this latitude, in km. + px_km = 2 * math.pi * 6371.0088 * math.cos(math.radians(lat)) / (TILE_PX * n) + reach_px = reach_km / max(px_km, 1e-9) + home = (int(cx // TILE_PX), int(cy // TILE_PX)) + x0, x1 = int((cx - reach_px) // TILE_PX), int((cx + reach_px) // TILE_PX) + y0, y1 = int((cy - reach_px) // TILE_PX), int((cy + reach_px) // TILE_PX) + tiles = [(zoom, home[0] % n, home[1])] + for ty in range(max(y0, 0), min(y1, n - 1) + 1): + for tx in range(x0, x1 + 1): + key = (zoom, tx % n, ty) + if key not in tiles: + tiles.append(key) + return tiles + + +def _candidates( + lat: float, + lon: float, + zoom: int, + reach_km: float, + load: TileLoader, + *, + peaks: list[_Candidate], +) -> list[_Candidate] | None: + """The named place points within ``reach_km`` of the node, in the tiles at ``zoom``. + + The peaks go into ``peaks`` (each peak one time only, also when two tiles have it). + + Returns: + The place candidates, or ``None`` when the tile that holds the node has no answer. + """ + found: dict[tuple[str, str], _Candidate] = {} + for i, (z, x, y) in enumerate(_tiles_near(lat, lon, zoom, reach_km)): + layers = load(z, x, y) + if layers is None: + if i == 0: + return None + continue + for layer in layers: + if layer.name not in PLACE_LAYERS: + continue + for feature in layer.features: + if not feature.rings or not feature.rings[0]: + continue + name = feature.name + if not name: + continue + px, py = feature.rings[0][0] + wx = (x + px / layer.extent) * TILE_PX + wy = (y + py / layer.extent) * TILE_PX + plat, plon = world_to_lonlat(wx, wy, z) + km = haversine_km(lat, lon, plat, plon) + if km > reach_km: + continue + cls = str(feature.tags.get("class") or "") + if layer.name == "mountain_peak": + if cls in _PEAK_CLASSES and all(p.name != name for p in peaks): + peaks.append(_Candidate(name, cls, km)) + continue + try: + rank = int(feature.tags.get("rank")) + except (TypeError, ValueError): + rank = _DEFAULT_RANK + key = (name, cls) + old = found.get(key) + if old is None or km < old.km: + found[key] = _Candidate(name, cls, km, min(rank, old.rank) if old else rank) + return list(found.values()) + + +def _most_specific(candidates: Iterable[_Candidate]) -> _Candidate | None: + """The nearest point of the most specific part of a town that is within its radius.""" + within = [c for c in candidates if c.km <= _AREA_RADIUS_KM.get(c.cls, -1.0)] + for cls in _AREA_RADIUS_KM: + of_class = [c for c in within if c.cls == cls] + if of_class: + return min(of_class, key=lambda c: c.km) + return None + + +def _best(candidates: Iterable[_Candidate], radius: dict[str, float]) -> _Candidate | None: + """The candidate that is nearest relative to the radius of its class, within that radius.""" + best: tuple[float, _Candidate] | None = None + for c in candidates: + r = radius.get(c.cls) + if r is None or c.km > r: + continue + score = c.km / r + if best is None or score < best[0]: + best = (score, c) + return None if best is None else best[1] + + +def _coarse(candidates: Iterable[_Candidate]) -> _Candidate | None: + """The settlement that holds the node most probably: the least weighted distance. + + The weight is the class weight, and the rank of the label (plus two, so that rank 1 + does not weigh three times rank 2) multiplies the distance. + """ + best: tuple[float, _Candidate] | None = None + for c in candidates: + weight = _SETTLEMENT_WEIGHT.get(c.cls) + if weight is None: + continue + score = c.km * (c.rank + 2) / weight + if best is None or score < best[0]: + best = (score, c) + return None if best is None else best[1] + + +def place_from_tiles(lat: float, lon: float, load: TileLoader) -> Place | None: + """Find the place of a location in the vector tiles. + + The steps: + + 1. Load the z11 tiles near the node: its settlements and the important peaks. + 2. If the node is in a town (a settlement or a suburb near it), also load the z14 tiles + near it, for the neighbourhoods, the quarters, and the boroughs. + 3. A named peak within :data:`PEAK_RADIUS_KM` is the place. + 4. Else the nearest point of the most specific part of a town (within its radius) is + the fine part, and the settlement with the least weighted distance is the coarse + part. + 5. Else the settlement that is nearest relative to its radius stands alone. + 6. Else the nearest settlement (looked for up to :data:`_FAR_REACH_KM` away) stands + alone, or the nearest named peak. + + Args: + lat: The latitude of the node. + lon: The longitude of the node. + load: Loads the decoded layers of one tile (``place`` and ``mountain_peak`` at + minimum), or returns ``None`` when it has no answer. + + Returns: + The place, an empty place when the tiles answered but hold nothing near, or ``None`` + when the tile that holds the node has no answer (offline and not cached). The + caller tries a ``None`` again later, and stores the other answers. + """ + if not usable_fix(lat, lon): + return None + peaks: list[_Candidate] = [] + coarse = _candidates(lat, lon, _COARSE_ZOOM, _COARSE_REACH_KM, load, peaks=peaks) + if coarse is None: + return None + # A town or a city near the node (two of its radii), or a suburb around it: there are + # neighbourhoods to look for. + urban = any( + (c.cls in ("town", "city") and c.km <= 2 * _SETTLEMENT_RADIUS_KM[c.cls]) + or (c.cls == "suburb" and c.km <= _AREA_RADIUS_KM["suburb"]) + for c in coarse + ) + fine: list[_Candidate] = [] + if urban: + fine = _candidates(lat, lon, _FINE_ZOOM, _FINE_REACH_KM, load, peaks=peaks) or [] + summit = min((p for p in peaks if p.km <= PEAK_RADIUS_KM), key=lambda p: p.km, default=None) + if summit is not None: + return Place(summit.name, None, SOURCE_TILES, peak=True) + everything = fine + coarse + area = _most_specific(everything) + if area is not None: + holder = _coarse(everything) + return Place(area.name, holder.name if holder else None, SOURCE_TILES) + settlement = _best(everything, _SETTLEMENT_RADIUS_KM) or _coarse(everything) + if settlement is None: + far = _candidates(lat, lon, _FAR_ZOOM, _FAR_REACH_KM, load, peaks=peaks) or [] + settlement = _coarse(far) + if settlement is not None: + return Place(settlement.name, None, SOURCE_TILES) + nearest_peak = min(peaks, key=lambda p: p.km, default=None) + if nearest_peak is not None: + return Place(nearest_peak.name, None, SOURCE_TILES) + return Place(None, None, SOURCE_TILES) + + +#: The address keys of a Nominatim answer, from the most precise. The first key that is +#: present is the fine part. +_NOMINATIM_FINE = ("neighbourhood", "quarter", "suburb", "city_district", "borough", "hamlet") + +#: The address keys of a settlement in a Nominatim answer, in the order of preference. +_NOMINATIM_COARSE = ("city", "town", "village", "municipality") + + +def place_from_nominatim(doc: object) -> Place | None: + """Make a place from the document that the Nominatim ``/reverse`` endpoint returned. + + Args: + doc: The parsed JSON (``format=jsonv2``, ``addressdetails=1``). + + Returns: + The place. An answer with no address (``{"error": "Unable to geocode"}``, the sea) + is an empty place, because Nominatim answered. ``None`` for a document that is not + an answer at all. + """ + if not isinstance(doc, dict): + return None + address = doc.get("address") + if not isinstance(address, dict): + if "error" in doc: + return Place(None, None, SOURCE_NOMINATIM) + return None + + def first(keys: tuple[str, ...]) -> str | None: + for key in keys: + value = address.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + fine = first(_NOMINATIM_FINE) + coarse = first(_NOMINATIM_COARSE) + if fine is None: + return Place(coarse, None, SOURCE_NOMINATIM) + return Place(fine, coarse if coarse != fine else None, SOURCE_NOMINATIM) diff --git a/meshterm/core/preferences.py b/meshterm/core/preferences.py index 81dddb6..32c44a5 100644 --- a/meshterm/core/preferences.py +++ b/meshterm/core/preferences.py @@ -154,6 +154,15 @@ class PrefSpec: #: sees, not the mechanism: an age is how long ago, and a timestamp is when. _CLI_TIME_CHOICES: dict[str, str] = {"relative": "ages", "absolute": "timestamps"} +#: The sources of the place of a node (refer to :mod:`meshterm.services.places`). With +#: ``tiles_nominatim``, the map tiles give the first answer at once, and Nominatim corrects it +#: later, when MeshTerm is online, because only Nominatim knows the boundaries of a place. +_PLACE_SOURCE_CHOICES: dict[str, str] = { + "tiles_nominatim": "tiles, then Nominatim", + "tiles": "map tiles", + "nominatim": "Nominatim", +} + #: The choices for the silence rule. They come from the ring of the Watchtower itself, so #: that the two lists always agree. _SILENCE_CHOICES: dict[int, str] = { @@ -396,6 +405,23 @@ PREFERENCES: tuple[PrefSpec, ...] = ( default="https://tiles.openfreemap.org/planet", relaunch=True, ), + PrefSpec( + key="place_source", + label="Place names", + help="Where the place of a node comes from", + group="Map", + value_type="enum", + default="tiles_nominatim", + choices=_PLACE_SOURCE_CHOICES, + ), + PrefSpec( + key="nominatim_url", + label="Nominatim", + help="The server that finds places by their boundaries", + group="Map", + value_type="str", + default="https://nominatim.openstreetmap.org", + ), # --- History kept ------------------------------------------------------------ PrefSpec( key="history_days", diff --git a/meshterm/persistence/db.py b/meshterm/persistence/db.py index 2bcaf1e..8bf77e8 100644 --- a/meshterm/persistence/db.py +++ b/meshterm/persistence/db.py @@ -12,7 +12,7 @@ import json import sqlite3 from pathlib import Path -SCHEMA_VERSION = 22 +SCHEMA_VERSION = 23 _SCHEMA = """ CREATE TABLE IF NOT EXISTS schema_meta ( @@ -174,6 +174,21 @@ CREATE TABLE IF NOT EXISTS messages ( -- at connect, so it arrived some time before received_at ); +-- The place of a node: the named area nearest its location (refer to meshterm.core.places). +-- It is derived data, found from the map tiles or from Nominatim, and stored so that each +-- location is looked up one time only. The node moves past MOVE_KM, and it is found again. +-- Both parts NULL is a stored "the source answered, and there is no place here". +CREATE TABLE IF NOT EXISTS node_places ( + node TEXT PRIMARY KEY, -- 12-hex canonical id (as observations.node) + lat REAL NOT NULL, -- the location that the place was found for + lon REAL NOT NULL, + fine TEXT, -- the most precise name: a neighbourhood, a village, a peak + coarse TEXT, -- the town or city that holds it, or NULL + peak INTEGER NOT NULL DEFAULT 0, -- fine is a peak near its summit + source TEXT NOT NULL, -- 'tiles' or 'nominatim' + found_at TEXT NOT NULL -- UTC ISO-8601 +); + CREATE INDEX IF NOT EXISTS idx_traces_run ON traces(run_id); CREATE INDEX IF NOT EXISTS idx_trace_hops_trace ON trace_hops(trace_id); CREATE INDEX IF NOT EXISTS idx_tx_samples_run ON tx_samples(run_id); diff --git a/meshterm/persistence/repository.py b/meshterm/persistence/repository.py index d806741..e4c5190 100644 --- a/meshterm/persistence/repository.py +++ b/meshterm/persistence/repository.py @@ -29,6 +29,7 @@ from ..core.models import ( TxLevelResult, utcnow, ) +from ..core.places import Place from ..core.regions import SCOPED_ROUTES, raw_scope_body from ..core.secrets import loggable from . import db @@ -2732,6 +2733,81 @@ class Repository: ) self._conn.commit() + # -- places ----------------------------------------------------------------- + + def node_locations(self, *, since: str | None = None) -> dict[str, tuple[float, float, str]]: + """The newest location that each node advertised. + + The background job of :mod:`meshterm.services.places` reads it to find the nodes + whose place it must find. With ``since``, the query reads only the newer rows, by + the ``observed_at`` index. Thus a job that asks each minute pays only for the rows + that arrived in that minute. + + Args: + since: Read only the observations after this UTC ISO-8601 stamp. + + Returns: + For each node: ``(lat, lon, observed_at)`` of its newest located observation. + """ + sql = ( + "SELECT node, lat, lon, observed_at FROM observations " + "WHERE node IS NOT NULL AND lat IS NOT NULL AND lon IS NOT NULL AND kind != 'packet'" + ) + params: list[Any] = [] + if since is not None: + sql += " AND observed_at > ?" + params.append(since) + newest: dict[str, tuple[float, float, str]] = {} + for row in self._conn.execute(sql, params): + old = newest.get(row["node"]) + if old is None or row["observed_at"] >= old[2]: + newest[row["node"]] = (row["lat"], row["lon"], row["observed_at"]) + return newest + + def node_places(self) -> dict[str, tuple[Place, float, float]]: + """Each stored place, with the location that it was found for. + + Returns: + For each node id: ``(place, lat, lon)``. + """ + rows = self._conn.execute( + "SELECT node, lat, lon, fine, coarse, peak, source FROM node_places" + ).fetchall() + return { + row["node"]: ( + Place(row["fine"], row["coarse"], row["source"], peak=bool(row["peak"])), + row["lat"], + row["lon"], + ) + for row in rows + } + + def save_node_place(self, node: str, lat: float, lon: float, place: Place) -> None: + """Store the place of a node, found for this location (it replaces an older one). + + Args: + node: The 12-hex id of the node. + lat: The latitude that the place was found for. + lon: The longitude that the place was found for. + place: The place (an empty place is a stored "nothing here"). + """ + self._conn.execute( + "INSERT OR REPLACE INTO node_places " + "(node, lat, lon, fine, coarse, peak, source, found_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ( + node, + lat, + lon, + place.fine, + place.coarse, + int(place.peak), + place.source, + utcnow().isoformat(), + ), + ) + self._conn.commit() + @staticmethod def _row_to_chat(row: sqlite3.Row) -> ChatMessage: """Rebuild a :class:`ChatMessage` from a ``messages`` row.""" diff --git a/meshterm/services/basemap.py b/meshterm/services/basemap.py index d9ab46e..06661a5 100644 --- a/meshterm/services/basemap.py +++ b/meshterm/services/basemap.py @@ -164,6 +164,11 @@ _CA_BUNDLES = ( ) +def tls_context() -> ssl.SSLContext | None: + """The TLS context for each HTTPS request of MeshTerm (refer to :func:`_tls_context`).""" + return _tls_context() + + @lru_cache(maxsize=1) def _tls_context() -> ssl.SSLContext | None: """The TLS context for tile downloads, with a CA bundle if the default has none. @@ -444,6 +449,29 @@ class BasemapSource: self._memo.move_to_end((z, x, y)) return hit[0] + def raw_tile(self, z: int, x: int, y: int) -> bytes | None: + """The raw ``.pbf`` bytes of a tile, from the cache or the network. This method blocks. + + This method is for a reader of layers that the map does not draw: the places of the + nodes read ``place`` and ``mountain_peak`` (refer to + :mod:`meshterm.services.places`). The cache keeps whole tiles, so a tile that the + map downloaded already has these layers, and a tile that this method downloads is + ready for the map. A download goes through :meth:`_build`, so it obeys the same + rule: only bytes that decode are written. + + Returns: + The bytes, or ``None`` when the tile is not available (offline and not cached, + or a tile that the source answered as empty). + """ + path = self._tile_path(z, x, y) + cached = self._read_cached(path) + if cached is not None: + return cached + if (z, x, y) in self._blank: + return None + self._build(z, x, y) + return self._read_cached(path) + def warm(self, z: int, x: int, y: int) -> None: """Make a tile fast to load later: download, decode, and write it, but hold nothing. diff --git a/meshterm/services/places.py b/meshterm/services/places.py new file mode 100644 index 0000000..2ab21e0 --- /dev/null +++ b/meshterm/services/places.py @@ -0,0 +1,454 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The places of the nodes: what each screen reads, and the slow job that finds them. + +The place of a node is the named area nearest its location: ``Villeray, Montréal``, a +village, or a peak (refer to :mod:`meshterm.core.places` for the rules). This module has two +parts. + +**The read side** (:meth:`PlaceService.get`) answers from memory, so a screen can ask at each +paint. A place that the user typed (:class:`~meshterm.core.place_store.PlaceStore`) wins over +a found place. A found place comes from the ``node_places`` table, which this service reads +one time. :attr:`PlaceService.revision` increases at each change, so that a screen with a +render cache knows when to draw again. + +**The job** finds the places, one node at a time, in the background. Each minute, it reads +the locations that arrived since its last read (``observed_at`` is indexed, so this read is +cheap). A node gets a place when it has none, or when it moved more than +:data:`~meshterm.core.places.MOVE_KM` from where its place was found. The +``place_source`` preference sets the source: + +* ``tiles``: the OpenStreetMap vector tiles of the map (:class:`BasemapSource`). The tiles are + in the cache when the map showed the area, and the job downloads the others. A tile is + shared with the map, so the map is ready where the job went first. +* ``nominatim``: the Nominatim ``/reverse`` endpoint at ``nominatim_url``. Nominatim knows the + boundaries of each place, so it tells the place that *contains* the node. +* ``tiles_nominatim`` (the default): the tiles give the first answer at once, and Nominatim + corrects it later, one node at a time, when MeshTerm is online. A peak that the tiles found + near its summit stays, because Nominatim does not answer with peaks. + +**Nominatim has a strict usage policy**, and the job obeys it: a user agent that names the +app, a maximum of one request each second (the job waits :data:`NOMINATIM_INTERVAL_S`), and a +cache of each answer (the table). The job asks again only for a node that moved. A screen can +put a node at the head of the queue (:meth:`PlaceService.want`), for example the node page +that the user opened. +""" + +from __future__ import annotations + +import asyncio +import json +import threading +import time +import urllib.parse +import urllib.request +from collections import OrderedDict +from typing import TYPE_CHECKING + +from .. import __version__ +from ..core.geo import usable_fix +from ..core.places import ( + PLACE_LAYERS, + SOURCE_NOMINATIM, + SOURCE_TILES, + SOURCE_USER, + Place, + moved, + place_from_nominatim, + place_from_tiles, +) + +if TYPE_CHECKING: + from ..context import AppContext + from ..core.mvt import Layer + +#: The time (seconds) between two reads of the new locations. +RESCAN_S = 60.0 + +#: The time (seconds) before the first read, so that the job does not slow the start. +START_DELAY_S = 8.0 + +#: The time (seconds) between two lookups in the tiles. A lookup in cached tiles takes a +#: few milliseconds. A lookup that downloads takes the time of its downloads, and this pause +#: comes after it. +TILES_PACE_S = 0.5 + +#: The time (seconds) between two Nominatim requests. The policy permits one each second. +#: Two seconds keep MeshTerm well inside that limit. +NOMINATIM_INTERVAL_S = 2.0 + +#: The time (seconds) that the job waits after a source gave no answer (offline, a timeout, +#: a refusal). It then tries again. +RETRY_S = 300.0 + +#: The network timeout of one Nominatim request (seconds). +_TIMEOUT_S = 12.0 + +#: The number of decoded tiles that the job keeps in memory. Nodes near each other share +#: their tiles, and the nodes of one mesh are near each other. +_TILE_MEMO = 24 + +#: Sent with each Nominatim request. The policy asks that the user agent names the +#: application, so that the people who run the server can contact it. +_USER_AGENT = f"MeshTerm/{__version__} (+https://github.com/jpmartineau/MeshTerm; node places)" + + +def _norm(node: str) -> str: + """The 12-hex id of a node, from a key prefix or a full public key.""" + return node.lower().removeprefix("0x")[:12] + + +def ask_nominatim(base_url: str, lat: float, lon: float) -> object | None: + """Ask Nominatim for the address at a location. This function blocks. + + Args: + base_url: The root of the Nominatim server (the ``nominatim_url`` preference). + lat: The latitude. + lon: The longitude. + + Returns: + The parsed JSON answer, or ``None`` when the server gave no answer. + """ + from .basemap import tls_context + + query = urllib.parse.urlencode( + { + "format": "jsonv2", + "lat": f"{lat:.6f}", + "lon": f"{lon:.6f}", + # Zoom 16 (streets) is the first level whose address names the neighbourhood. + # Zoom 14 stops at the borough. MeshTerm reads only the area keys, never the road. + "zoom": 16, + "addressdetails": 1, + } + ) + url = f"{base_url.rstrip('/')}/reverse?{query}" + request = urllib.request.Request(url, headers={"User-Agent": _USER_AGENT}) + try: + with urllib.request.urlopen(request, timeout=_TIMEOUT_S, context=tls_context()) as resp: + return json.loads(resp.read()) + except Exception: # noqa: BLE001 - offline, a timeout, a refusal, or bad JSON is no answer + return None + + +class PlaceService: + """The places of the nodes: a read side from memory, and a background job that finds them.""" + + def __init__(self, ctx: AppContext) -> None: + """Make an idle service that is bound to an application context. + + Args: + ctx: The shared application context (the repository, the stores, the + preferences, and the tile source). + """ + self._ctx = ctx + self._found: dict[str, tuple[Place, float, float]] | None = None + self._revision = 0 + self._task: asyncio.Task | None = None + self._wake = asyncio.Event() + #: The newest ``observed_at`` that the job read, so that the next read is short. + self._since: str | None = None + #: The newest location of each node that the job knows. + self._locations: dict[str, tuple[float, float]] = {} + #: The nodes that a screen asked for, first in the queue. + self._wanted: list[str] = [] + #: The monotonic time before which each source stays silent after it gave no answer, + #: and the same time for each node that got no answer. Thus one node that a source + #: cannot answer does not stop the queue. + self._node_after: dict[str, float] = {} + self._tiles_after = 0.0 + self._nominatim_after = 0.0 + self._scan_after = 0.0 + self._tile_memo: OrderedDict[tuple[int, int, int], list[Layer]] = OrderedDict() + self._tile_lock = threading.Lock() + + # -- the read side ---------------------------------------------------------- + + @property + def revision(self) -> int: + """A number that increases each time a place changes.""" + return self._revision + + def _stored(self) -> dict[str, tuple[Place, float, float]]: + """The found places, read from the database at the first use.""" + if self._found is None: + try: + self._found = self._ctx.repo.node_places() + except Exception as exc: # noqa: BLE001 - no places is better than no screen + self._ctx.log.debug("places: read failed: %s", exc) + self._found = {} + return self._found + + def get(self, node: str | None) -> Place | None: + """The place of a node, or ``None`` when MeshTerm does not know one. + + Args: + node: The key prefix or the full public key of the node. + + Returns: + The place that the user typed, else the found place. An empty found place (the + source answered, but there is no place near) is ``None`` here too. + """ + if not node: + return None + key = _norm(node) + typed = self._ctx.place_store.get(key) + if typed: + return Place(typed, None, SOURCE_USER) + hit = self._stored().get(key) + if hit is None or hit[0].empty: + return None + return hit[0] + + def typed(self, node: str | None) -> str | None: + """The place that the user typed for a node, or ``None``.""" + return self._ctx.place_store.get(node) if node else None + + def set_typed(self, node: str, name: str | None) -> None: + """Type a place for a node, or clear the typed place with ``None`` or blank text.""" + self._ctx.place_store.set(node, name, when=int(time.time())) + self._revision += 1 + + def want(self, nodes: list[str | None]) -> None: + """Put these nodes at the head of the queue of the job (a screen shows them now). + + A node that already has its place from the current source is not queued again. + """ + added = False + for node in nodes: + if not node: + continue + key = _norm(node) + if key not in self._wanted: + self._wanted.append(key) + added = True + if added: + self._wake.set() + + # -- the job ---------------------------------------------------------------- + + @property + def active(self) -> bool: + """True while the job runs.""" + return self._task is not None and not self._task.done() + + async def start(self) -> None: + """Start the job. Idempotent. The first read waits :data:`START_DELAY_S`.""" + if self.active: + return + self._scan_after = time.monotonic() + START_DELAY_S + self._task = asyncio.ensure_future(self._run()) + self._ctx.log.info("place finder started") + + async def stop(self) -> None: + """Stop the job. Idempotent.""" + task, self._task = self._task, None + if task is not None: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + self._ctx.log.info("place finder stopped") + + async def aclose(self) -> None: + """Stop the job at the end of the session (an alias for :meth:`stop`).""" + await self.stop() + + async def _run(self) -> None: + """Do one step, then wait the time that the step returned, or until a screen asks.""" + while True: + delay = RESCAN_S + try: + delay = await self.step() + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 - one bad node must not stop the job + self._ctx.log.debug("places: step failed: %s", exc) + self._wake.clear() + try: + await asyncio.wait_for(self._wake.wait(), timeout=delay) + except asyncio.TimeoutError: + pass + + def _scan(self) -> None: + """Read the locations that arrived since the last read. + + The heard adverts give most locations. The contact table of the device adds the + nodes that advertised before MeshTerm listened, and the self-info adds our node. + Both come from the cache of the device state, never from the device: the job never + sends a command. + """ + fresh = self._ctx.repo.node_locations(since=self._since) + for node, (lat, lon, stamp) in fresh.items(): + if self._since is None or stamp > self._since: + self._since = stamp + if usable_fix(lat, lon): + self._locations[node] = (lat, lon) + devstate = getattr(self._ctx, "devstate", None) + if devstate is None: + return + for contact in devstate.peek_contacts() or []: + ident = (contact.public_key or contact.key_prefix or "").lower().removeprefix("0x") + self._locate(ident[:12], contact.lat, contact.lon, newer=False) + info = devstate.peek_self_info() or {} + ident = str(info.get("public_key") or "").lower().removeprefix("0x")[:12] + self._locate(ident, info.get("adv_lat"), info.get("adv_lon"), newer=True) + + def _locate(self, node: str, lat: object, lon: object, *, newer: bool) -> None: + """Add a location that did not come from the heard adverts. + + Args: + node: The 12-hex id of the node (blank to skip). + lat: The latitude, in any form that ``float`` accepts. + lon: The longitude, in any form that ``float`` accepts. + newer: Whether this location replaces a heard one. Our own fix does, because it + is the current setting. The location in a contact is only the newest that + the device kept, so a heard advert is as new or newer. + """ + if not node: + return + try: + flat, flon = float(lat), float(lon) # type: ignore[arg-type] + except (TypeError, ValueError): + return + if usable_fix(flat, flon) and (newer or node not in self._locations): + self._locations[node] = (flat, flon) + + def want_at(self, node: str | None, lat: float | None, lon: float | None) -> None: + """Put a node at the head of the queue with a location that a screen knows. + + The node page uses it: the page knows the location of the node (also our own node, + which MeshTerm never hears), and the user looks at it now. + """ + if not node or lat is None or lon is None: + return + key = _norm(node) + self._locate(key, lat, lon, newer=True) + self.want([key]) + + def _step_for(self, node: str) -> str | None: + """The source that the node needs next, or ``None`` when its place is current.""" + location = self._locations.get(node) + if location is None: + return None + lat, lon = location + mode = self._ctx.preferences.place_source + stored = self._stored().get(node) + first = SOURCE_NOMINATIM if mode == "nominatim" else SOURCE_TILES + if stored is None or moved(stored[1], stored[2], lat, lon): + return first + place = stored[0] + if mode != "tiles" and place.source == SOURCE_TILES and not place.peak: + return SOURCE_NOMINATIM + return None + + def _next(self, now: float) -> tuple[str, str] | None: + """The next ``(node, source)`` to look up, the wanted nodes first.""" + ready = { + SOURCE_TILES: now >= self._tiles_after, + SOURCE_NOMINATIM: now >= self._nominatim_after, + } + while self._wanted: + node = self._wanted[0] + source = self._step_for(node) + if source is None or not ready[source] or now < self._node_after.get(node, 0.0): + self._wanted.pop(0) + continue + return node, source + # The tiles first for all the nodes: they are fast, and they give each node an + # answer. Nominatim then corrects them at its own slow pace. + for wanted_source in (SOURCE_TILES, SOURCE_NOMINATIM): + if not ready[wanted_source]: + continue + for node in self._locations: + if now < self._node_after.get(node, 0.0): + continue + if self._step_for(node) == wanted_source: + return node, wanted_source + return None + + async def step(self) -> float: + """Do one lookup, and return the time until the next step. + + The job calls this method. A test can call it directly. + """ + now = time.monotonic() + if now >= self._scan_after: + self._scan_after = now + RESCAN_S + self._scan() + work = self._next(now) + if work is None: + soonest = min( + (t for t in (self._tiles_after, self._nominatim_after) if t > now), + default=self._scan_after, + ) + return max(1.0, min(self._scan_after, soonest) - now) + node, source = work + lat, lon = self._locations[node] + if source == SOURCE_TILES: + place = await asyncio.to_thread(place_from_tiles, lat, lon, self._load_tile) + if place is None: + self._no_answer(node, SOURCE_TILES) + return 1.0 + self._save(node, lat, lon, place) + return TILES_PACE_S + url = self._ctx.preferences.nominatim_url + self._nominatim_after = time.monotonic() + NOMINATIM_INTERVAL_S + doc = await asyncio.to_thread(ask_nominatim, url, lat, lon) + place = place_from_nominatim(doc) + if place is None: + self._no_answer(node, SOURCE_NOMINATIM) + return 1.0 + stored = self._stored().get(node) + if place.empty and stored is not None and not stored[0].empty: + # Nominatim found nothing (the sea, a lake). The answer of the tiles stays, and + # it is marked as checked, so that the job does not ask again. + old = stored[0] + place = Place(old.fine, old.coarse, SOURCE_NOMINATIM, peak=old.peak) + self._save(node, lat, lon, place) + return NOMINATIM_INTERVAL_S + + def _no_answer(self, node: str, source: str) -> None: + """Make a source and a node wait :data:`RETRY_S` after the source gave no answer. + + Offline, each node fails in the same way, so the source waits. But only the node + waits as long, so that a source that cannot answer for this one node answers for the + next one at the end of a short pause. + """ + now = time.monotonic() + self._node_after[node] = now + RETRY_S + pause = now + 30.0 + if source == SOURCE_TILES: + self._tiles_after = max(self._tiles_after, pause) + else: + self._nominatim_after = max(self._nominatim_after, pause) + + def _save(self, node: str, lat: float, lon: float, place: Place) -> None: + """Store a found place, and tell the screens that a place changed.""" + self._ctx.repo.save_node_place(node, lat, lon, place) + self._stored()[node] = (place, lat, lon) + self._revision += 1 + self._ctx.log.debug("places: %s -> %r (%s)", node, place.label, place.source) + + def _load_tile(self, z: int, x: int, y: int) -> list[Layer] | None: + """The ``place`` and ``mountain_peak`` layers of a tile. This method blocks.""" + from ..core.mvt import decode_tile + + key = (z, x, y) + with self._tile_lock: + hit = self._tile_memo.get(key) + if hit is not None: + self._tile_memo.move_to_end(key) + return hit + raw = self._ctx.basemap_source.raw_tile(z, x, y) + if raw is None: + return None + try: + layers = decode_tile(raw, layers=PLACE_LAYERS) + except Exception: # noqa: BLE001 - a corrupt tile is no answer + return None + with self._tile_lock: + self._tile_memo[key] = layers + while len(self._tile_memo) > _TILE_MEMO: + self._tile_memo.popitem(last=False) + return layers diff --git a/meshterm/ui/menu.py b/meshterm/ui/menu.py index 1615a9f..faa237f 100644 --- a/meshterm/ui/menu.py +++ b/meshterm/ui/menu.py @@ -1086,6 +1086,9 @@ async def _resume_monitor(ctx: AppContext) -> Exception | None: # reads, and it skips silently without a device, so it is also safe to run from the # start. await ctx.battery.start() + # The place finder reads only stored locations and the network (tiles, Nominatim), never + # the device. Its first read waits some seconds, so it does not slow the start. + await ctx.places.start() if not ctx.settings.connect_on_start: return None failure: Exception | None = None diff --git a/tests/test_places.py b/tests/test_places.py new file mode 100644 index 0000000..e825ff9 --- /dev/null +++ b/tests/test_places.py @@ -0,0 +1,342 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The place of a node: the rules, the typed places, the stored places, and the job. + +The tiles here are made in the test: each place is a point feature at a known location, so +a test states what the map holds and what the answer must be. No test reads the network. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +from datetime import datetime, timedelta, timezone +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from meshterm.core.geo import TILE_PX, lonlat_to_world +from meshterm.core.models import Observation +from meshterm.core.mvt import GEOM_POINT, Feature, Layer +from meshterm.core.place_store import PlaceStore +from meshterm.core.places import ( + SOURCE_NOMINATIM, + SOURCE_TILES, + SOURCE_USER, + Place, + moved, + place_from_nominatim, + place_from_tiles, +) +from meshterm.core.preferences import Preferences +from meshterm.persistence.repository import Repository +from meshterm.services import places as places_service +from meshterm.services.places import PlaceService + +_EXTENT = 4096 + +#: A point in a city, and the offsets (km) that the tests put places at. +_LAT, _LON = 45.5440, -73.6150 +_KM_LAT = 1 / 111.2 # degrees of latitude in one km + + +def _north(km: float) -> tuple[float, float]: + """A location ``km`` north of the test point.""" + return _LAT + km * _KM_LAT, _LON + + +class _Map: + """Tiles made from named points: each point goes into its tile at each zoom.""" + + def __init__(self) -> None: + self.points: list[tuple[str, str, str, float, float, int]] = [] + self.missing: set[tuple[int, int, int]] = set() + self.loads: list[tuple[int, int, int]] = [] + + def add(self, layer: str, cls: str, name: str, at: tuple[float, float], rank: int = 10) -> None: + self.points.append((layer, cls, name, at[0], at[1], rank)) + + def load(self, z: int, x: int, y: int) -> list[Layer] | None: + self.loads.append((z, x, y)) + if (z, x, y) in self.missing: + return None + layers = {name: Layer(name, _EXTENT) for name in ("place", "mountain_peak")} + for layer, cls, name, lat, lon, rank in self.points: + wx, wy = lonlat_to_world(lat, lon, z) + if int(wx // TILE_PX) != x or int(wy // TILE_PX) != y: + continue + px = int((wx / TILE_PX - x) * _EXTENT) + py = int((wy / TILE_PX - y) * _EXTENT) + tags = {"class": cls, "name": name, "rank": rank} + layers[layer].features.append(Feature(GEOM_POINT, [[(px, py)]], tags)) + return list(layers.values()) + + +# -- the rules ---------------------------------------------------------------- + + +def test_a_neighbourhood_gets_the_city_that_holds_it() -> None: + world = _Map() + world.add("place", "neighbourhood", "Villeray", _north(0.5)) + world.add("place", "city", "Montréal", _north(-6.0), rank=3) + place = place_from_tiles(_LAT, _LON, world.load) + assert place == Place("Villeray", "Montréal", SOURCE_TILES) + assert place.label == "Villeray, Montréal" + assert place.short == "Villeray" + + +def test_the_most_specific_area_wins_over_a_nearer_borough() -> None: + world = _Map() + world.add("place", "suburb", "Villeray–Saint-Michel–Parc-Extension", _north(0.2)) + world.add("place", "neighbourhood", "Villeray", _north(1.0)) + world.add("place", "city", "Montréal", _north(-6.0), rank=3) + assert place_from_tiles(_LAT, _LON, world.load).fine == "Villeray" + + +def test_a_large_city_wins_over_a_nearer_small_one() -> None: + """The rank of a label weighs the distance: a node in Ahuntsic is in Montréal.""" + world = _Map() + world.add("place", "neighbourhood", "Ahuntsic", _north(0.3)) + world.add("place", "city", "Laval", _north(4.4), rank=11) + world.add("place", "city", "Montréal", _north(-9.4), rank=3) + assert place_from_tiles(_LAT, _LON, world.load).coarse == "Montréal" + + +def test_a_peak_near_the_summit_wins() -> None: + world = _Map() + world.add("mountain_peak", "peak", "Mont-Royal", _north(0.2)) + world.add("place", "neighbourhood", "Villeray", _north(0.5)) + world.add("place", "city", "Montréal", _north(-6.0), rank=3) + place = place_from_tiles(_LAT, _LON, world.load) + assert place == Place("Mont-Royal", None, SOURCE_TILES, peak=True) + + +def test_a_peak_past_its_radius_does_not_win() -> None: + world = _Map() + world.add("mountain_peak", "peak", "Mont-Royal", _north(0.6)) + world.add("place", "neighbourhood", "Villeray", _north(0.5)) + world.add("place", "city", "Montréal", _north(-6.0), rank=3) + assert place_from_tiles(_LAT, _LON, world.load).fine == "Villeray" + + +def test_a_village_stands_alone_and_needs_no_fine_tiles() -> None: + world = _Map() + world.add("place", "village", "Val-David", _north(1.5)) + place = place_from_tiles(_LAT, _LON, world.load) + assert place == Place("Val-David", None, SOURCE_TILES) + assert {z for z, _, _ in world.loads} == {11} # the country costs only z11 tiles + + +def test_a_node_far_from_all_places_gets_the_nearest_settlement() -> None: + world = _Map() + world.add("place", "town", "Mont-Laurier", _north(30.0)) + assert place_from_tiles(_LAT, _LON, world.load).fine == "Mont-Laurier" + + +def test_no_place_near_is_an_empty_answer() -> None: + place = place_from_tiles(_LAT, _LON, _Map().load) + assert place is not None and place.empty + assert place.label == "" and place.short == "" + + +def test_a_missing_home_tile_is_no_answer() -> None: + world = _Map() + world.add("place", "village", "Val-David", _north(1.5)) + wx, wy = lonlat_to_world(_LAT, _LON, 11) + world.missing.add((11, int(wx // TILE_PX), int(wy // TILE_PX))) + assert place_from_tiles(_LAT, _LON, world.load) is None + + +def test_a_bad_location_is_no_answer() -> None: + assert place_from_tiles(0.0, 0.0, _Map().load) is None + assert place_from_tiles(-97.0, -1042.0, _Map().load) is None + + +def test_a_label_names_a_city_one_time() -> None: + assert Place("Montréal", "Montréal", SOURCE_TILES).label == "Montréal" + assert Place(None, "Laval", SOURCE_TILES).short == "Laval" + + +def test_moved_is_past_half_a_km() -> None: + assert not moved(_LAT, _LON, *_north(0.4)) + assert moved(_LAT, _LON, *_north(0.6)) + + +def test_nominatim_gives_fine_and_coarse() -> None: + doc = {"address": {"neighbourhood": "Villeray", "suburb": "VSP", "city": "Montréal"}} + assert place_from_nominatim(doc) == Place("Villeray", "Montréal", SOURCE_NOMINATIM) + + +def test_nominatim_town_alone() -> None: + doc = {"address": {"village": "Val-David", "county": "Les Laurentides"}} + assert place_from_nominatim(doc) == Place("Val-David", None, SOURCE_NOMINATIM) + + +def test_nominatim_error_is_an_empty_answer_and_junk_is_none() -> None: + assert place_from_nominatim({"error": "Unable to geocode"}).empty + assert place_from_nominatim(None) is None + assert place_from_nominatim({"nothing": 1}) is None + + +# -- the typed places --------------------------------------------------------- + + +def test_a_typed_place_is_kept_and_cleared(tmp_path: Path) -> None: + path = tmp_path / "places.json" + store = PlaceStore(path) + store.set("A1B2C3D4E5F6AABB", "Toit du stade", when=100) + assert PlaceStore(path).get("a1b2c3d4e5f6") == "Toit du stade" + store.set("a1b2c3d4e5f6", " ", when=200) + assert PlaceStore(path).get("a1b2c3d4e5f6") is None + + +def test_a_field_no_record_knows_is_gone_at_the_next_write(tmp_path: Path) -> None: + path = tmp_path / "places.json" + path.write_text( + json.dumps({"places": {"a1b2c3d4e5f6": {"name": "X", "set_at": 1, "old": 2}}}), + encoding="utf-8", + ) + store = PlaceStore(path) + store.set("b1b2c3d4e5f6", "Y", when=3) + saved = json.loads(path.read_text(encoding="utf-8")) + assert saved["places"]["a1b2c3d4e5f6"] == {"name": "X", "set_at": 1} + + +# -- the stored places -------------------------------------------------------- + + +def _heard(repo: Repository, node: str, lat: float, lon: float, minutes_ago: int) -> None: + run = repo.start_run("monitor", {}, None) + when = datetime.now(timezone.utc) - timedelta(minutes=minutes_ago) + repo.record_observation(run, Observation(node=node, lat=lat, lon=lon, observed_at=when)) + + +def test_node_locations_gives_the_newest(tmp_path: Path) -> None: + repo = Repository(tmp_path / "p.db") + _heard(repo, "aaaaaaaaaaaa", 45.0, -73.0, minutes_ago=30) + _heard(repo, "aaaaaaaaaaaa", 46.0, -74.0, minutes_ago=5) + _heard(repo, "bbbbbbbbbbbb", 47.0, -75.0, minutes_ago=10) + locations = repo.node_locations() + assert locations["aaaaaaaaaaaa"][:2] == (46.0, -74.0) + stamp = locations["bbbbbbbbbbbb"][2] + assert set(repo.node_locations(since=stamp)) == {"aaaaaaaaaaaa"} + + +def test_a_stored_place_reads_back(tmp_path: Path) -> None: + repo = Repository(tmp_path / "p.db") + repo.save_node_place("aaaaaaaaaaaa", 45.0, -73.0, Place("Mont-Royal", None, "tiles", True)) + repo.save_node_place("bbbbbbbbbbbb", 46.0, -74.0, Place(None, None, "nominatim")) + stored = repo.node_places() + assert stored["aaaaaaaaaaaa"] == (Place("Mont-Royal", None, "tiles", True), 45.0, -73.0) + assert stored["bbbbbbbbbbbb"][0].empty + + +# -- the service -------------------------------------------------------------- + + +def _service(tmp_path: Path, source: str = "tiles_nominatim") -> tuple[PlaceService, Repository]: + repo = Repository(tmp_path / "p.db") + ctx = SimpleNamespace( + repo=repo, + place_store=PlaceStore(tmp_path / "places.json"), + preferences=Preferences(values={"place_source": source}), + log=logging.getLogger("test"), + ) + return PlaceService(ctx), repo # type: ignore[arg-type] + + +def _run(coro): # noqa: ANN001, ANN202 - a small helper for the async steps + return asyncio.run(coro) + + +def test_a_typed_place_wins_and_an_empty_place_is_none(tmp_path: Path) -> None: + service, repo = _service(tmp_path) + repo.save_node_place("aaaaaaaaaaaa", 45.0, -73.0, Place("Villeray", "Montréal", "tiles")) + repo.save_node_place("bbbbbbbbbbbb", 45.0, -73.0, Place(None, None, "tiles")) + assert service.get("AAAAAAAAAAAA0000").label == "Villeray, Montréal" + assert service.get("bbbbbbbbbbbb") is None + before = service.revision + service.set_typed("aaaaaaaaaaaa", "Le toit") + assert service.revision > before + assert service.get("aaaaaaaaaaaa") == Place("Le toit", None, SOURCE_USER) + + +def test_the_job_uses_the_tiles_then_nominatim( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + service, repo = _service(tmp_path) + _heard(repo, "aaaaaaaaaaaa", _LAT, _LON, minutes_ago=1) + world = _Map() + world.add("place", "neighbourhood", "Ahuntsic", _north(0.3)) + world.add("place", "city", "Laval", _north(4.4), rank=3) + monkeypatch.setattr(service, "_load_tile", world.load) + service._scan_after = 0.0 + _run(service.step()) + assert service.get("aaaaaaaaaaaa").label == "Ahuntsic, Laval" + asked: list[tuple[float, float]] = [] + + def nominatim(url: str, lat: float, lon: float) -> object: + asked.append((lat, lon)) + return {"address": {"neighbourhood": "Ahuntsic", "city": "Montréal"}} + + monkeypatch.setattr(places_service, "ask_nominatim", nominatim) + _run(service.step()) + assert service.get("aaaaaaaaaaaa") == Place("Ahuntsic", "Montréal", SOURCE_NOMINATIM) + assert len(asked) == 1 + _run(service.step()) + assert len(asked) == 1 # one answer for each location: never asked again + + +def test_tiles_only_never_asks_nominatim(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + service, repo = _service(tmp_path, source="tiles") + _heard(repo, "aaaaaaaaaaaa", _LAT, _LON, minutes_ago=1) + world = _Map() + world.add("place", "village", "Val-David", _north(1.0)) + monkeypatch.setattr(service, "_load_tile", world.load) + monkeypatch.setattr(places_service, "ask_nominatim", lambda *a: pytest.fail("asked")) + service._scan_after = 0.0 + _run(service.step()) + _run(service.step()) + assert service.get("aaaaaaaaaaaa").fine == "Val-David" + + +def test_a_peak_stays_when_nominatim_could_correct( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + service, repo = _service(tmp_path) + _heard(repo, "aaaaaaaaaaaa", _LAT, _LON, minutes_ago=1) + repo.save_node_place("aaaaaaaaaaaa", _LAT, _LON, Place("Mont-Royal", None, "tiles", True)) + monkeypatch.setattr(places_service, "ask_nominatim", lambda *a: pytest.fail("asked")) + service._scan_after = 0.0 + _run(service.step()) + assert service.get("aaaaaaaaaaaa").peak + + +def test_a_node_that_moved_is_found_again(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + service, repo = _service(tmp_path, source="tiles") + repo.save_node_place("aaaaaaaaaaaa", *_north(-5.0), Place("Old", None, "tiles")) + _heard(repo, "aaaaaaaaaaaa", _LAT, _LON, minutes_ago=1) + world = _Map() + world.add("place", "village", "New", _north(1.0)) + monkeypatch.setattr(service, "_load_tile", world.load) + service._scan_after = 0.0 + _run(service.step()) + assert service.get("aaaaaaaaaaaa").fine == "New" + + +def test_no_answer_makes_the_node_wait(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + service, repo = _service(tmp_path, source="tiles") + _heard(repo, "aaaaaaaaaaaa", _LAT, _LON, minutes_ago=1) + calls: list[int] = [] + + def offline(z: int, x: int, y: int) -> None: + calls.append(z) + return None + + monkeypatch.setattr(service, "_load_tile", offline) + service._scan_after = 0.0 + _run(service.step()) + _run(service.step()) + assert len(calls) == 1 # the second step does not ask again at once + assert service.get("aaaaaaaaaaaa") is None