Nodes get a place: the named area nearest their location

A background job finds the place of each node that advertises a location
(Villeray, Montréal; a village; a peak near its summit). The map tiles give
the first answer offline; Nominatim corrects it when online (place_source,
default tiles then Nominatim; nominatim_url). Places are stored in the new
node_places table, a typed place in places.json wins over a found one.

Co-Authored-By: Claude Haiku 5.5 <noreply@anthropic.com>
This commit is contained in:
Jean-Pierre MartineauandClaude Haiku 5.5 committed 2026-11-03 02:47:05 -05:00
1 parent 4ded134c05
commit 6c63697e21
12 files changed
+1496 -1

No files matched your search

+2
View File
@@ -82,6 +82,8 @@ them instead of hand-rolling:
| path | an ordered hop spec you compose or force (`a1,3d,…`) |
| route | the concrete node sequence a trace walked or will walk |
| via | prefix for a packet/message's relay chain |
| location | the coordinates a node advertises (latitude, longitude) — `Set location`, the location preview, ^L locate |
| place | the named area nearest a node's location: a neighbourhood, a village, a peak (`core/places.py`, `services/places.py`). Read as *fine, coarse* (`Villeray, Montréal`); a village, town, city or peak stands alone; a path chip shows the fine part. Found from the map tiles, then Nominatim (`place_source`); a place the user typed wins. Never "location name" |
| region | a named area a repeater relays floods for (`region put yul`); shown bare, never `#yul` — the `#` belongs to key derivation only |
| scope | the region a flood is limited to — a packet's, a channel's send scope, the device's default scope. A direct packet has none. MeshCore's sense and the only one: the menu's This node / Managed nodes / This app sections name *whose* a setting is, never a "scope" |
| unscoped | a plain flood, relayed by any repeater that allows the wildcard `*` |
+2
View File
@@ -184,6 +184,8 @@ and in quoted text.
| route | The sequence of nodes that a trace went through or will go through | |
| hop | One node in a path or a route | |
| via | The prefix for the relay chain of a packet or a message | |
| location | The coordinates (latitude and longitude) that a node advertises | coordinates (alone) |
| place | The named area nearest the location of a node: a neighbourhood, a village, a town, or a peak. It has a fine part and a coarse part (`Villeray, Montréal`) | location name, locality |
| region | A named area for which a repeater relays floods (`region put yul`). Shown without `#` | |
| scope | The region to which a flood is limited | |
| unscoped | A flood that each repeater relays when it permits the wildcard `*` | |
+25
View File
@@ -24,6 +24,7 @@ from .core.courier_store import CourierStore
from .core.device_store import DeviceStore
from .core.discovery import DiscoveredDevice, discover_devices
from .core.mute_store import MuteStore
from .core.place_store import PlaceStore
from .core.preferences import PREFERENCES_FILENAME, Preferences, report_dropped
from .core.preferences import install as install_preferences
from .core.region_store import RegionStore
@@ -46,6 +47,7 @@ if TYPE_CHECKING:
from .services.device_state import DeviceState
from .services.event_hub import EventHub
from .services.monitor_service import MonitorService
from .services.places import PlaceService
from .services.rooms import RoomService
from .services.watchtower import WatchtowerService
from .ui.surface import Ui
@@ -101,6 +103,9 @@ class AppContext:
room_store: The store for the rooms that this machine joined: the password of each
room and the access that it granted (if the caller does not inject it, the
default is ``<config_dir>/rooms.json``).
place_store: The store for the places that the user typed for nodes. A typed place
wins over a found place (if the caller does not inject it, the default is
``<config_dir>/places.json``).
mock: Whether the context uses the simulator device.
demo: Whether the simulator is the demo world of ``--demo``
(:mod:`meshterm.demo`), which implies ``mock``.
@@ -145,6 +150,7 @@ class AppContext:
settings_store: SettingsStore | None = None
contact_store: ContactStore | None = None
room_store: RoomStore | None = None
place_store: PlaceStore | None = None
profile: DeviceProfile | None = None
mock: bool = False
demo: bool = False
@@ -189,6 +195,7 @@ class AppContext:
_clock_sync: ClockSync | None = field(default=None, init=False, repr=False)
_courier: CourierService | None = field(default=None, init=False, repr=False)
_battery: BatteryService | None = field(default=None, init=False, repr=False)
_places: PlaceService | None = field(default=None, init=False, repr=False)
_devstate: DeviceState | None = field(default=None, init=False, repr=False)
_basemap_source: BasemapSource | None = field(default=None, init=False, repr=False)
_ui: Ui | None = field(default=None, init=False, repr=False)
@@ -225,6 +232,8 @@ class AppContext:
self.contact_store = ContactStore(self.settings.config_dir / "contacts.json")
if self.room_store is None:
self.room_store = RoomStore(self.settings.config_dir / "rooms.json")
if self.place_store is None:
self.place_store = PlaceStore(self.settings.config_dir / "places.json")
@property
def profile_name(self) -> str | None:
@@ -481,6 +490,20 @@ class AppContext:
self._courier = CourierService(self)
return self._courier
@property
def places(self) -> PlaceService:
"""Return the places of the nodes, and create the service at first use.
A screen reads a place from it at each paint (:meth:`PlaceService.get`). The
interactive session starts its background job with the other always-on services.
A scripted CLI run only reads the stored places, and never starts the job.
"""
if self._places is None:
from .services.places import PlaceService
self._places = PlaceService(self)
return self._places
@property
def battery(self) -> BatteryService:
"""Return the battery poller of the session, and create it at first use.
@@ -1047,6 +1070,8 @@ class AppContext:
await self._courier.aclose()
if self._battery is not None:
await self._battery.aclose()
if self._places is not None:
await self._places.aclose()
if self._watchtower is not None:
await self._watchtower.aclose()
if self._clock_sync is not None:
+137
View File
@@ -0,0 +1,137 @@
# SPDX-License-Identifier: Apache-2.0
"""The places that the user typed for nodes.
MeshTerm finds the place of a node from its location (refer to :mod:`meshterm.core.places`).
But the user can know better: a repeater on the roof of a known building, or a node whose
location is far from where it really is. The node page lets the user type a place for the
node, and this store keeps it. A typed place wins over each found place, on each screen.
The store finds a place by the key prefix of the node (12 hex digits, the same id as
``observations.node``). A place belongs to a node and not to a contact: a repeater in a path
is often no contact of the device. The file is ``<config_dir>/places.json``, global machine
state like the other choices of the user. After the first read, the store answers reads from
memory, because the Contacts list and each path line read it at each paint. A write occurs
only when the user changes a place, and the write is atomic.
"""
from __future__ import annotations
import json
from dataclasses import dataclass
from pathlib import Path
from typing import ClassVar
from .atomicwrite import write_atomically
def _norm(node: str) -> str:
"""The 12-hex id of a node, from a key prefix or a full public key."""
return node.lower().removeprefix("0x")[:12]
@dataclass(frozen=True, slots=True)
class TypedPlace:
"""The place that the user typed for one node.
Attributes:
name: The place, as the user typed it.
set_at: When the user typed it (seconds since the epoch).
"""
name: str
set_at: int
#: The names that left this record. A retired name is never used again with a new
#: meaning (refer to :data:`meshterm.core.preferences.RETIRED` for the reason).
RETIRED: ClassVar[frozenset[str]] = frozenset()
class PlaceStore:
"""Reads and writes the places that the user typed, from memory first."""
def __init__(self, path: Path) -> None:
"""Open the store on the location of a JSON file.
Args:
path: The path to the JSON state file. The store makes the file at the first
typed place.
"""
self._path = path
self._places: dict[str, TypedPlace] | None = None
@property
def _state(self) -> dict[str, TypedPlace]:
"""The typed places by node id, read from disk at the first access."""
if self._places is None:
self._places = self._load()
return self._places
def _load(self) -> dict[str, TypedPlace]:
"""Parse the file into typed records (empty for a bad or missing file)."""
try:
data = json.loads(self._path.read_text(encoding="utf-8"))
except (OSError, ValueError):
return {}
entries = data.get("places") if isinstance(data, dict) else None
if not isinstance(entries, dict):
return {}
places: dict[str, TypedPlace] = {}
for node, entry in entries.items():
if not isinstance(node, str) or not isinstance(entry, dict):
continue
name = entry.get("name")
if not isinstance(name, str) or not name.strip():
continue
try:
set_at = int(entry.get("set_at") or 0)
except (TypeError, ValueError):
set_at = 0
places[_norm(node)] = TypedPlace(name.strip(), set_at)
return places
def get(self, node: str | None) -> str | None:
"""The place that the user typed for a node, or ``None``.
Args:
node: The key prefix or the full public key of the node.
"""
if not node:
return None
typed = self._state.get(_norm(node))
return typed.name if typed else None
def all(self) -> dict[str, str]:
"""Each typed place by node id, as a copy that the caller can keep."""
return {node: typed.name for node, typed in self._state.items()}
def set(self, node: str, name: str | None, *, when: int) -> None:
"""Type a place for a node, or clear it with ``None`` or an empty name.
Args:
node: The key prefix or the full public key of the node.
name: The place. ``None`` or blank text clears the typed place, and the found
place shows again.
when: The time of the change (seconds since the epoch).
"""
key = _norm(node)
name = (name or "").strip()
current = self._state.get(key)
if not name:
if current is None:
return
del self._state[key]
else:
if current is not None and current.name == name:
return
self._state[key] = TypedPlace(name, when)
self._save()
def _save(self) -> None:
"""Store all the typed places atomically, from the records."""
data = {
"places": {
node: {"name": typed.name, "set_at": typed.set_at}
for node, typed in sorted(self._state.items())
}
}
write_atomically(self._path, json.dumps(data, indent=2, ensure_ascii=False))
+385
View File
@@ -0,0 +1,385 @@
# SPDX-License-Identifier: Apache-2.0
"""The place of a node: the named area nearest its location, in pure functions.
Many nodes have a name that says nothing about where they are: a call sign (``VE2XYZ``) or
the name of the hardware (``RAK-1W``). A node that advertises a location has a better answer
near it on the map. This module finds that answer, the **place** of the node: a
neighbourhood, a village, a town, or a peak. It has no I/O. A caller gives it the decoded
tiles (:func:`place_from_tiles`) or the document that Nominatim returned
(:func:`place_from_nominatim`), and it returns a :class:`Place`.
A place has two parts. The **fine** part is the most precise name: a neighbourhood in a
city, or the village itself in the country. The **coarse** part is the town or the city
that holds a fine part that is not itself a town: ``Villeray, Montréal``. A village, a town,
a city, and a peak stand alone. A path chip shows only the fine part, because a chip must be
short (:attr:`Place.short`).
**The tiles give points, not areas.** The OpenMapTiles ``place`` layer has one point for
each place, where its label goes. It has no boundary. Thus the tiles can tell the nearest
place, but not the place that contains a node. Near a border, the answer can be the
neighbouring place. :func:`place_from_tiles` limits this error: each class has a radius
past which its point no longer names the node, and the candidate that is nearest relative
to its own radius wins. Nominatim knows the boundaries, and when the preference permits,
its answer replaces the answer of the tiles (refer to :mod:`meshterm.services.places`).
**A peak wins near its summit.** Many repeaters are on a summit, and there the name of the
peak tells more than the name of the town below it. A named peak within
:data:`PEAK_RADIUS_KM` of the node is the place. Nominatim does not replace such a place.
"""
from __future__ import annotations
import math
from collections.abc import Callable, Iterable
from dataclasses import dataclass
from typing import TYPE_CHECKING
from .geo import TILE_PX, haversine_km, lonlat_to_world, usable_fix, world_to_lonlat
if TYPE_CHECKING:
from .mvt import Layer
#: Loads the decoded layers of one tile ``(z, x, y)``, or ``None`` when it has no answer.
TileLoader = Callable[[int, int, int], list[Layer] | None]
#: Where a place came from. ``tiles`` and ``nominatim`` are found. ``user`` is the place
#: that the user typed for a node, and it wins over a found place.
SOURCE_TILES = "tiles"
SOURCE_NOMINATIM = "nominatim"
SOURCE_USER = "user"
#: The layers of a tile that :func:`place_from_tiles` reads.
PLACE_LAYERS = frozenset({"place", "mountain_peak"})
#: The distance (km) within which a named peak is the place of a node.
PEAK_RADIUS_KM = 0.3
#: The distance (km) that a node must move before MeshTerm finds its place again.
MOVE_KM = 0.5
#: The classes that are a part of a town or a city, from the most specific, with the radius
#: (km) past which their point no longer names a node. The most specific class that has a
#: point within its radius gives the fine part: ``Villeray``, not the borough around it. A
#: hamlet is here and not with the settlements. Its name is often known only near it, so it
#: gets the name of its town as the coarse part. An island is last, because a large island
#: holds a full city.
_AREA_RADIUS_KM: dict[str, float] = {
"neighbourhood": 1.2,
"quarter": 1.8,
"hamlet": 1.5,
"suburb": 2.5,
"borough": 4.0,
"island": 1.5,
}
#: The settlements, with the radius (km) within which one is the fine part, alone.
_SETTLEMENT_RADIUS_KM: dict[str, float] = {
"village": 2.5,
"town": 4.0,
"city": 6.0,
}
#: The weight of a settlement when MeshTerm chooses the coarse part. The distance is divided
#: by the weight, so a city a little farther away wins over a small town nearer. A node in a
#: district of a city is usually nearer to the point of a neighbouring town than to the
#: centre of its own city.
_SETTLEMENT_WEIGHT: dict[str, float] = {"village": 1.0, "town": 2.0, "city": 4.0}
#: The ``rank`` that MeshTerm uses for a settlement whose tile gives none. OpenMapTiles ranks
#: the labels of a tile from 1, the most important. A large city has a low rank. For a node
#: in Ahuntsic, Montréal (rank 3, 9.4 km) must win over Laval (rank 11, 4.4 km), and the
#: class weight alone cannot see that difference. Thus the rank multiplies the distance
#: (refer to :func:`_coarse`).
_DEFAULT_RANK = 10
#: The classes of ``mountain_peak`` that have a summit.
_PEAK_CLASSES = frozenset({"peak", "volcano"})
#: The zoom of the tiles for the settlements and the peaks, and how far (km) MeshTerm looks
#: in them. At z11, a tile holds cities, towns, villages, hamlets, suburbs, islands, and the
#: important peaks, at approximately 40 KB a tile.
_COARSE_ZOOM = 11
_COARSE_REACH_KM = 15.0
#: The zoom of the tiles for the parts of a town, and how far (km) MeshTerm looks in them.
#: Only z14 has the neighbourhoods, the quarters, and the boroughs. MeshTerm loads these
#: tiles only for a node that is in a town (refer to :func:`place_from_tiles`).
_FINE_ZOOM = 14
_FINE_REACH_KM = 2.0
#: The zoom and the reach (km) for a node far from all settlements at :data:`_COARSE_ZOOM`.
_FAR_ZOOM = 9
_FAR_REACH_KM = 50.0
@dataclass(frozen=True, slots=True)
class Place:
"""The place of a node.
Attributes:
fine: The most precise name (a neighbourhood, a village, a peak), or ``None``.
coarse: The town or the city that holds :attr:`fine`, or ``None`` when the fine
part stands alone.
source: :data:`SOURCE_TILES`, :data:`SOURCE_NOMINATIM`, or :data:`SOURCE_USER`.
peak: Whether :attr:`fine` is a peak near its summit. Nominatim does not replace
such a place.
"""
fine: str | None
coarse: str | None
source: str
peak: bool = False
@property
def empty(self) -> bool:
"""Whether the source answered, but found no place. MeshTerm stores this answer too."""
return not self.fine and not self.coarse
@property
def label(self) -> str:
"""The full name: ``fine, coarse``, or the one part that there is (``""`` if empty)."""
parts = [p for p in (self.fine, self.coarse) if p]
if len(parts) == 2 and parts[0] == parts[1]:
parts = parts[:1]
return ", ".join(parts)
@property
def short(self) -> str:
"""The name for a path chip: the fine part, else the coarse part (``""`` if empty)."""
return self.fine or self.coarse or ""
def moved(lat1: float, lon1: float, lat2: float, lon2: float) -> bool:
"""Whether a node moved far enough from where its place was found to find it again."""
return haversine_km(lat1, lon1, lat2, lon2) > MOVE_KM
@dataclass(frozen=True, slots=True)
class _Candidate:
"""One named point from a tile, with its class and its distance from the node."""
name: str
cls: str
km: float
rank: int = _DEFAULT_RANK
def _tiles_near(lat: float, lon: float, zoom: int, reach_km: float) -> list[tuple[int, int, int]]:
"""The tiles at ``zoom`` that a square of ``reach_km`` around the point touches.
The tile that holds the point comes first. If it has no answer, the caller has no answer.
"""
n = 2**zoom
cx, cy = lonlat_to_world(lat, lon, zoom)
# The length of one world pixel at this latitude, in km.
px_km = 2 * math.pi * 6371.0088 * math.cos(math.radians(lat)) / (TILE_PX * n)
reach_px = reach_km / max(px_km, 1e-9)
home = (int(cx // TILE_PX), int(cy // TILE_PX))
x0, x1 = int((cx - reach_px) // TILE_PX), int((cx + reach_px) // TILE_PX)
y0, y1 = int((cy - reach_px) // TILE_PX), int((cy + reach_px) // TILE_PX)
tiles = [(zoom, home[0] % n, home[1])]
for ty in range(max(y0, 0), min(y1, n - 1) + 1):
for tx in range(x0, x1 + 1):
key = (zoom, tx % n, ty)
if key not in tiles:
tiles.append(key)
return tiles
def _candidates(
lat: float,
lon: float,
zoom: int,
reach_km: float,
load: TileLoader,
*,
peaks: list[_Candidate],
) -> list[_Candidate] | None:
"""The named place points within ``reach_km`` of the node, in the tiles at ``zoom``.
The peaks go into ``peaks`` (each peak one time only, also when two tiles have it).
Returns:
The place candidates, or ``None`` when the tile that holds the node has no answer.
"""
found: dict[tuple[str, str], _Candidate] = {}
for i, (z, x, y) in enumerate(_tiles_near(lat, lon, zoom, reach_km)):
layers = load(z, x, y)
if layers is None:
if i == 0:
return None
continue
for layer in layers:
if layer.name not in PLACE_LAYERS:
continue
for feature in layer.features:
if not feature.rings or not feature.rings[0]:
continue
name = feature.name
if not name:
continue
px, py = feature.rings[0][0]
wx = (x + px / layer.extent) * TILE_PX
wy = (y + py / layer.extent) * TILE_PX
plat, plon = world_to_lonlat(wx, wy, z)
km = haversine_km(lat, lon, plat, plon)
if km > reach_km:
continue
cls = str(feature.tags.get("class") or "")
if layer.name == "mountain_peak":
if cls in _PEAK_CLASSES and all(p.name != name for p in peaks):
peaks.append(_Candidate(name, cls, km))
continue
try:
rank = int(feature.tags.get("rank"))
except (TypeError, ValueError):
rank = _DEFAULT_RANK
key = (name, cls)
old = found.get(key)
if old is None or km < old.km:
found[key] = _Candidate(name, cls, km, min(rank, old.rank) if old else rank)
return list(found.values())
def _most_specific(candidates: Iterable[_Candidate]) -> _Candidate | None:
"""The nearest point of the most specific part of a town that is within its radius."""
within = [c for c in candidates if c.km <= _AREA_RADIUS_KM.get(c.cls, -1.0)]
for cls in _AREA_RADIUS_KM:
of_class = [c for c in within if c.cls == cls]
if of_class:
return min(of_class, key=lambda c: c.km)
return None
def _best(candidates: Iterable[_Candidate], radius: dict[str, float]) -> _Candidate | None:
"""The candidate that is nearest relative to the radius of its class, within that radius."""
best: tuple[float, _Candidate] | None = None
for c in candidates:
r = radius.get(c.cls)
if r is None or c.km > r:
continue
score = c.km / r
if best is None or score < best[0]:
best = (score, c)
return None if best is None else best[1]
def _coarse(candidates: Iterable[_Candidate]) -> _Candidate | None:
"""The settlement that holds the node most probably: the least weighted distance.
The weight is the class weight, and the rank of the label (plus two, so that rank 1
does not weigh three times rank 2) multiplies the distance.
"""
best: tuple[float, _Candidate] | None = None
for c in candidates:
weight = _SETTLEMENT_WEIGHT.get(c.cls)
if weight is None:
continue
score = c.km * (c.rank + 2) / weight
if best is None or score < best[0]:
best = (score, c)
return None if best is None else best[1]
def place_from_tiles(lat: float, lon: float, load: TileLoader) -> Place | None:
"""Find the place of a location in the vector tiles.
The steps:
1. Load the z11 tiles near the node: its settlements and the important peaks.
2. If the node is in a town (a settlement or a suburb near it), also load the z14 tiles
near it, for the neighbourhoods, the quarters, and the boroughs.
3. A named peak within :data:`PEAK_RADIUS_KM` is the place.
4. Else the nearest point of the most specific part of a town (within its radius) is
the fine part, and the settlement with the least weighted distance is the coarse
part.
5. Else the settlement that is nearest relative to its radius stands alone.
6. Else the nearest settlement (looked for up to :data:`_FAR_REACH_KM` away) stands
alone, or the nearest named peak.
Args:
lat: The latitude of the node.
lon: The longitude of the node.
load: Loads the decoded layers of one tile (``place`` and ``mountain_peak`` at
minimum), or returns ``None`` when it has no answer.
Returns:
The place, an empty place when the tiles answered but hold nothing near, or ``None``
when the tile that holds the node has no answer (offline and not cached). The
caller tries a ``None`` again later, and stores the other answers.
"""
if not usable_fix(lat, lon):
return None
peaks: list[_Candidate] = []
coarse = _candidates(lat, lon, _COARSE_ZOOM, _COARSE_REACH_KM, load, peaks=peaks)
if coarse is None:
return None
# A town or a city near the node (two of its radii), or a suburb around it: there are
# neighbourhoods to look for.
urban = any(
(c.cls in ("town", "city") and c.km <= 2 * _SETTLEMENT_RADIUS_KM[c.cls])
or (c.cls == "suburb" and c.km <= _AREA_RADIUS_KM["suburb"])
for c in coarse
)
fine: list[_Candidate] = []
if urban:
fine = _candidates(lat, lon, _FINE_ZOOM, _FINE_REACH_KM, load, peaks=peaks) or []
summit = min((p for p in peaks if p.km <= PEAK_RADIUS_KM), key=lambda p: p.km, default=None)
if summit is not None:
return Place(summit.name, None, SOURCE_TILES, peak=True)
everything = fine + coarse
area = _most_specific(everything)
if area is not None:
holder = _coarse(everything)
return Place(area.name, holder.name if holder else None, SOURCE_TILES)
settlement = _best(everything, _SETTLEMENT_RADIUS_KM) or _coarse(everything)
if settlement is None:
far = _candidates(lat, lon, _FAR_ZOOM, _FAR_REACH_KM, load, peaks=peaks) or []
settlement = _coarse(far)
if settlement is not None:
return Place(settlement.name, None, SOURCE_TILES)
nearest_peak = min(peaks, key=lambda p: p.km, default=None)
if nearest_peak is not None:
return Place(nearest_peak.name, None, SOURCE_TILES)
return Place(None, None, SOURCE_TILES)
#: The address keys of a Nominatim answer, from the most precise. The first key that is
#: present is the fine part.
_NOMINATIM_FINE = ("neighbourhood", "quarter", "suburb", "city_district", "borough", "hamlet")
#: The address keys of a settlement in a Nominatim answer, in the order of preference.
_NOMINATIM_COARSE = ("city", "town", "village", "municipality")
def place_from_nominatim(doc: object) -> Place | None:
"""Make a place from the document that the Nominatim ``/reverse`` endpoint returned.
Args:
doc: The parsed JSON (``format=jsonv2``, ``addressdetails=1``).
Returns:
The place. An answer with no address (``{"error": "Unable to geocode"}``, the sea)
is an empty place, because Nominatim answered. ``None`` for a document that is not
an answer at all.
"""
if not isinstance(doc, dict):
return None
address = doc.get("address")
if not isinstance(address, dict):
if "error" in doc:
return Place(None, None, SOURCE_NOMINATIM)
return None
def first(keys: tuple[str, ...]) -> str | None:
for key in keys:
value = address.get(key)
if isinstance(value, str) and value.strip():
return value.strip()
return None
fine = first(_NOMINATIM_FINE)
coarse = first(_NOMINATIM_COARSE)
if fine is None:
return Place(coarse, None, SOURCE_NOMINATIM)
return Place(fine, coarse if coarse != fine else None, SOURCE_NOMINATIM)
+26
View File
@@ -154,6 +154,15 @@ class PrefSpec:
#: sees, not the mechanism: an age is how long ago, and a timestamp is when.
_CLI_TIME_CHOICES: dict[str, str] = {"relative": "ages", "absolute": "timestamps"}
#: The sources of the place of a node (refer to :mod:`meshterm.services.places`). With
#: ``tiles_nominatim``, the map tiles give the first answer at once, and Nominatim corrects it
#: later, when MeshTerm is online, because only Nominatim knows the boundaries of a place.
_PLACE_SOURCE_CHOICES: dict[str, str] = {
"tiles_nominatim": "tiles, then Nominatim",
"tiles": "map tiles",
"nominatim": "Nominatim",
}
#: The choices for the silence rule. They come from the ring of the Watchtower itself, so
#: that the two lists always agree.
_SILENCE_CHOICES: dict[int, str] = {
@@ -396,6 +405,23 @@ PREFERENCES: tuple[PrefSpec, ...] = (
default="https://tiles.openfreemap.org/planet",
relaunch=True,
),
PrefSpec(
key="place_source",
label="Place names",
help="Where the place of a node comes from",
group="Map",
value_type="enum",
default="tiles_nominatim",
choices=_PLACE_SOURCE_CHOICES,
),
PrefSpec(
key="nominatim_url",
label="Nominatim",
help="The server that finds places by their boundaries",
group="Map",
value_type="str",
default="https://nominatim.openstreetmap.org",
),
# --- History kept ------------------------------------------------------------
PrefSpec(
key="history_days",
+16 -1
View File
@@ -12,7 +12,7 @@ import json
import sqlite3
from pathlib import Path
SCHEMA_VERSION = 22
SCHEMA_VERSION = 23
_SCHEMA = """
CREATE TABLE IF NOT EXISTS schema_meta (
@@ -174,6 +174,21 @@ CREATE TABLE IF NOT EXISTS messages (
-- at connect, so it arrived some time before received_at
);
-- The place of a node: the named area nearest its location (refer to meshterm.core.places).
-- It is derived data, found from the map tiles or from Nominatim, and stored so that each
-- location is looked up one time only. The node moves past MOVE_KM, and it is found again.
-- Both parts NULL is a stored "the source answered, and there is no place here".
CREATE TABLE IF NOT EXISTS node_places (
node TEXT PRIMARY KEY, -- 12-hex canonical id (as observations.node)
lat REAL NOT NULL, -- the location that the place was found for
lon REAL NOT NULL,
fine TEXT, -- the most precise name: a neighbourhood, a village, a peak
coarse TEXT, -- the town or city that holds it, or NULL
peak INTEGER NOT NULL DEFAULT 0, -- fine is a peak near its summit
source TEXT NOT NULL, -- 'tiles' or 'nominatim'
found_at TEXT NOT NULL -- UTC ISO-8601
);
CREATE INDEX IF NOT EXISTS idx_traces_run ON traces(run_id);
CREATE INDEX IF NOT EXISTS idx_trace_hops_trace ON trace_hops(trace_id);
CREATE INDEX IF NOT EXISTS idx_tx_samples_run ON tx_samples(run_id);
+76
View File
@@ -29,6 +29,7 @@ from ..core.models import (
TxLevelResult,
utcnow,
)
from ..core.places import Place
from ..core.regions import SCOPED_ROUTES, raw_scope_body
from ..core.secrets import loggable
from . import db
@@ -2732,6 +2733,81 @@ class Repository:
)
self._conn.commit()
# -- places -----------------------------------------------------------------
def node_locations(self, *, since: str | None = None) -> dict[str, tuple[float, float, str]]:
"""The newest location that each node advertised.
The background job of :mod:`meshterm.services.places` reads it to find the nodes
whose place it must find. With ``since``, the query reads only the newer rows, by
the ``observed_at`` index. Thus a job that asks each minute pays only for the rows
that arrived in that minute.
Args:
since: Read only the observations after this UTC ISO-8601 stamp.
Returns:
For each node: ``(lat, lon, observed_at)`` of its newest located observation.
"""
sql = (
"SELECT node, lat, lon, observed_at FROM observations "
"WHERE node IS NOT NULL AND lat IS NOT NULL AND lon IS NOT NULL AND kind != 'packet'"
)
params: list[Any] = []
if since is not None:
sql += " AND observed_at > ?"
params.append(since)
newest: dict[str, tuple[float, float, str]] = {}
for row in self._conn.execute(sql, params):
old = newest.get(row["node"])
if old is None or row["observed_at"] >= old[2]:
newest[row["node"]] = (row["lat"], row["lon"], row["observed_at"])
return newest
def node_places(self) -> dict[str, tuple[Place, float, float]]:
"""Each stored place, with the location that it was found for.
Returns:
For each node id: ``(place, lat, lon)``.
"""
rows = self._conn.execute(
"SELECT node, lat, lon, fine, coarse, peak, source FROM node_places"
).fetchall()
return {
row["node"]: (
Place(row["fine"], row["coarse"], row["source"], peak=bool(row["peak"])),
row["lat"],
row["lon"],
)
for row in rows
}
def save_node_place(self, node: str, lat: float, lon: float, place: Place) -> None:
"""Store the place of a node, found for this location (it replaces an older one).
Args:
node: The 12-hex id of the node.
lat: The latitude that the place was found for.
lon: The longitude that the place was found for.
place: The place (an empty place is a stored "nothing here").
"""
self._conn.execute(
"INSERT OR REPLACE INTO node_places "
"(node, lat, lon, fine, coarse, peak, source, found_at) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
(
node,
lat,
lon,
place.fine,
place.coarse,
int(place.peak),
place.source,
utcnow().isoformat(),
),
)
self._conn.commit()
@staticmethod
def _row_to_chat(row: sqlite3.Row) -> ChatMessage:
"""Rebuild a :class:`ChatMessage` from a ``messages`` row."""
+28
View File
@@ -164,6 +164,11 @@ _CA_BUNDLES = (
)
def tls_context() -> ssl.SSLContext | None:
"""The TLS context for each HTTPS request of MeshTerm (refer to :func:`_tls_context`)."""
return _tls_context()
@lru_cache(maxsize=1)
def _tls_context() -> ssl.SSLContext | None:
"""The TLS context for tile downloads, with a CA bundle if the default has none.
@@ -444,6 +449,29 @@ class BasemapSource:
self._memo.move_to_end((z, x, y))
return hit[0]
def raw_tile(self, z: int, x: int, y: int) -> bytes | None:
"""The raw ``.pbf`` bytes of a tile, from the cache or the network. This method blocks.
This method is for a reader of layers that the map does not draw: the places of the
nodes read ``place`` and ``mountain_peak`` (refer to
:mod:`meshterm.services.places`). The cache keeps whole tiles, so a tile that the
map downloaded already has these layers, and a tile that this method downloads is
ready for the map. A download goes through :meth:`_build`, so it obeys the same
rule: only bytes that decode are written.
Returns:
The bytes, or ``None`` when the tile is not available (offline and not cached,
or a tile that the source answered as empty).
"""
path = self._tile_path(z, x, y)
cached = self._read_cached(path)
if cached is not None:
return cached
if (z, x, y) in self._blank:
return None
self._build(z, x, y)
return self._read_cached(path)
def warm(self, z: int, x: int, y: int) -> None:
"""Make a tile fast to load later: download, decode, and write it, but hold nothing.
+454
View File
@@ -0,0 +1,454 @@
# SPDX-License-Identifier: Apache-2.0
"""The places of the nodes: what each screen reads, and the slow job that finds them.
The place of a node is the named area nearest its location: ``Villeray, Montréal``, a
village, or a peak (refer to :mod:`meshterm.core.places` for the rules). This module has two
parts.
**The read side** (:meth:`PlaceService.get`) answers from memory, so a screen can ask at each
paint. A place that the user typed (:class:`~meshterm.core.place_store.PlaceStore`) wins over
a found place. A found place comes from the ``node_places`` table, which this service reads
one time. :attr:`PlaceService.revision` increases at each change, so that a screen with a
render cache knows when to draw again.
**The job** finds the places, one node at a time, in the background. Each minute, it reads
the locations that arrived since its last read (``observed_at`` is indexed, so this read is
cheap). A node gets a place when it has none, or when it moved more than
:data:`~meshterm.core.places.MOVE_KM` from where its place was found. The
``place_source`` preference sets the source:
* ``tiles``: the OpenStreetMap vector tiles of the map (:class:`BasemapSource`). The tiles are
in the cache when the map showed the area, and the job downloads the others. A tile is
shared with the map, so the map is ready where the job went first.
* ``nominatim``: the Nominatim ``/reverse`` endpoint at ``nominatim_url``. Nominatim knows the
boundaries of each place, so it tells the place that *contains* the node.
* ``tiles_nominatim`` (the default): the tiles give the first answer at once, and Nominatim
corrects it later, one node at a time, when MeshTerm is online. A peak that the tiles found
near its summit stays, because Nominatim does not answer with peaks.
**Nominatim has a strict usage policy**, and the job obeys it: a user agent that names the
app, a maximum of one request each second (the job waits :data:`NOMINATIM_INTERVAL_S`), and a
cache of each answer (the table). The job asks again only for a node that moved. A screen can
put a node at the head of the queue (:meth:`PlaceService.want`), for example the node page
that the user opened.
"""
from __future__ import annotations
import asyncio
import json
import threading
import time
import urllib.parse
import urllib.request
from collections import OrderedDict
from typing import TYPE_CHECKING
from .. import __version__
from ..core.geo import usable_fix
from ..core.places import (
PLACE_LAYERS,
SOURCE_NOMINATIM,
SOURCE_TILES,
SOURCE_USER,
Place,
moved,
place_from_nominatim,
place_from_tiles,
)
if TYPE_CHECKING:
from ..context import AppContext
from ..core.mvt import Layer
#: The time (seconds) between two reads of the new locations.
RESCAN_S = 60.0
#: The time (seconds) before the first read, so that the job does not slow the start.
START_DELAY_S = 8.0
#: The time (seconds) between two lookups in the tiles. A lookup in cached tiles takes a
#: few milliseconds. A lookup that downloads takes the time of its downloads, and this pause
#: comes after it.
TILES_PACE_S = 0.5
#: The time (seconds) between two Nominatim requests. The policy permits one each second.
#: Two seconds keep MeshTerm well inside that limit.
NOMINATIM_INTERVAL_S = 2.0
#: The time (seconds) that the job waits after a source gave no answer (offline, a timeout,
#: a refusal). It then tries again.
RETRY_S = 300.0
#: The network timeout of one Nominatim request (seconds).
_TIMEOUT_S = 12.0
#: The number of decoded tiles that the job keeps in memory. Nodes near each other share
#: their tiles, and the nodes of one mesh are near each other.
_TILE_MEMO = 24
#: Sent with each Nominatim request. The policy asks that the user agent names the
#: application, so that the people who run the server can contact it.
_USER_AGENT = f"MeshTerm/{__version__} (+https://github.com/jpmartineau/MeshTerm; node places)"
def _norm(node: str) -> str:
"""The 12-hex id of a node, from a key prefix or a full public key."""
return node.lower().removeprefix("0x")[:12]
def ask_nominatim(base_url: str, lat: float, lon: float) -> object | None:
"""Ask Nominatim for the address at a location. This function blocks.
Args:
base_url: The root of the Nominatim server (the ``nominatim_url`` preference).
lat: The latitude.
lon: The longitude.
Returns:
The parsed JSON answer, or ``None`` when the server gave no answer.
"""
from .basemap import tls_context
query = urllib.parse.urlencode(
{
"format": "jsonv2",
"lat": f"{lat:.6f}",
"lon": f"{lon:.6f}",
# Zoom 16 (streets) is the first level whose address names the neighbourhood.
# Zoom 14 stops at the borough. MeshTerm reads only the area keys, never the road.
"zoom": 16,
"addressdetails": 1,
}
)
url = f"{base_url.rstrip('/')}/reverse?{query}"
request = urllib.request.Request(url, headers={"User-Agent": _USER_AGENT})
try:
with urllib.request.urlopen(request, timeout=_TIMEOUT_S, context=tls_context()) as resp:
return json.loads(resp.read())
except Exception: # noqa: BLE001 - offline, a timeout, a refusal, or bad JSON is no answer
return None
class PlaceService:
"""The places of the nodes: a read side from memory, and a background job that finds them."""
def __init__(self, ctx: AppContext) -> None:
"""Make an idle service that is bound to an application context.
Args:
ctx: The shared application context (the repository, the stores, the
preferences, and the tile source).
"""
self._ctx = ctx
self._found: dict[str, tuple[Place, float, float]] | None = None
self._revision = 0
self._task: asyncio.Task | None = None
self._wake = asyncio.Event()
#: The newest ``observed_at`` that the job read, so that the next read is short.
self._since: str | None = None
#: The newest location of each node that the job knows.
self._locations: dict[str, tuple[float, float]] = {}
#: The nodes that a screen asked for, first in the queue.
self._wanted: list[str] = []
#: The monotonic time before which each source stays silent after it gave no answer,
#: and the same time for each node that got no answer. Thus one node that a source
#: cannot answer does not stop the queue.
self._node_after: dict[str, float] = {}
self._tiles_after = 0.0
self._nominatim_after = 0.0
self._scan_after = 0.0
self._tile_memo: OrderedDict[tuple[int, int, int], list[Layer]] = OrderedDict()
self._tile_lock = threading.Lock()
# -- the read side ----------------------------------------------------------
@property
def revision(self) -> int:
"""A number that increases each time a place changes."""
return self._revision
def _stored(self) -> dict[str, tuple[Place, float, float]]:
"""The found places, read from the database at the first use."""
if self._found is None:
try:
self._found = self._ctx.repo.node_places()
except Exception as exc: # noqa: BLE001 - no places is better than no screen
self._ctx.log.debug("places: read failed: %s", exc)
self._found = {}
return self._found
def get(self, node: str | None) -> Place | None:
"""The place of a node, or ``None`` when MeshTerm does not know one.
Args:
node: The key prefix or the full public key of the node.
Returns:
The place that the user typed, else the found place. An empty found place (the
source answered, but there is no place near) is ``None`` here too.
"""
if not node:
return None
key = _norm(node)
typed = self._ctx.place_store.get(key)
if typed:
return Place(typed, None, SOURCE_USER)
hit = self._stored().get(key)
if hit is None or hit[0].empty:
return None
return hit[0]
def typed(self, node: str | None) -> str | None:
"""The place that the user typed for a node, or ``None``."""
return self._ctx.place_store.get(node) if node else None
def set_typed(self, node: str, name: str | None) -> None:
"""Type a place for a node, or clear the typed place with ``None`` or blank text."""
self._ctx.place_store.set(node, name, when=int(time.time()))
self._revision += 1
def want(self, nodes: list[str | None]) -> None:
"""Put these nodes at the head of the queue of the job (a screen shows them now).
A node that already has its place from the current source is not queued again.
"""
added = False
for node in nodes:
if not node:
continue
key = _norm(node)
if key not in self._wanted:
self._wanted.append(key)
added = True
if added:
self._wake.set()
# -- the job ----------------------------------------------------------------
@property
def active(self) -> bool:
"""True while the job runs."""
return self._task is not None and not self._task.done()
async def start(self) -> None:
"""Start the job. Idempotent. The first read waits :data:`START_DELAY_S`."""
if self.active:
return
self._scan_after = time.monotonic() + START_DELAY_S
self._task = asyncio.ensure_future(self._run())
self._ctx.log.info("place finder started")
async def stop(self) -> None:
"""Stop the job. Idempotent."""
task, self._task = self._task, None
if task is not None:
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
self._ctx.log.info("place finder stopped")
async def aclose(self) -> None:
"""Stop the job at the end of the session (an alias for :meth:`stop`)."""
await self.stop()
async def _run(self) -> None:
"""Do one step, then wait the time that the step returned, or until a screen asks."""
while True:
delay = RESCAN_S
try:
delay = await self.step()
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001 - one bad node must not stop the job
self._ctx.log.debug("places: step failed: %s", exc)
self._wake.clear()
try:
await asyncio.wait_for(self._wake.wait(), timeout=delay)
except asyncio.TimeoutError:
pass
def _scan(self) -> None:
"""Read the locations that arrived since the last read.
The heard adverts give most locations. The contact table of the device adds the
nodes that advertised before MeshTerm listened, and the self-info adds our node.
Both come from the cache of the device state, never from the device: the job never
sends a command.
"""
fresh = self._ctx.repo.node_locations(since=self._since)
for node, (lat, lon, stamp) in fresh.items():
if self._since is None or stamp > self._since:
self._since = stamp
if usable_fix(lat, lon):
self._locations[node] = (lat, lon)
devstate = getattr(self._ctx, "devstate", None)
if devstate is None:
return
for contact in devstate.peek_contacts() or []:
ident = (contact.public_key or contact.key_prefix or "").lower().removeprefix("0x")
self._locate(ident[:12], contact.lat, contact.lon, newer=False)
info = devstate.peek_self_info() or {}
ident = str(info.get("public_key") or "").lower().removeprefix("0x")[:12]
self._locate(ident, info.get("adv_lat"), info.get("adv_lon"), newer=True)
def _locate(self, node: str, lat: object, lon: object, *, newer: bool) -> None:
"""Add a location that did not come from the heard adverts.
Args:
node: The 12-hex id of the node (blank to skip).
lat: The latitude, in any form that ``float`` accepts.
lon: The longitude, in any form that ``float`` accepts.
newer: Whether this location replaces a heard one. Our own fix does, because it
is the current setting. The location in a contact is only the newest that
the device kept, so a heard advert is as new or newer.
"""
if not node:
return
try:
flat, flon = float(lat), float(lon) # type: ignore[arg-type]
except (TypeError, ValueError):
return
if usable_fix(flat, flon) and (newer or node not in self._locations):
self._locations[node] = (flat, flon)
def want_at(self, node: str | None, lat: float | None, lon: float | None) -> None:
"""Put a node at the head of the queue with a location that a screen knows.
The node page uses it: the page knows the location of the node (also our own node,
which MeshTerm never hears), and the user looks at it now.
"""
if not node or lat is None or lon is None:
return
key = _norm(node)
self._locate(key, lat, lon, newer=True)
self.want([key])
def _step_for(self, node: str) -> str | None:
"""The source that the node needs next, or ``None`` when its place is current."""
location = self._locations.get(node)
if location is None:
return None
lat, lon = location
mode = self._ctx.preferences.place_source
stored = self._stored().get(node)
first = SOURCE_NOMINATIM if mode == "nominatim" else SOURCE_TILES
if stored is None or moved(stored[1], stored[2], lat, lon):
return first
place = stored[0]
if mode != "tiles" and place.source == SOURCE_TILES and not place.peak:
return SOURCE_NOMINATIM
return None
def _next(self, now: float) -> tuple[str, str] | None:
"""The next ``(node, source)`` to look up, the wanted nodes first."""
ready = {
SOURCE_TILES: now >= self._tiles_after,
SOURCE_NOMINATIM: now >= self._nominatim_after,
}
while self._wanted:
node = self._wanted[0]
source = self._step_for(node)
if source is None or not ready[source] or now < self._node_after.get(node, 0.0):
self._wanted.pop(0)
continue
return node, source
# The tiles first for all the nodes: they are fast, and they give each node an
# answer. Nominatim then corrects them at its own slow pace.
for wanted_source in (SOURCE_TILES, SOURCE_NOMINATIM):
if not ready[wanted_source]:
continue
for node in self._locations:
if now < self._node_after.get(node, 0.0):
continue
if self._step_for(node) == wanted_source:
return node, wanted_source
return None
async def step(self) -> float:
"""Do one lookup, and return the time until the next step.
The job calls this method. A test can call it directly.
"""
now = time.monotonic()
if now >= self._scan_after:
self._scan_after = now + RESCAN_S
self._scan()
work = self._next(now)
if work is None:
soonest = min(
(t for t in (self._tiles_after, self._nominatim_after) if t > now),
default=self._scan_after,
)
return max(1.0, min(self._scan_after, soonest) - now)
node, source = work
lat, lon = self._locations[node]
if source == SOURCE_TILES:
place = await asyncio.to_thread(place_from_tiles, lat, lon, self._load_tile)
if place is None:
self._no_answer(node, SOURCE_TILES)
return 1.0
self._save(node, lat, lon, place)
return TILES_PACE_S
url = self._ctx.preferences.nominatim_url
self._nominatim_after = time.monotonic() + NOMINATIM_INTERVAL_S
doc = await asyncio.to_thread(ask_nominatim, url, lat, lon)
place = place_from_nominatim(doc)
if place is None:
self._no_answer(node, SOURCE_NOMINATIM)
return 1.0
stored = self._stored().get(node)
if place.empty and stored is not None and not stored[0].empty:
# Nominatim found nothing (the sea, a lake). The answer of the tiles stays, and
# it is marked as checked, so that the job does not ask again.
old = stored[0]
place = Place(old.fine, old.coarse, SOURCE_NOMINATIM, peak=old.peak)
self._save(node, lat, lon, place)
return NOMINATIM_INTERVAL_S
def _no_answer(self, node: str, source: str) -> None:
"""Make a source and a node wait :data:`RETRY_S` after the source gave no answer.
Offline, each node fails in the same way, so the source waits. But only the node
waits as long, so that a source that cannot answer for this one node answers for the
next one at the end of a short pause.
"""
now = time.monotonic()
self._node_after[node] = now + RETRY_S
pause = now + 30.0
if source == SOURCE_TILES:
self._tiles_after = max(self._tiles_after, pause)
else:
self._nominatim_after = max(self._nominatim_after, pause)
def _save(self, node: str, lat: float, lon: float, place: Place) -> None:
"""Store a found place, and tell the screens that a place changed."""
self._ctx.repo.save_node_place(node, lat, lon, place)
self._stored()[node] = (place, lat, lon)
self._revision += 1
self._ctx.log.debug("places: %s -> %r (%s)", node, place.label, place.source)
def _load_tile(self, z: int, x: int, y: int) -> list[Layer] | None:
"""The ``place`` and ``mountain_peak`` layers of a tile. This method blocks."""
from ..core.mvt import decode_tile
key = (z, x, y)
with self._tile_lock:
hit = self._tile_memo.get(key)
if hit is not None:
self._tile_memo.move_to_end(key)
return hit
raw = self._ctx.basemap_source.raw_tile(z, x, y)
if raw is None:
return None
try:
layers = decode_tile(raw, layers=PLACE_LAYERS)
except Exception: # noqa: BLE001 - a corrupt tile is no answer
return None
with self._tile_lock:
self._tile_memo[key] = layers
while len(self._tile_memo) > _TILE_MEMO:
self._tile_memo.popitem(last=False)
return layers
+3
View File
@@ -1086,6 +1086,9 @@ async def _resume_monitor(ctx: AppContext) -> Exception | None:
# reads, and it skips silently without a device, so it is also safe to run from the
# start.
await ctx.battery.start()
# The place finder reads only stored locations and the network (tiles, Nominatim), never
# the device. Its first read waits some seconds, so it does not slow the start.
await ctx.places.start()
if not ctx.settings.connect_on_start:
return None
failure: Exception | None = None
+342
View File
@@ -0,0 +1,342 @@
# SPDX-License-Identifier: Apache-2.0
"""The place of a node: the rules, the typed places, the stored places, and the job.
The tiles here are made in the test: each place is a point feature at a known location, so
a test states what the map holds and what the answer must be. No test reads the network.
"""
from __future__ import annotations
import asyncio
import json
import logging
from datetime import datetime, timedelta, timezone
from pathlib import Path
from types import SimpleNamespace
import pytest
from meshterm.core.geo import TILE_PX, lonlat_to_world
from meshterm.core.models import Observation
from meshterm.core.mvt import GEOM_POINT, Feature, Layer
from meshterm.core.place_store import PlaceStore
from meshterm.core.places import (
SOURCE_NOMINATIM,
SOURCE_TILES,
SOURCE_USER,
Place,
moved,
place_from_nominatim,
place_from_tiles,
)
from meshterm.core.preferences import Preferences
from meshterm.persistence.repository import Repository
from meshterm.services import places as places_service
from meshterm.services.places import PlaceService
_EXTENT = 4096
#: A point in a city, and the offsets (km) that the tests put places at.
_LAT, _LON = 45.5440, -73.6150
_KM_LAT = 1 / 111.2 # degrees of latitude in one km
def _north(km: float) -> tuple[float, float]:
"""A location ``km`` north of the test point."""
return _LAT + km * _KM_LAT, _LON
class _Map:
"""Tiles made from named points: each point goes into its tile at each zoom."""
def __init__(self) -> None:
self.points: list[tuple[str, str, str, float, float, int]] = []
self.missing: set[tuple[int, int, int]] = set()
self.loads: list[tuple[int, int, int]] = []
def add(self, layer: str, cls: str, name: str, at: tuple[float, float], rank: int = 10) -> None:
self.points.append((layer, cls, name, at[0], at[1], rank))
def load(self, z: int, x: int, y: int) -> list[Layer] | None:
self.loads.append((z, x, y))
if (z, x, y) in self.missing:
return None
layers = {name: Layer(name, _EXTENT) for name in ("place", "mountain_peak")}
for layer, cls, name, lat, lon, rank in self.points:
wx, wy = lonlat_to_world(lat, lon, z)
if int(wx // TILE_PX) != x or int(wy // TILE_PX) != y:
continue
px = int((wx / TILE_PX - x) * _EXTENT)
py = int((wy / TILE_PX - y) * _EXTENT)
tags = {"class": cls, "name": name, "rank": rank}
layers[layer].features.append(Feature(GEOM_POINT, [[(px, py)]], tags))
return list(layers.values())
# -- the rules ----------------------------------------------------------------
def test_a_neighbourhood_gets_the_city_that_holds_it() -> None:
world = _Map()
world.add("place", "neighbourhood", "Villeray", _north(0.5))
world.add("place", "city", "Montréal", _north(-6.0), rank=3)
place = place_from_tiles(_LAT, _LON, world.load)
assert place == Place("Villeray", "Montréal", SOURCE_TILES)
assert place.label == "Villeray, Montréal"
assert place.short == "Villeray"
def test_the_most_specific_area_wins_over_a_nearer_borough() -> None:
world = _Map()
world.add("place", "suburb", "Villeray–Saint-Michel–Parc-Extension", _north(0.2))
world.add("place", "neighbourhood", "Villeray", _north(1.0))
world.add("place", "city", "Montréal", _north(-6.0), rank=3)
assert place_from_tiles(_LAT, _LON, world.load).fine == "Villeray"
def test_a_large_city_wins_over_a_nearer_small_one() -> None:
"""The rank of a label weighs the distance: a node in Ahuntsic is in Montréal."""
world = _Map()
world.add("place", "neighbourhood", "Ahuntsic", _north(0.3))
world.add("place", "city", "Laval", _north(4.4), rank=11)
world.add("place", "city", "Montréal", _north(-9.4), rank=3)
assert place_from_tiles(_LAT, _LON, world.load).coarse == "Montréal"
def test_a_peak_near_the_summit_wins() -> None:
world = _Map()
world.add("mountain_peak", "peak", "Mont-Royal", _north(0.2))
world.add("place", "neighbourhood", "Villeray", _north(0.5))
world.add("place", "city", "Montréal", _north(-6.0), rank=3)
place = place_from_tiles(_LAT, _LON, world.load)
assert place == Place("Mont-Royal", None, SOURCE_TILES, peak=True)
def test_a_peak_past_its_radius_does_not_win() -> None:
world = _Map()
world.add("mountain_peak", "peak", "Mont-Royal", _north(0.6))
world.add("place", "neighbourhood", "Villeray", _north(0.5))
world.add("place", "city", "Montréal", _north(-6.0), rank=3)
assert place_from_tiles(_LAT, _LON, world.load).fine == "Villeray"
def test_a_village_stands_alone_and_needs_no_fine_tiles() -> None:
world = _Map()
world.add("place", "village", "Val-David", _north(1.5))
place = place_from_tiles(_LAT, _LON, world.load)
assert place == Place("Val-David", None, SOURCE_TILES)
assert {z for z, _, _ in world.loads} == {11} # the country costs only z11 tiles
def test_a_node_far_from_all_places_gets_the_nearest_settlement() -> None:
world = _Map()
world.add("place", "town", "Mont-Laurier", _north(30.0))
assert place_from_tiles(_LAT, _LON, world.load).fine == "Mont-Laurier"
def test_no_place_near_is_an_empty_answer() -> None:
place = place_from_tiles(_LAT, _LON, _Map().load)
assert place is not None and place.empty
assert place.label == "" and place.short == ""
def test_a_missing_home_tile_is_no_answer() -> None:
world = _Map()
world.add("place", "village", "Val-David", _north(1.5))
wx, wy = lonlat_to_world(_LAT, _LON, 11)
world.missing.add((11, int(wx // TILE_PX), int(wy // TILE_PX)))
assert place_from_tiles(_LAT, _LON, world.load) is None
def test_a_bad_location_is_no_answer() -> None:
assert place_from_tiles(0.0, 0.0, _Map().load) is None
assert place_from_tiles(-97.0, -1042.0, _Map().load) is None
def test_a_label_names_a_city_one_time() -> None:
assert Place("Montréal", "Montréal", SOURCE_TILES).label == "Montréal"
assert Place(None, "Laval", SOURCE_TILES).short == "Laval"
def test_moved_is_past_half_a_km() -> None:
assert not moved(_LAT, _LON, *_north(0.4))
assert moved(_LAT, _LON, *_north(0.6))
def test_nominatim_gives_fine_and_coarse() -> None:
doc = {"address": {"neighbourhood": "Villeray", "suburb": "VSP", "city": "Montréal"}}
assert place_from_nominatim(doc) == Place("Villeray", "Montréal", SOURCE_NOMINATIM)
def test_nominatim_town_alone() -> None:
doc = {"address": {"village": "Val-David", "county": "Les Laurentides"}}
assert place_from_nominatim(doc) == Place("Val-David", None, SOURCE_NOMINATIM)
def test_nominatim_error_is_an_empty_answer_and_junk_is_none() -> None:
assert place_from_nominatim({"error": "Unable to geocode"}).empty
assert place_from_nominatim(None) is None
assert place_from_nominatim({"nothing": 1}) is None
# -- the typed places ---------------------------------------------------------
def test_a_typed_place_is_kept_and_cleared(tmp_path: Path) -> None:
path = tmp_path / "places.json"
store = PlaceStore(path)
store.set("A1B2C3D4E5F6AABB", "Toit du stade", when=100)
assert PlaceStore(path).get("a1b2c3d4e5f6") == "Toit du stade"
store.set("a1b2c3d4e5f6", " ", when=200)
assert PlaceStore(path).get("a1b2c3d4e5f6") is None
def test_a_field_no_record_knows_is_gone_at_the_next_write(tmp_path: Path) -> None:
path = tmp_path / "places.json"
path.write_text(
json.dumps({"places": {"a1b2c3d4e5f6": {"name": "X", "set_at": 1, "old": 2}}}),
encoding="utf-8",
)
store = PlaceStore(path)
store.set("b1b2c3d4e5f6", "Y", when=3)
saved = json.loads(path.read_text(encoding="utf-8"))
assert saved["places"]["a1b2c3d4e5f6"] == {"name": "X", "set_at": 1}
# -- the stored places --------------------------------------------------------
def _heard(repo: Repository, node: str, lat: float, lon: float, minutes_ago: int) -> None:
run = repo.start_run("monitor", {}, None)
when = datetime.now(timezone.utc) - timedelta(minutes=minutes_ago)
repo.record_observation(run, Observation(node=node, lat=lat, lon=lon, observed_at=when))
def test_node_locations_gives_the_newest(tmp_path: Path) -> None:
repo = Repository(tmp_path / "p.db")
_heard(repo, "aaaaaaaaaaaa", 45.0, -73.0, minutes_ago=30)
_heard(repo, "aaaaaaaaaaaa", 46.0, -74.0, minutes_ago=5)
_heard(repo, "bbbbbbbbbbbb", 47.0, -75.0, minutes_ago=10)
locations = repo.node_locations()
assert locations["aaaaaaaaaaaa"][:2] == (46.0, -74.0)
stamp = locations["bbbbbbbbbbbb"][2]
assert set(repo.node_locations(since=stamp)) == {"aaaaaaaaaaaa"}
def test_a_stored_place_reads_back(tmp_path: Path) -> None:
repo = Repository(tmp_path / "p.db")
repo.save_node_place("aaaaaaaaaaaa", 45.0, -73.0, Place("Mont-Royal", None, "tiles", True))
repo.save_node_place("bbbbbbbbbbbb", 46.0, -74.0, Place(None, None, "nominatim"))
stored = repo.node_places()
assert stored["aaaaaaaaaaaa"] == (Place("Mont-Royal", None, "tiles", True), 45.0, -73.0)
assert stored["bbbbbbbbbbbb"][0].empty
# -- the service --------------------------------------------------------------
def _service(tmp_path: Path, source: str = "tiles_nominatim") -> tuple[PlaceService, Repository]:
repo = Repository(tmp_path / "p.db")
ctx = SimpleNamespace(
repo=repo,
place_store=PlaceStore(tmp_path / "places.json"),
preferences=Preferences(values={"place_source": source}),
log=logging.getLogger("test"),
)
return PlaceService(ctx), repo # type: ignore[arg-type]
def _run(coro): # noqa: ANN001, ANN202 - a small helper for the async steps
return asyncio.run(coro)
def test_a_typed_place_wins_and_an_empty_place_is_none(tmp_path: Path) -> None:
service, repo = _service(tmp_path)
repo.save_node_place("aaaaaaaaaaaa", 45.0, -73.0, Place("Villeray", "Montréal", "tiles"))
repo.save_node_place("bbbbbbbbbbbb", 45.0, -73.0, Place(None, None, "tiles"))
assert service.get("AAAAAAAAAAAA0000").label == "Villeray, Montréal"
assert service.get("bbbbbbbbbbbb") is None
before = service.revision
service.set_typed("aaaaaaaaaaaa", "Le toit")
assert service.revision > before
assert service.get("aaaaaaaaaaaa") == Place("Le toit", None, SOURCE_USER)
def test_the_job_uses_the_tiles_then_nominatim(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
service, repo = _service(tmp_path)
_heard(repo, "aaaaaaaaaaaa", _LAT, _LON, minutes_ago=1)
world = _Map()
world.add("place", "neighbourhood", "Ahuntsic", _north(0.3))
world.add("place", "city", "Laval", _north(4.4), rank=3)
monkeypatch.setattr(service, "_load_tile", world.load)
service._scan_after = 0.0
_run(service.step())
assert service.get("aaaaaaaaaaaa").label == "Ahuntsic, Laval"
asked: list[tuple[float, float]] = []
def nominatim(url: str, lat: float, lon: float) -> object:
asked.append((lat, lon))
return {"address": {"neighbourhood": "Ahuntsic", "city": "Montréal"}}
monkeypatch.setattr(places_service, "ask_nominatim", nominatim)
_run(service.step())
assert service.get("aaaaaaaaaaaa") == Place("Ahuntsic", "Montréal", SOURCE_NOMINATIM)
assert len(asked) == 1
_run(service.step())
assert len(asked) == 1 # one answer for each location: never asked again
def test_tiles_only_never_asks_nominatim(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
service, repo = _service(tmp_path, source="tiles")
_heard(repo, "aaaaaaaaaaaa", _LAT, _LON, minutes_ago=1)
world = _Map()
world.add("place", "village", "Val-David", _north(1.0))
monkeypatch.setattr(service, "_load_tile", world.load)
monkeypatch.setattr(places_service, "ask_nominatim", lambda *a: pytest.fail("asked"))
service._scan_after = 0.0
_run(service.step())
_run(service.step())
assert service.get("aaaaaaaaaaaa").fine == "Val-David"
def test_a_peak_stays_when_nominatim_could_correct(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
service, repo = _service(tmp_path)
_heard(repo, "aaaaaaaaaaaa", _LAT, _LON, minutes_ago=1)
repo.save_node_place("aaaaaaaaaaaa", _LAT, _LON, Place("Mont-Royal", None, "tiles", True))
monkeypatch.setattr(places_service, "ask_nominatim", lambda *a: pytest.fail("asked"))
service._scan_after = 0.0
_run(service.step())
assert service.get("aaaaaaaaaaaa").peak
def test_a_node_that_moved_is_found_again(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
service, repo = _service(tmp_path, source="tiles")
repo.save_node_place("aaaaaaaaaaaa", *_north(-5.0), Place("Old", None, "tiles"))
_heard(repo, "aaaaaaaaaaaa", _LAT, _LON, minutes_ago=1)
world = _Map()
world.add("place", "village", "New", _north(1.0))
monkeypatch.setattr(service, "_load_tile", world.load)
service._scan_after = 0.0
_run(service.step())
assert service.get("aaaaaaaaaaaa").fine == "New"
def test_no_answer_makes_the_node_wait(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
service, repo = _service(tmp_path, source="tiles")
_heard(repo, "aaaaaaaaaaaa", _LAT, _LON, minutes_ago=1)
calls: list[int] = []
def offline(z: int, x: int, y: int) -> None:
calls.append(z)
return None
monkeypatch.setattr(service, "_load_tile", offline)
service._scan_after = 0.0
_run(service.step())
_run(service.step())
assert len(calls) == 1 # the second step does not ask again at once
assert service.get("aaaaaaaaaaaa") is None