MeshTerm/meshterm/services/basemap.py

"""Fetch and cache OpenStreetMap vector tiles for the terminal map.

This is the only networked part of the map. It resolves a tile source's (versioned) tile-URL
template from its TileJSON, fetches ``.pbf`` vector tiles over HTTPS, and caches them on disk
so panning back over ground you've seen is instant and later sessions work offline. Decoding
is delegated to the pure :mod:`meshterm.core.mvt`, and narrowed to the layers the renderer
actually draws (``layers``) — a planet tile carries buildings, house numbers and POIs the
terminal map has no pixels for, and they are the bulk of its decode cost. Only the decode
narrows: whole tiles are still cached, so drawing more later costs a re-decode, never a
re-download.

Decoding is also written down. Beside the raw tiles sits a ``decoded/`` sidecar holding
the parsed layers (see :func:`~meshterm.core.mvt.dumps_layers`), so a tile is parsed once
ever rather than once per session — measured ~14x cheaper to reload on the PicoCalc. That
half of the cache is pure derived data: it is never what makes a tile "known", it carries
a stamp of the layer set it was decoded under so a narrowed blob is never served to a
caller wanting more, and it is the half that gets a size budget, because anything evicted
costs only the decode it was saving.

The default source is **OpenFreeMap** (openfreemap.org) — full-planet OpenStreetMap vector
tiles, free and requiring no API key. Everything here is best-effort: with no network and no
cached tiles the loader simply returns ``None`` and the map falls back to plotting nodes on a
blank grid, never raising into the UI.

**Only tiles that decode to real geometry are ever written to disk.** A cache is a memory,
and the one thing it must not remember is a lie: on a flaky link (the PicoCalc's Wi-Fi, most
of all) a timeout, a reset, or a body cut short is *silence*, not an empty tile, and storing
it leaves a blank square on the map for every future session. So a fetch is only cached once
its bytes have decoded to at least one feature; anything else is either dropped on the floor
(no answer — try again next pan) or remembered in memory for this session alone (the source
answered, and there is genuinely nothing there). Cache entries written by earlier builds that
can't decode are pruned on read, which heals a cache already poisoned this way.

Fetches are blocking (stdlib ``urllib``); callers on an event loop should run
:meth:`BasemapSource.load_tile` via ``asyncio.to_thread`` so the UI stays responsive.
"""

from __future__ import annotations

import json
import logging
import os
import threading
import urllib.error
import urllib.request
from collections import OrderedDict
from pathlib import Path
from typing import Container, NamedTuple, Optional

from ..core.mvt import Layer, decode_tile, dumps_layers, loads_layers

#: OpenFreeMap planet TileJSON — its ``tiles`` array holds the current versioned template.
DEFAULT_TILEJSON_URL = "https://tiles.openfreemap.org/planet"

#: Sent on every request; identifies the client per common tile-usage etiquette.
_USER_AGENT = "MeshTerm/0.1 (+https://github.com/; mesh node map)"

#: Fallback max tile zoom if the TileJSON doesn't declare one (OpenFreeMap serves 14).
_DEFAULT_MAX_ZOOM = 14

#: HTTP statuses that are the source *answering* "there is no such resource". Everything
#: else — 429, 5xx, and every transport error — is the absence of an answer.
_ABSENT_STATUSES = frozenset({404, 410})

#: Ceiling on the decoded sidecar cache. Decoded tiles run ~2.4x the size of the bytes
#: they came from, and unlike those bytes they can be rebuilt from what's already on disk,
#: so this half of the cache is the half that gets a budget.
_DEFAULT_MAX_DECODED_BYTES = 64 * 1024 * 1024

#: How much must be written before the sidecar budget is checked again. The check walks
#: the directory, which is slow on the SD card the PicoCalc runs from.
_PRUNE_AFTER_BYTES = 8 * 1024 * 1024

#: Decoded tiles held in RAM. A viewport spans at most a handful of tiles, so this covers
#: the view plus the ring a pan or a zoom step reaches into, and little more — the PicoCalc
#: has 100 MB of RAM in total and decoded layers are not small.
_MEMO_TILES = 24

_log = logging.getLogger(__name__)


class _Response(NamedTuple):
    """One HTTP GET's outcome: whether the server answered, and what it said.

    The distinction is the whole point. A definitive answer can be acted on — bytes to
    decode, or a 404 meaning there is no tile at those coordinates. Offline, DNS failure,
    timeout, connection reset, throttling, a server fault, a body short of its declared
    length: none of those tell us anything about the tile, and must never be mistaken for
    the source saying it is empty.

    Attributes:
        answered: Whether the server gave a definitive answer.
        body: The body it gave; empty when the answer was "no such resource".
    """

    answered: bool
    body: bytes


class BasemapSource:
    """A cached OpenStreetMap vector-tile source.

    Attributes:
        cache_dir: Directory under which tiles and the resolved TileJSON are cached.
    """

    def __init__(
        self,
        cache_dir: Path,
        *,
        tilejson_url: str = DEFAULT_TILEJSON_URL,
        timeout: float = 12.0,
        layers: Optional[Container[str]] = None,
        max_decoded_bytes: int = _DEFAULT_MAX_DECODED_BYTES,
    ) -> None:
        """Open a tile source backed by an on-disk cache.

        Args:
            cache_dir: Where fetched tiles and TileJSON metadata are stored.
            tilejson_url: URL of the source's TileJSON document.
            timeout: Per-request network timeout in seconds.
            layers: Restrict decoding to these layer names (the renderer passes
                :data:`meshterm.ui.map_render.DRAWN_LAYERS`). Only the *decode* narrows —
                the cache still stores whole tiles — so this is a pure CPU saving that a
                later renderer can widen without re-fetching anything.
            max_decoded_bytes: Ceiling on the decoded sidecar cache, which is derived data
                and so is the one part of the cache that can be thrown away freely.
        """
        self.cache_dir = cache_dir
        self._tilejson_url = tilejson_url
        self._timeout = timeout
        self._layers = layers
        self._max_decoded_bytes = max_decoded_bytes
        self._decoded_written = 0
        # What a sidecar was decoded *under*. A blob written for one layer set must never
        # be served to a caller expecting another, so the set travels with the bytes.
        self._stamp = "all" if layers is None else ",".join(sorted(layers))  # type: ignore[arg-type]
        self._template: Optional[str] = None
        self._max_zoom: Optional[int] = None
        self._resolved = False  # whether we've tried (success or offline) this session
        # Tiles the source answered "nothing here" for this session. Held in memory rather
        # than on disk so the claim expires with the process: a blank tile is cheap to
        # re-ask about, and a stale one on disk is a permanent hole in the map.
        self._blank: set[tuple[int, int, int]] = set()
        # Decoded layers kept in RAM, most-recently-used last. The sidecar already spares
        # the protobuf decode, but a marshal load off the SD card is still ~96 ms on the
        # PicoCalc — and a map paints its whole viewport's worth of tiles on *every*
        # frame, so panning one dot re-read every tile that had not moved. Bounded because
        # decoded layers are the fattest thing this class holds.
        self._memo: "OrderedDict[tuple[int, int, int], list[Layer]]" = OrderedDict()

    # -- metadata ---------------------------------------------------------------

    def _tilejson_path(self) -> Path:
        return self.cache_dir / "tilejson.json"

    def _resolve(self) -> None:
        """Resolve and cache the tile-URL template and max zoom (best-effort, once)."""
        if self._resolved:
            return
        self._resolved = True
        # Prefer a freshly fetched TileJSON (the template is versioned and rotates), but fall
        # back to a previously cached copy so a session started offline can still use disk
        # tiles and even re-fetch if the template is still valid.
        resp = self._http_get(self._tilejson_url)
        data = resp.body if resp.answered and resp.body else None
        if data is not None:
            try:
                self._tilejson_path().parent.mkdir(parents=True, exist_ok=True)
                self._tilejson_path().write_bytes(data)
            except OSError:  # pragma: no cover - cache write failure is non-fatal
                pass
        if data is None and self._tilejson_path().exists():
            try:
                data = self._tilejson_path().read_bytes()
            except OSError:  # pragma: no cover
                data = None
        if data is None:
            return
        try:
            doc = json.loads(data)
            tiles = doc.get("tiles") or []
            if tiles:
                self._template = str(tiles[0])
            self._max_zoom = int(doc.get("maxzoom", _DEFAULT_MAX_ZOOM))
        except (ValueError, TypeError):  # pragma: no cover - malformed TileJSON
            pass

    @property
    def max_zoom(self) -> int:
        """The highest zoom the source serves (resolved lazily; sensible default offline)."""
        self._resolve()
        return self._max_zoom if self._max_zoom is not None else _DEFAULT_MAX_ZOOM

    @property
    def available(self) -> bool:
        """Whether a tile-URL template is known (network reachable, or one was cached)."""
        self._resolve()
        return self._template is not None

    # -- tiles ------------------------------------------------------------------

    def _tile_path(self, z: int, x: int, y: int) -> Path:
        return self.cache_dir / "tiles" / str(z) / str(x) / f"{y}.pbf"

    def _decoded_path(self, z: int, x: int, y: int) -> Path:
        return self.cache_dir / "decoded" / str(z) / str(x) / f"{y}.bin"

    def load_tile(self, z: int, x: int, y: int) -> Optional[list[Layer]]:
        """Return the decoded layers for a tile, from cache or the network.

        Three places are tried in cost order: the decoded sidecar (a ``marshal`` load),
        the raw ``.pbf`` (a full protobuf decode), then the network. The sidecar is pure
        derived data — it is never the reason a tile is considered known, and losing it
        costs only the decode it was there to save.

        The raw cache only ever holds tiles that decoded to real geometry. A tile the
        source never answered for is left uncached, so the next pan over it asks again
        instead of drawing a permanent blank; a cached file that no longer decodes is
        pruned for the same reason.

        Args:
            z: Tile zoom.
            x: Tile x index.
            y: Tile y index.

        Returns:
            The decoded layers, or ``None`` if the tile is unavailable (offline and
            uncached, or a genuinely empty/missing tile).
        """
        key = (z, x, y)
        if key in self._blank:
            return None
        hot = self._memo.get(key)
        if hot is not None:
            self._memo.move_to_end(key)
            return hot
        ready = self._read_decoded(z, x, y)
        if ready is not None:
            self._remember(key, ready)
            return ready
        path = self._tile_path(z, x, y)
        cached = self._read_cached(path)
        if cached is not None:
            layers = self._decode(cached, key)
            if layers is not None:
                self._write_decoded(z, x, y, layers)
                self._remember(key, layers)
                return layers
            # Nothing drawable came out: a zero-byte marker from a build that cached
            # network failures, or bytes truncated by a link (or a power cut) mid-write.
            # Either way it is a blank square for ever unless we drop it and re-ask.
            _log.debug("dropping unusable cached tile %s/%s/%s", z, x, y)
            self._discard_cached(path)
        raw = self._fetch_tile(z, x, y)
        if raw is None:
            return None  # no answer — nothing learned, so nothing is written down
        layers = self._decode(raw, key)
        if layers is None:
            self._blank.add(key)  # the source answered: there is genuinely nothing here
            return None
        self._write_cached(path, raw)
        self._write_decoded(z, x, y, layers)
        self._remember(key, layers)
        return layers

    def _remember(self, key: tuple[int, int, int], layers: list[Layer]) -> None:
        """Hold a decoded tile in RAM, evicting the least recently used past the budget."""
        self._memo[key] = layers
        self._memo.move_to_end(key)
        while len(self._memo) > _MEMO_TILES:
            self._memo.popitem(last=False)

    # -- the decoded sidecar ----------------------------------------------------

    def _read_decoded(self, z: int, x: int, y: int) -> Optional[list[Layer]]:
        """Return a tile's already-decoded layers, or ``None`` to decode it properly."""
        blob = self._read_cached(self._decoded_path(z, x, y))
        return None if blob is None else loads_layers(blob, stamp=self._stamp)

    def _write_decoded(self, z: int, x: int, y: int, layers: list[Layer]) -> None:
        """Write a tile's decoded layers beside the raw bytes, and keep the dir bounded."""
        try:
            blob = dumps_layers(layers, stamp=self._stamp)
        except ValueError:  # pragma: no cover - a tag type marshal can't represent
            return
        self._write_cached(self._decoded_path(z, x, y), blob)
        # Sweeping the tree costs a directory walk on an SD card, so amortise it over a
        # good many writes rather than checking the budget on every tile.
        self._decoded_written += len(blob)
        if self._decoded_written >= _PRUNE_AFTER_BYTES:
            self._decoded_written = 0
            self._prune_decoded()

    def _prune_decoded(self) -> None:
        """Drop the least recently written sidecars until the budget is met.

        Decoded tiles run ~2.4x the size of the bytes they came from, and the raw cache is
        already unbounded, so this side of it gets a ceiling. Eviction is by modification
        time, which on a write-once file is also the order it was first needed in.
        """
        root = self.cache_dir / "decoded"
        try:
            files = [(p.stat().st_mtime, p.stat().st_size, p) for p in root.rglob("*.bin")]
        except OSError:  # pragma: no cover - a cache we can't walk is still readable
            return
        total = sum(size for _, size, _ in files)
        if total <= self._max_decoded_bytes:
            return
        for _, size, p in sorted(files):
            if total <= self._max_decoded_bytes:
                break
            self._discard_cached(p)
            total -= size
        _log.debug("pruned decoded tile cache to %.1f MB", total / 1024 / 1024)

    def _decode(self, raw: bytes, key: tuple[int, int, int]) -> Optional[list[Layer]]:
        """Decode a tile's bytes, or ``None`` if they aren't a tile at all.

        The question this answers is "are these real tile bytes?", and the answer decides
        whether a cache entry is kept or pruned — so it must not be confused with "is
        there anything here I feel like drawing". Under ``layers`` most of a tile's
        content is deliberately left undecoded, and a tile whose every layer we skip
        would otherwise read as blank and get its (perfectly good) cache entry deleted
        and re-downloaded every session.

        So the test is whether the bytes parsed into any layer at all. That is sound
        because the three ways a tile is *not* real are all distinguishable without
        looking at features: empty bytes decode to no layers, and truncated or junk bytes
        raise out of the parser rather than yielding a well-formed featureless layer.

        Args:
            raw: The tile's raw ``.pbf`` bytes.
            key: The ``(z, x, y)`` the bytes claim to be, for the log line.

        Returns:
            The decoded layers, or ``None`` for empty or corrupt bytes.
        """
        try:
            layers = decode_tile(raw, layers=self._layers)
        except Exception as exc:  # noqa: BLE001 - a corrupt tile must not crash the map
            _log.debug("failed to decode tile %s/%s/%s: %s", *key, exc)
            return None
        return layers or None

    def _fetch_tile(self, z: int, x: int, y: int) -> Optional[bytes]:
        """Fetch a tile's raw bytes from the network.

        Args:
            z: Tile zoom.
            x: Tile x index.
            y: Tile y index.

        Returns:
            The body the source gave — possibly empty, meaning "no tile at those
            coordinates" — or ``None`` if it never answered (offline, timed out,
            throttled, faulted, or cut short).
        """
        self._resolve()
        if self._template is None:
            return None
        url = (
            self._template.replace("{z}", str(z)).replace("{x}", str(x)).replace("{y}", str(y))
        )
        resp = self._http_get(url)
        return resp.body if resp.answered else None

    @staticmethod
    def _read_cached(path: Path) -> Optional[bytes]:
        try:
            return path.read_bytes() if path.exists() else None
        except OSError:  # pragma: no cover
            return None

    @staticmethod
    def _write_cached(path: Path, raw: bytes) -> None:
        """Write a tile to the cache atomically, so a half-written one is never read back."""
        # Unique per writer: the map screen and a mini-map share one source, and both may
        # be fetching the same tile in their own worker threads.
        tmp = path.with_name(f"{path.name}.{os.getpid()}-{threading.get_ident()}.part")
        try:
            path.parent.mkdir(parents=True, exist_ok=True)
            tmp.write_bytes(raw)
            os.replace(tmp, path)
        except OSError:  # pragma: no cover - cache write failure is non-fatal
            try:
                tmp.unlink(missing_ok=True)
            except OSError:
                pass

    @staticmethod
    def _discard_cached(path: Path) -> None:
        """Drop a cache entry that proved unusable, so the tile can be fetched afresh."""
        try:
            path.unlink(missing_ok=True)
        except OSError:  # pragma: no cover - a cache we can't prune is still readable
            pass

    def _http_get(self, url: str) -> _Response:
        """GET a URL, telling a definitive answer apart from no answer at all.

        Args:
            url: The absolute URL to fetch.

        Returns:
            The outcome — see :class:`_Response`.
        """
        req = urllib.request.Request(url, headers={"User-Agent": _USER_AGENT})
        try:
            with urllib.request.urlopen(req, timeout=self._timeout) as resp:
                body = resp.read()
                declared = (resp.headers.get("Content-Length") or "").strip()
                # A flaky link can end a read early without raising. A body short of its
                # declared length is a truncation, not a tile — and not an empty one.
                if declared.isdigit() and len(body) != int(declared):
                    _log.debug("short read for %s: %d of %s bytes", url, len(body), declared)
                    return _Response(False, b"")
                return _Response(True, body)
        except urllib.error.HTTPError as exc:
            answered = exc.code in _ABSENT_STATUSES
            if not answered:
                _log.debug("fetch failed for %s: HTTP %s", url, exc.code)
            return _Response(answered, b"")
        except Exception as exc:  # noqa: BLE001 - offline / timeout / reset are "no answer"
            _log.debug("fetch failed for %s: %s", url, exc)
            return _Response(False, b"")