diff --git a/meshterm/core/mvt.py b/meshterm/core/mvt.py index c3f9d16..149e81d 100644 --- a/meshterm/core/mvt.py +++ b/meshterm/core/mvt.py @@ -14,6 +14,7 @@ projected to the screen by :class:`meshterm.core.geo.Viewport`. from __future__ import annotations import gzip +import marshal from dataclasses import dataclass, field from typing import Any, Container, Optional @@ -328,3 +329,74 @@ def decode_tile(data: bytes, *, layers: Optional[Container[str]] = None) -> list else: r.skip(wire) return out + + +# -- the decoded form, for callers that would rather not decode twice ------------------ + +#: Bumped whenever :func:`dumps_layers` changes the shape it writes. +_WIRE_VERSION = 1 + + +def dumps_layers(layers: list[Layer], *, stamp: str = "") -> bytes: + """Serialise decoded layers to bytes that :func:`loads_layers` can restore. + + Decoding a vector tile is by far the most expensive thing this app does (a 153 KB + tile costs ~365 ms on the PicoCalc's Cortex-A7 even after the layer narrowing), and + the result is a pure function of the bytes and the layer set. Writing it down means a + later session pays a ``marshal`` load and some object construction — measured ~14x + cheaper — instead of parsing the protobuf again. + + ``marshal`` is the format because it is stdlib, C-speed, and understands the plain + tuples/lists/dicts/scalars a decoded tile reduces to. It is deliberately *not* + ``pickle``: this reads a file off disk, and marshal cannot be made to import a module + or call a constructor. It is still only safe against data we wrote ourselves, which + is why the cache lives under the app's own directory and why every load is guarded — + see :func:`loads_layers`. + + Args: + layers: The decoded layers to write down. + stamp: An opaque caller token describing *how* these were decoded (the layer set, + typically). :func:`loads_layers` refuses a blob whose stamp differs, which is + what stops a narrowed decode from being served to a caller that wants more. + + Returns: + The encoded bytes. + """ + payload = [ + (layer.name, layer.extent, + [(f.geom_type, f.rings, f.tags) for f in layer.features]) + for layer in layers + ] + return marshal.dumps((_WIRE_VERSION, marshal.version, stamp, payload)) + + +def loads_layers(blob: bytes, *, stamp: str = "") -> Optional[list[Layer]]: + """Restore layers written by :func:`dumps_layers`, or ``None`` if they can't be used. + + Every way the blob can fail to be what this build expects — a bumped wire version, a + Python whose ``marshal`` writes a different format, a different layer set, a file + truncated by a power cut — resolves to ``None``, meaning "decode the tile again". + Nothing here raises into the map: a derived cache that can't be read is not an error, + it is just a cache miss. + + Args: + blob: Bytes previously produced by :func:`dumps_layers`. + stamp: The token the caller expects; a blob written under any other is rejected. + + Returns: + The layers, or ``None`` to mean "re-decode". + """ + try: + version, marshal_version, written_stamp, payload = marshal.loads(blob) + if (version, marshal_version, written_stamp) != (_WIRE_VERSION, marshal.version, stamp): + return None + return [ + Layer( + name=name, + extent=extent, + features=[Feature(geom_type=g, rings=r, tags=t) for g, r, t in feats], + ) + for name, extent, feats in payload + ] + except Exception: # noqa: BLE001 - any malformed blob is simply a cache miss + return None diff --git a/meshterm/services/basemap.py b/meshterm/services/basemap.py index 29d05b4..95afd5f 100644 --- a/meshterm/services/basemap.py +++ b/meshterm/services/basemap.py @@ -9,6 +9,14 @@ terminal map has no pixels for, and they are the bulk of its decode cost. Only t narrows: whole tiles are still cached, so drawing more later costs a re-decode, never a re-download. +Decoding is also written down. Beside the raw tiles sits a ``decoded/`` sidecar holding +the parsed layers (see :func:`~meshterm.core.mvt.dumps_layers`), so a tile is parsed once +ever rather than once per session — measured ~14x cheaper to reload on the PicoCalc. That +half of the cache is pure derived data: it is never what makes a tile "known", it carries +a stamp of the layer set it was decoded under so a narrowed blob is never served to a +caller wanting more, and it is the half that gets a size budget, because anything evicted +costs only the decode it was saving. + The default source is **OpenFreeMap** (openfreemap.org) — full-planet OpenStreetMap vector tiles, free and requiring no API key. Everything here is best-effort: with no network and no cached tiles the loader simply returns ``None`` and the map falls back to plotting nodes on a @@ -38,7 +46,7 @@ import urllib.request from pathlib import Path from typing import Container, NamedTuple, Optional -from ..core.mvt import Layer, decode_tile +from ..core.mvt import Layer, decode_tile, dumps_layers, loads_layers #: OpenFreeMap planet TileJSON — its ``tiles`` array holds the current versioned template. DEFAULT_TILEJSON_URL = "https://tiles.openfreemap.org/planet" @@ -53,6 +61,15 @@ _DEFAULT_MAX_ZOOM = 14 #: else — 429, 5xx, and every transport error — is the absence of an answer. _ABSENT_STATUSES = frozenset({404, 410}) +#: Ceiling on the decoded sidecar cache. Decoded tiles run ~2.4x the size of the bytes +#: they came from, and unlike those bytes they can be rebuilt from what's already on disk, +#: so this half of the cache is the half that gets a budget. +_DEFAULT_MAX_DECODED_BYTES = 64 * 1024 * 1024 + +#: How much must be written before the sidecar budget is checked again. The check walks +#: the directory, which is slow on the SD card the PicoCalc runs from. +_PRUNE_AFTER_BYTES = 8 * 1024 * 1024 + _log = logging.getLogger(__name__) @@ -88,6 +105,7 @@ class BasemapSource: tilejson_url: str = DEFAULT_TILEJSON_URL, timeout: float = 12.0, layers: Optional[Container[str]] = None, + max_decoded_bytes: int = _DEFAULT_MAX_DECODED_BYTES, ) -> None: """Open a tile source backed by an on-disk cache. @@ -99,11 +117,18 @@ class BasemapSource: :data:`meshterm.ui.map_render.DRAWN_LAYERS`). Only the *decode* narrows — the cache still stores whole tiles — so this is a pure CPU saving that a later renderer can widen without re-fetching anything. + max_decoded_bytes: Ceiling on the decoded sidecar cache, which is derived data + and so is the one part of the cache that can be thrown away freely. """ self.cache_dir = cache_dir self._tilejson_url = tilejson_url self._timeout = timeout self._layers = layers + self._max_decoded_bytes = max_decoded_bytes + self._decoded_written = 0 + # What a sidecar was decoded *under*. A blob written for one layer set must never + # be served to a caller expecting another, so the set travels with the bytes. + self._stamp = "all" if layers is None else ",".join(sorted(layers)) # type: ignore[arg-type] self._template: Optional[str] = None self._max_zoom: Optional[int] = None self._resolved = False # whether we've tried (success or offline) this session @@ -166,13 +191,21 @@ class BasemapSource: def _tile_path(self, z: int, x: int, y: int) -> Path: return self.cache_dir / "tiles" / str(z) / str(x) / f"{y}.pbf" + def _decoded_path(self, z: int, x: int, y: int) -> Path: + return self.cache_dir / "decoded" / str(z) / str(x) / f"{y}.bin" + def load_tile(self, z: int, x: int, y: int) -> Optional[list[Layer]]: """Return the decoded layers for a tile, from cache or the network. - The cache only ever holds tiles that decoded to real geometry. A tile the source - never answered for is left uncached, so the next pan over it asks again instead of - drawing a permanent blank; a cached file that no longer decodes is pruned for the - same reason. + Three places are tried in cost order: the decoded sidecar (a ``marshal`` load), + the raw ``.pbf`` (a full protobuf decode), then the network. The sidecar is pure + derived data — it is never the reason a tile is considered known, and losing it + costs only the decode it was there to save. + + The raw cache only ever holds tiles that decoded to real geometry. A tile the + source never answered for is left uncached, so the next pan over it asks again + instead of drawing a permanent blank; a cached file that no longer decodes is + pruned for the same reason. Args: z: Tile zoom. @@ -186,11 +219,15 @@ class BasemapSource: key = (z, x, y) if key in self._blank: return None + ready = self._read_decoded(z, x, y) + if ready is not None: + return ready path = self._tile_path(z, x, y) cached = self._read_cached(path) if cached is not None: layers = self._decode(cached, key) if layers is not None: + self._write_decoded(z, x, y, layers) return layers # Nothing drawable came out: a zero-byte marker from a build that cached # network failures, or bytes truncated by a link (or a power cut) mid-write. @@ -205,8 +242,52 @@ class BasemapSource: self._blank.add(key) # the source answered: there is genuinely nothing here return None self._write_cached(path, raw) + self._write_decoded(z, x, y, layers) return layers + # -- the decoded sidecar ---------------------------------------------------- + + def _read_decoded(self, z: int, x: int, y: int) -> Optional[list[Layer]]: + """Return a tile's already-decoded layers, or ``None`` to decode it properly.""" + blob = self._read_cached(self._decoded_path(z, x, y)) + return None if blob is None else loads_layers(blob, stamp=self._stamp) + + def _write_decoded(self, z: int, x: int, y: int, layers: list[Layer]) -> None: + """Write a tile's decoded layers beside the raw bytes, and keep the dir bounded.""" + try: + blob = dumps_layers(layers, stamp=self._stamp) + except ValueError: # pragma: no cover - a tag type marshal can't represent + return + self._write_cached(self._decoded_path(z, x, y), blob) + # Sweeping the tree costs a directory walk on an SD card, so amortise it over a + # good many writes rather than checking the budget on every tile. + self._decoded_written += len(blob) + if self._decoded_written >= _PRUNE_AFTER_BYTES: + self._decoded_written = 0 + self._prune_decoded() + + def _prune_decoded(self) -> None: + """Drop the least recently written sidecars until the budget is met. + + Decoded tiles run ~2.4x the size of the bytes they came from, and the raw cache is + already unbounded, so this side of it gets a ceiling. Eviction is by modification + time, which on a write-once file is also the order it was first needed in. + """ + root = self.cache_dir / "decoded" + try: + files = [(p.stat().st_mtime, p.stat().st_size, p) for p in root.rglob("*.bin")] + except OSError: # pragma: no cover - a cache we can't walk is still readable + return + total = sum(size for _, size, _ in files) + if total <= self._max_decoded_bytes: + return + for _, size, p in sorted(files): + if total <= self._max_decoded_bytes: + break + self._discard_cached(p) + total -= size + _log.debug("pruned decoded tile cache to %.1f MB", total / 1024 / 1024) + def _decode(self, raw: bytes, key: tuple[int, int, int]) -> Optional[list[Layer]]: """Decode a tile's bytes, or ``None`` if they aren't a tile at all. diff --git a/tests/test_map.py b/tests/test_map.py index 13700fc..8bc32b5 100644 --- a/tests/test_map.py +++ b/tests/test_map.py @@ -9,6 +9,7 @@ against the :class:`MockDevice` simulator with the basemap disabled (no network from __future__ import annotations import io +import os import re from pathlib import Path @@ -471,6 +472,103 @@ def test_basemap_source_keeps_a_tile_whose_layers_are_all_undrawn(tmp_path: Path assert tile.exists(), "the cache entry must survive" +def test_decoded_layers_survive_a_round_trip() -> None: + """The sidecar encoding restores exactly what the decoder produced.""" + from meshterm.core.mvt import dumps_layers, loads_layers + from meshterm.ui.map_render import DRAWN_LAYERS + + original = decode_tile(_FIXTURE.read_bytes(), layers=DRAWN_LAYERS) + restored = loads_layers(dumps_layers(original, stamp="x"), stamp="x") + + assert restored is not None + assert [(l.name, l.extent) for l in restored] == [(l.name, l.extent) for l in original] + for before, after in zip(original, restored): + assert after.features == before.features + + +def test_decoded_layers_refuse_a_blob_from_another_layer_set() -> None: + """A narrowed blob must never be served to a caller that wants more of the tile.""" + from meshterm.core.mvt import dumps_layers, loads_layers + + blob = dumps_layers(decode_tile(_FIXTURE.read_bytes(), layers={"water"}), stamp="water") + assert loads_layers(blob, stamp="water") is not None + assert loads_layers(blob, stamp="water,place") is None + + +def test_decoded_layers_treat_damage_as_a_miss() -> None: + """A truncated or foreign sidecar is a cache miss, never an exception.""" + from meshterm.core.mvt import dumps_layers, loads_layers + + blob = dumps_layers(decode_tile(_FIXTURE.read_bytes(), layers={"water"}), stamp="s") + assert loads_layers(blob[: len(blob) // 2], stamp="s") is None + assert loads_layers(b"", stamp="s") is None + assert loads_layers(b"not marshal at all", stamp="s") is None + + +def test_basemap_source_writes_and_reuses_a_decoded_sidecar(tmp_path: Path) -> None: + """The second load parses nothing: it comes back from the sidecar.""" + cache = tmp_path / "cache" + tile = cache / "tiles" / "14" / "4843" / "5861.pbf" + tile.parent.mkdir(parents=True) + tile.write_bytes(_FIXTURE.read_bytes()) + + src = _offline_source(cache) + first = src.load_tile(14, 4843, 5861) + sidecar = cache / "decoded" / "14" / "4843" / "5861.bin" + assert first is not None + assert sidecar.exists(), "decoding should have been written down" + + # Break the raw tile: a second load that still works can only have used the sidecar. + tile.write_bytes(b"junk that cannot decode") + second = src.load_tile(14, 4843, 5861) + assert second is not None + assert [(l.name, len(l.features)) for l in second] == [ + (l.name, len(l.features)) for l in first + ] + + +def test_basemap_source_ignores_a_sidecar_from_another_layer_set(tmp_path: Path) -> None: + """Widening what the map draws re-decodes rather than serving the narrower blob.""" + from meshterm.services.basemap import BasemapSource + + cache = tmp_path / "cache" + tile = cache / "tiles" / "14" / "4843" / "5861.pbf" + tile.parent.mkdir(parents=True) + tile.write_bytes(_FIXTURE.read_bytes()) + + def source(layers): + return BasemapSource( + cache, tilejson_url="http://127.0.0.1:1/none", timeout=0.2, layers=layers + ) + + narrow = source(frozenset({"water"})) + assert narrow.load_tile(14, 4843, 5861) is not None + + wider = source(frozenset({"water", "place"})) + layers = {layer.name: layer for layer in wider.load_tile(14, 4843, 5861) or []} + assert layers["place"].features, "the wider set must have been decoded afresh" + + +def test_basemap_source_prunes_the_decoded_cache_to_its_budget(tmp_path: Path) -> None: + """The sidecar half of the cache is bounded; the raw tiles it derives from are not.""" + from meshterm.services.basemap import BasemapSource + + cache = tmp_path / "cache" + decoded = cache / "decoded" / "14" / "1" + decoded.mkdir(parents=True) + for i in range(6): + blob = decoded / f"{i}.bin" + blob.write_bytes(b"x" * 1000) + os.utime(blob, (i, i)) # oldest first + + # 6 KB present, 3 KB allowed: the three oldest go, the three newest stay. + src = BasemapSource(cache, tilejson_url="http://127.0.0.1:1/none", max_decoded_bytes=3000) + src._prune_decoded() + + left = sorted(p.stem for p in decoded.glob("*.bin")) + assert left == ["3", "4", "5"], "the oldest sidecars go first" + + def test_basemap_source_remembers_a_blank_tile_for_the_session(tmp_path: Path) -> None: """An answered-empty tile isn't re-requested this session (but isn't written down).""" from meshterm.services import basemap as basemap_mod