diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..3fc5ab4 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,41 @@ +# scripts/ + +Repo-side tools. None of these ship in the wheel. + +| Script | What it is for | +|---|---| +| `basemap-doctor.py` | Why a user's map has no basemap — certificates, DNS, proxy, or colour depth | +| `picocalc/` | Calculinux device setup: console font, palette, Wi-Fi, radio bridge | +| `uconsole/` | The uConsole SPI bridge service | + +## basemap-doctor.py + +The map is the only networked part of MeshTerm, and every fetch failure on that path is +logged at DEBUG while `log_level` defaults to WARNING — so a reader whose tiles never +arrive sees a blank ground and no explanation. This prints the explanation and the fix. + +Send a user this, to run **on the machine with the problem**: + +```bash +curl -fsSL https://raw.githubusercontent.com/jpmartineau/MeshTerm/main/scripts/basemap-doctor.py -o /tmp/basemap-doctor.py +python3 /tmp/basemap-doctor.py +``` + +Any `python3` will do: if that one has no MeshTerm, the script reads the interpreter out +of the installed `meshterm` command's shebang and hands itself over, so the answer always +comes from **the app's own Python** — a Mac routinely has three and only that one has the +trust store that matters. Save it to a file as above rather than piping into `python3 -`: +with no file on disk there is nothing to hand over. + +Have them paste the whole output back. It names the interpreter, the certificate store, +the tile source, the cached-tile count, `COLORTERM`, and then one verdict: + +- **curl reached it, Python did not** — the Python's certificate store is empty. A + python.org install on macOS ships that way until `Install Certificates.command` is run. +- **neither reached it** — the network path: a firewall, a proxy, a DNS filter, or an + outage. Nothing to fix in MeshTerm. +- **both reached it** — tiles are fine and the complaint is colour: an unset `COLORTERM` + means 256 colours, where the basemap's greens and blues collide. Terminal.app cannot do + 24-bit colour at all. + +Exit status is 0 when tiles arrive and 1 when they do not, so it can be scripted. diff --git a/scripts/basemap-doctor.py b/scripts/basemap-doctor.py new file mode 100644 index 0000000..90e70eb --- /dev/null +++ b/scripts/basemap-doctor.py @@ -0,0 +1,427 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Diagnose why MeshTerm's map has no basemap, on the machine that has the problem. + +The map is the only networked part of MeshTerm: it fetches OpenStreetMap vector tiles over +HTTPS from the source named by the ``basemap_tilejson_url`` preference. Every failure on +that path — a missing certificate store, DNS, a timeout, a middlebox — is swallowed and +logged at DEBUG (:mod:`meshterm.services.basemap`), and ``log_level`` defaults to WARNING, +so a reader whose tiles never arrive sees a blank ground and no explanation anywhere. + +This script is that explanation. It is standalone and stdlib-only so it can be pasted onto +any machine running any version of MeshTerm, and it prints one verdict with the fix. + +It runs under MeshTerm's *own* interpreter, which is the one thing that matters here: a Mac +routinely has three Pythons, only one of them is the app's, and only that one's certificate +store is the one the map actually uses. Started under any other, it finds the installed +``meshterm`` command, reads the interpreter out of its shebang and hands itself over — so +the user needs no idea which Python they installed into. Save it to a file rather than +piping it into ``python3 -``: with no file on disk there is nothing to hand over. + +The decisive test is an A/B. ``curl`` verifies against the operating system's trust store +and Python verifies against its own, so curl succeeding where Python fails means the +certificates are at fault and nothing about the network is — the one comparison that +separates the two causes that look identical from inside the app. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import platform +import shutil +import socket +import ssl +import subprocess +import sys +import urllib.error +import urllib.request +from pathlib import Path +from typing import NamedTuple + +#: Where tiles come from unless the preference says otherwise. Kept in step with +#: ``basemap.DEFAULT_TILEJSON_URL`` by hand: this script may run against an install whose +#: ``meshterm`` it cannot import, so it cannot read the constant it is mirroring. +FALLBACK_TILEJSON_URL = "https://tiles.openfreemap.org/planet" + +#: Per-request timeout, matching ``BasemapSource``'s own so that a link which is merely +#: slow fails here for the same reason it fails in the app rather than for a stricter one. +TIMEOUT = 12.0 + +#: What the app sends, so a source that filters by agent treats this run the same way. +USER_AGENT = "MeshTerm-basemap-doctor (+https://github.com/jpmartineau/MeshTerm)" + +#: A tile that exists: zoom 9 over central Poland. The TileJSON resolving is only half the +#: path — the tiles themselves are often a different host — so the check fetches real +#: geometry before it reports success. +SAMPLE_TILE = (9, 285, 167) + + +def heading(text: str) -> None: + """Print a section heading.""" + print(f"\n=== {text} ===") + + +def line(label: str, value: object) -> None: + """Print one aligned ``label: value`` fact.""" + print(f" {label:<20} {value}") + + +def find_meshterm() -> tuple[str | None, str | None, str | None]: + """Report MeshTerm's version, tile URL and config dir, if this interpreter has them. + + Returns: + ``(version, tilejson_url, config_dir)``, each ``None`` where it could not be read. + """ + try: + import meshterm + from meshterm.core.config import default_config_dir + from meshterm.core.preferences import Preferences + except Exception: # noqa: BLE001 - any import failure means "not this interpreter" + return None, None, None + config_dir = default_config_dir() + url = None + try: + url = Preferences.load(config_dir / "preferences.toml").basemap_tilejson_url + except Exception: # noqa: BLE001 - an unreadable preferences file is itself a finding + pass + return getattr(meshterm, "__version__", "?"), url, str(config_dir) + + +#: Set on the re-executed child, so a second interpreter that still cannot import MeshTerm +#: gets on with the diagnosis instead of handing itself on forever. +REEXEC_ENV = "MESHTERM_DOCTOR_REEXEC" + + +def meshterm_interpreter() -> str | None: + """The interpreter the installed ``meshterm`` command runs under, if it can be read. + + A console script on Unix is a text file whose shebang names the exact interpreter the + app was installed into — and that interpreter's certificate store is the one the map + actually uses, which no other Python on the machine can speak for. On Windows the + console script is a binary launcher with no shebang, so this finds nothing and the + diagnosis proceeds with a banner saying which interpreter it did use. + """ + script = shutil.which("meshterm") + if script is None: + return None + try: + with open(script, "rb") as fh: + first = fh.readline(512) + except OSError: + return None + if not first.startswith(b"#!"): + return None + # A shebang may carry arguments (``#!/usr/bin/env python3``); the interpreter is the + # first word, and an ``env`` line names no absolute path worth re-execing. + interpreter = first[2:].decode("utf-8", "replace").strip().split(" ")[0] + return interpreter if interpreter and Path(interpreter).is_file() else None + + +def reexec_under_meshterm() -> None: + """Hand this script to MeshTerm's own interpreter when it is not the one running it. + + The user should not have to know which of their three Pythons the app was installed + into — that detail is exactly what the failure is made of. + """ + if os.environ.get(REEXEC_ENV): + return + try: + if importlib.util.find_spec("meshterm") is not None: + return + except (ImportError, ValueError): # a broken or shadowed package: keep looking + pass + target = meshterm_interpreter() + here = Path(globals().get("__file__", "")).resolve() if globals().get("__file__") else None + if target is None or here is None or not here.is_file(): + return + if Path(target).resolve() == Path(sys.executable).resolve(): + return + print(f" (handing over to MeshTerm's own interpreter: {target})") + # The child inherits this stdout, so the parent's buffer has to go out first or the + # banner lands after the report it introduces. Block buffering makes that certain the + # moment the output is piped into a file or a paste, which is how it is meant to + # travel. (``os.execve`` would avoid the extra process, but on Windows it is not a + # real exec: it spawns and exits, and segfaults outright under a redirect.) + sys.stdout.flush() + child = subprocess.run([target, str(here), *sys.argv[1:]], env={**os.environ, REEXEC_ENV: "1"}) + sys.exit(child.returncode) + + +def exists_marker(path: str) -> str: + """Return a marker saying whether ``path`` is actually on disk.""" + return "" if Path(path).exists() else " <- MISSING" + + +def describe_env() -> tuple[str | None, str | None]: + """Print interpreter, platform and MeshTerm facts; return the tile URL and config dir.""" + heading("This machine") + line("Python", sys.version.split()[0]) + line("Interpreter", sys.executable) + line("OS", f"{platform.system()} {platform.release()} ({platform.machine()})") + line("TERM", os.environ.get("TERM", "(unset)")) + # Not a tile question, but it is the other thing that makes a map look wrong and it + # costs nothing to answer in the same round trip: no COLORTERM means 256 colours, and + # the basemap's greens and blues are what collide when they are quantized. + # Windows is exempt: prompt_toolkit's Windows output reports 24-bit outright, so an + # unset COLORTERM there says nothing. Everywhere else it is the whole of the evidence. + unset = "(unset)" if os.name == "nt" else "(unset) <- only 256 colours" + line("COLORTERM", os.environ.get("COLORTERM") or unset) + + version, url, config_dir = find_meshterm() + if version is None: + line("MeshTerm", "NOT importable here <- probably the wrong interpreter") + else: + line("MeshTerm", version) + line("Config dir", config_dir) + return url, config_dir + + +def describe_trust() -> None: + """Print the certificate store this Python would verify a tile server against.""" + heading("Certificate store") + line("OpenSSL", ssl.OPENSSL_VERSION) + paths = ssl.get_default_verify_paths() + for label, value in (("CA file", paths.cafile), ("CA path", paths.capath)): + line(label, f"{value}{exists_marker(value)}" if value else "(none)") + try: + import certifi + + line("certifi", certifi.where()) + except ImportError: + line("certifi", "not installed") + for var in ( + "SSL_CERT_FILE", + "SSL_CERT_DIR", + "REQUESTS_CA_BUNDLE", + "HTTPS_PROXY", + "https_proxy", + "ALL_PROXY", + ): + if os.environ.get(var): + line(var, os.environ[var]) + loaded = len(ssl.create_default_context().get_ca_certs()) + line("Certs loaded", loaded if loaded else "0 <- nothing to verify against") + + +#: ``curl`` exit codes that mean TLS, as opposed to never getting that far. The whole +#: value of asking curl is that it verifies against the *operating system's* trust store, +#: so "curl got a certificate it disliked" and "curl could not reach the host" are +#: opposite answers and must not be collapsed into "curl failed". +CURL_TLS_CODES = frozenset({35, 51, 58, 59, 60, 66, 77, 83, 90, 91}) + + +class Curl(NamedTuple): + """What the system ``curl`` made of the same URL. + + Attributes: + verdict: ``ok`` (the server answered, whatever its status), ``tls`` (a certificate + or handshake failure), ``unreachable`` (never got that far), or ``absent`` + (no ``curl`` on this machine to ask). + detail: The status or error, for the transcript. + """ + + verdict: str + detail: str + + +def curl_says(url: str) -> Curl: + """Fetch ``url`` with the system ``curl``, which verifies against the OS trust store.""" + curl = shutil.which("curl") + if curl is None: + return Curl("absent", "no curl on this machine to compare against") + argv = [ + curl, + "-sS", + "-o", + os.devnull, + "-w", + "%{http_code}", + "--max-time", + str(int(TIMEOUT)), + url, + ] + try: + proc = subprocess.run(argv, capture_output=True, text=True, timeout=TIMEOUT + 5) + except (subprocess.TimeoutExpired, OSError): + return Curl("unreachable", "timed out") + if proc.returncode == 0: + # Any HTTP status at all means the transport worked, which is the only thing this + # comparison is asking about - a 301 or a 404 still proves TLS and the route. + return Curl("ok", f"HTTP {proc.stdout.strip()}") + kind = "tls" if proc.returncode in CURL_TLS_CODES else "unreachable" + return Curl(kind, f"exit {proc.returncode}: {proc.stderr.strip() or 'no detail'}") + + +def fetch(url: str) -> tuple[bool, object]: + """GET ``url`` exactly the way the app does. Returns ``(ok, body-or-exception)``.""" + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + try: + with urllib.request.urlopen(req, timeout=TIMEOUT) as resp: + return True, resp.read() + except Exception as exc: # noqa: BLE001 - classifying the failure is the whole job + return False, exc + + +#: What to do about a certificate store this Python cannot verify with. The macOS case is +#: first because it is the one that reaches a user who did nothing wrong: a python.org +#: install ships with an empty store until its own installer command is run. +CERT_FIX = ( + "Give this Python a certificate store. On macOS a python.org install ships\n" + " without one until you run its own command:\n" + " open /Applications/Python*/Install\\ Certificates.command\n" + " Or point it at certifi's, in the environment MeshTerm runs in:\n" + " pip install certifi\n" + ' export SSL_CERT_FILE="$(python3 -m certifi)" # then relaunch MeshTerm' +) + +#: What to do when TLS is being terminated in the middle rather than at the tile server. +INTERCEPT_FIX = ( + "Trust the interceptor's root certificate in this Python (SSL_CERT_FILE can\n" + " point at a bundle that includes it), or exempt the tile host from HTTPS\n" + " inspection. MeshTerm has no way to accept a certificate it cannot verify." +) + + +def classify(exc: BaseException) -> tuple[str, str, str]: + """Turn a fetch exception into ``(kind, what happened, what to do about it)``. + + The *kind* is what the verdict reasons with: a certificate failure and a timeout point + at opposite halves of the world, and the ``curl`` comparison means something different + for each. + """ + reason = getattr(exc, "reason", exc) + if isinstance(exc, urllib.error.HTTPError): + return ( + "http", + f"the server answered HTTP {exc.code}", + "The network is fine. A 4xx means the tile URL is wrong (check the " + "'Map tiles' preference); a 5xx is the tile server itself.", + ) + if isinstance(reason, ssl.SSLCertVerificationError): + return ("cert", "this Python could not verify the tile server's certificate", CERT_FIX) + if isinstance(reason, ssl.SSLError): + return ("tls", "the TLS handshake failed", INTERCEPT_FIX) + if isinstance(reason, socket.gaierror): + return ( + "dns", + "the tile host does not resolve", + "DNS. Check the machine is online, and that no DNS filter or ad-blocker " + "is swallowing the tile host.", + ) + if isinstance(reason, TimeoutError): + return ( + "timeout", + "the connection timed out", + "The host is unreachable, or something is dropping the traffic silently: " + "a firewall, or a link too slow for a 12-second timeout.", + ) + if isinstance(reason, ConnectionResetError): + return ( + "reset", + "the connection was reset", + "Something in the path is cutting the connection - a firewall or a " + "middlebox rather than the tile server.", + ) + return ("other", f"of {type(reason).__name__}: {reason}", "") + + +def verdict(kind: str, cause: str, fix: str, curl: Curl) -> None: + """Print one coherent story from what Python saw and what ``curl`` saw. + + The two readings have to be reconciled rather than printed side by side: a certificate + Python rejects *and curl accepts* is MeshTerm's own trust store, while one they both + reject is the network re-signing traffic or a genuinely bad server - the same Python + exception, opposite causes, opposite fixes. + """ + print(f" The map cannot fetch tiles because {cause}.") + if kind in ("cert", "tls"): + if curl.verdict == "ok": + print(" The system curl reached that very URL, verifying against the OS trust") + print(" store, so the network, the ISP and the tile server are all innocent:") + print(" the fault is this Python's own certificate store.") + print(f"\n Fix: {CERT_FIX}") + return + if curl.verdict == "tls": + print(" The system curl refuses the same certificate, so this is not just") + print(" MeshTerm's trust store: either something on this network is") + print(" intercepting TLS and re-signing it, or the certificate really is bad.") + print(f"\n Fix: {INTERCEPT_FIX}") + return + if curl.verdict == "unreachable": + print(" The system curl could not reach the host at all, so start with the") + print(" network - a firewall, a proxy or a DNS filter - before the certificates.") + return + elif curl.verdict == "ok": + print(" The system curl reached that very URL, so the route exists and this") + print(" Python is going somewhere else - check the proxy variables above.") + elif curl.verdict in ("tls", "unreachable"): + print(" The system curl could not reach it either, so this is the network path") + print(" rather than MeshTerm - a firewall, a proxy, a DNS filter or an outage.") + if fix: + print(f"\n Fix: {fix}") + + +def count_cached(config_dir: str | None) -> None: + """Print how many tiles are already on disk, which the map can draw with no network.""" + if not config_dir: + return + tiles = Path(config_dir) / "tilecache" / "tiles" + if not tiles.exists(): + line("Tile cache", "empty - no tile has ever arrived") + return + line("Tile cache", f"{sum(1 for _ in tiles.rglob('*.pbf'))} tiles at {tiles.parent}") + + +def main() -> int: + """Run every check and print one verdict. Returns a process exit status.""" + print("MeshTerm basemap doctor") + reexec_under_meshterm() + url, config_dir = describe_env() + tilejson_url = url or FALLBACK_TILEJSON_URL + describe_trust() + + heading("Reaching the tile source") + line("Tile source", tilejson_url + ("" if url else " (default)")) + count_cached(config_dir) + + curl = curl_says(tilejson_url) + line("System curl", curl.detail) + + ok, result = fetch(tilejson_url) + line("This Python", "OK" if ok else f"FAILED - {type(result).__name__}") + + tile_ok = False + if ok: + try: + template = json.loads(result)["tiles"][0] + z, x, y = SAMPLE_TILE + tile_url = template.replace("{z}", str(z)).replace("{x}", str(x)).replace("{y}", str(y)) + tile_ok, tile_result = fetch(tile_url) + line("One real tile", f"OK, {len(tile_result)} bytes" if tile_ok else "FAILED") + if not tile_ok: + result = tile_result + except (ValueError, KeyError, IndexError) as exc: + line("TileJSON", f"unreadable: {exc}") + + heading("Verdict") + if ok and tile_ok: + print(" Tiles reach this machine, so the basemap is NOT a network problem.") + print(" If the map still looks wrong it is the terminal: COLORTERM unset means") + print(" 256 colours, and the basemap's greens and blues collide once they are") + print(" quantized. Apple's Terminal.app cannot do 24-bit colour at all - try") + print(" iTerm2, Ghostty or kitty.") + return 0 + + failure = result if isinstance(result, BaseException) else RuntimeError("no answer") + verdict(*classify(failure), curl) + prefs = Path(config_dir) / "preferences.toml" if config_dir else Path("~/.meshterm") + print("\n Meanwhile the map still works - it plots nodes without a basemap.") + print(" Per-request detail goes to the log if you set log_level = DEBUG in") + print(f" {prefs} and reopen the map.") + return 1 + + +if __name__ == "__main__": + sys.exit(main())