From 4afad090481fd6c7788f05188fcc6dbedd221f3d Mon Sep 17 00:00:00 2001 From: mikecarper Date: Mon, 26 Oct 2026 19:41:30 -0700 Subject: [PATCH] Add a guided USB-to-LoRa updater for repeaters and companions --- .github/workflows/run-unit-tests.yml | 8 + docs/lora_ota_automation.md | 3 + docs/ota_easy.md | 3 + docs/ota_station_g2_to_ikoka_stick.md | 3 + docs/ota_wizard.md | 174 +++++++ mkdocs.yml | 1 + tools/lora_ota/ota_wizard.py | 457 ++++++++++++++++ tools/lora_ota/ota_wizard_bundle.py | 158 ++++++ tools/lora_ota/ota_wizard_catalog.py | 466 +++++++++++++++++ tools/lora_ota/ota_wizard_companion.py | 550 ++++++++++++++++++++ tools/lora_ota/ota_wizard_packages.py | 210 ++++++++ tools/lora_ota/ota_wizard_repeater.py | 459 ++++++++++++++++ tools/lora_ota/ota_wizard_session.py | 499 ++++++++++++++++++ tools/lora_ota/requirements-wizard.txt | 4 + tools/lora_ota/test_ota_wizard.py | 225 ++++++++ tools/lora_ota/test_ota_wizard_bundle.py | 231 ++++++++ tools/lora_ota/test_ota_wizard_catalog.py | 244 +++++++++ tools/lora_ota/test_ota_wizard_companion.py | 464 +++++++++++++++++ tools/lora_ota/test_ota_wizard_packages.py | 248 +++++++++ tools/lora_ota/test_ota_wizard_repeater.py | 353 +++++++++++++ tools/lora_ota/test_ota_wizard_session.py | 322 ++++++++++++ 21 files changed, 5082 insertions(+) create mode 100644 docs/ota_wizard.md create mode 100644 tools/lora_ota/ota_wizard.py create mode 100644 tools/lora_ota/ota_wizard_bundle.py create mode 100644 tools/lora_ota/ota_wizard_catalog.py create mode 100644 tools/lora_ota/ota_wizard_companion.py create mode 100644 tools/lora_ota/ota_wizard_packages.py create mode 100644 tools/lora_ota/ota_wizard_repeater.py create mode 100644 tools/lora_ota/ota_wizard_session.py create mode 100644 tools/lora_ota/requirements-wizard.txt create mode 100644 tools/lora_ota/test_ota_wizard.py create mode 100644 tools/lora_ota/test_ota_wizard_bundle.py create mode 100644 tools/lora_ota/test_ota_wizard_catalog.py create mode 100644 tools/lora_ota/test_ota_wizard_companion.py create mode 100644 tools/lora_ota/test_ota_wizard_packages.py create mode 100644 tools/lora_ota/test_ota_wizard_repeater.py create mode 100644 tools/lora_ota/test_ota_wizard_session.py diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 1b63da757..5f0a83a94 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -324,6 +324,14 @@ jobs: - name: Verify OTA orchestration and automatic relay policy cleanup run: python3 -B -m unittest discover -s tools/lora_ota -p 'test_lora_ota*.py' -v + - name: Verify guided direct USB updates and source firmware controls + run: | + python3 -B -m unittest discover -s tools/lora_ota -p 'test_ota_wizard*.py' -v + python3 -B test/test_direct_ota_admin_client.py -v + python3 -B test/test_companion_direct_admin_guard.py -v + python3 -B test/test_repeater_ota_serial_control.py -v + python3 -B test/test_ota_zero_hop_route.py -v + - name: Verify all signed OTAFIX 2.4.6 release images env: MESHCORE_BOOTLOADER_MOTA_ZIP: ${{ runner.temp }}/OTAFIX-2.4.6-bootloader-mota.zip diff --git a/docs/lora_ota_automation.md b/docs/lora_ota_automation.md index 9cd3abe4b..7c28b64f1 100644 --- a/docs/lora_ota_automation.md +++ b/docs/lora_ota_automation.md @@ -4,6 +4,9 @@ # Scripted LoRa OTA from start to finish +For a guided zero-hop update through one USB Repeater or Full Companion, +use the [Python updater](ota_wizard.md). + For a manual example using a Pi-connected repeater and the app's admin console, see [Station G2 to Ikoka Stick](ota_station_g2_to_ikoka_stick.md). diff --git a/docs/ota_easy.md b/docs/ota_easy.md index 0251224ae..492be26dd 100644 --- a/docs/ota_easy.md +++ b/docs/ota_easy.md @@ -4,6 +4,9 @@ # Easy firmware updates over LoRa +For automatic board detection, firmware preparation, installation, and +settings recovery through one USB radio, use the [guided Python updater](ota_wizard.md). + For **[1.18.0.1 USA Cascade](releases/1.18.0.1.md)**, choose the exact board, role and storage layout in the [firmware picker](firmware_picker.md). For a new nRF52 installation, use the matching diff --git a/docs/ota_station_g2_to_ikoka_stick.md b/docs/ota_station_g2_to_ikoka_stick.md index 04df9d3be..35db64fe0 100644 --- a/docs/ota_station_g2_to_ikoka_stick.md +++ b/docs/ota_station_g2_to_ikoka_stick.md @@ -4,6 +4,9 @@ # LoRa OTA example: Station G2 on a Pi to an Ikoka Stick +For the automatic version using only the Pi's USB-connected source radio, +start with the [guided LoRa updater](ota_wizard.md). + Update the Ikoka Stick's bootloader to **OTAFIX 2.4.12**, then install **1.18.0.1 Full Repeater** firmware. The Station G2 stays connected to the Pi by USB and serves the files. Use the app's authenticated Repeater Admin console for the Ikoka diff --git a/docs/ota_wizard.md b/docs/ota_wizard.md new file mode 100644 index 000000000..0937670cc --- /dev/null +++ b/docs/ota_wizard.md @@ -0,0 +1,174 @@ + +

View this page on MeshCore Docs.

+ + +# Guided LoRa updates + +Run one Python command to update a directly reachable Repeater or Room Server. +The wizard identifies the board, chooses compatible firmware, prepares the +transfer, installs it, checks the result, and restores the original radio, +RX power-saving, and OTA settings. On supported nRF52 boards it also checks +for a newer compatible signed bootloader and includes that update in the plan. + +## Connect a source radio + +Use either connection: + +| Computer connection | Source radio | Destination | +| --- | --- | --- | +| Raspberry Pi → USB | ESP32 Repeater with OTA and `ota.peer` support | Repeater or Room Server | +| Computer or Pi → USB | Full Companion with USB OTA source and direct admin support | Repeater or Room Server | + +A **Station G2 Repeater connected to a Pi** can manage and serve an update to +an **Ikoka Stick Repeater** through that single USB connection. It does not +need a second local radio or the phone app. Install a source firmware build +containing `ota.peer` first; older Repeater firmware cannot manage the remote +node through USB. Check with `ota.peer status` in its USB console. + +The Full Companion source also needs a current build containing the wizard's +direct admin guard. An older Full Companion may serve OTA files but cannot +provide this guarded admin connection; the wizard will ask for a source +firmware upgrade instead of proceeding with that connection. + +The source and destination must already use the same main-channel settings +and hear each other directly. The destination needs working LoRa OTA support +and its admin password. A stock nRF52 bootloader may need a matching local +installation before it can accept remote updates. + +The wizard sends discovery, admin commands, and OTA traffic directly with no +relay hops. An older destination may put its first login reply in a flood +envelope; the wizard accepts it only if it arrives with zero hops, establishes +a direct return route, then requires a fresh direct identity check. + +Close other apps using the USB port before starting. + +## Start + +From the MeshCore checkout: + +```bash +python3 tools/lora_ota/ota_wizard.py +``` + +On Windows, use `py` in place of `python3`. The first run installs USB and +verification dependencies in a private Python environment. On Raspberry Pi OS, +install `python3-venv` if Python reports that it is missing. + +The wizard selects the USB radio and destination automatically when there is +only one of each. Otherwise, choose from the displayed list. Enter the +destination admin password once, choose firmware, then review the plan and +press Enter to start. Password entry is hidden and passwords are never saved. + +Firmware choices are: + +- **Published GitHub firmware:** the default, matched to the exact board and + role. A running version that is already current is left in place. +- **Custom firmware file:** an application DFU ZIP, BIN, HEX, full mOTA, or a + prepared update bundle. +- **Build custom Full firmware from source:** offered on a computer. This + needs the firmware build tools and Bash; it is not offered by default on + a Pi. The generated version is newer than the detected installed version. + +The displayed plan includes package sizes, firmware version, and any +bootloader or trusted update signer being installed. After a reboot, the +wizard checks the running firmware hash and version, or the bootloader's +identity and version, before reporting success. + +## Differential or full updates + +The wizard looks for the **exact running firmware** in its cache and published +GitHub assets. It uses a differential package when it has a compatible builder, +can match that exact base, and the resulting package is smaller. A file with +the same version label but a different body hash is not a usable base. + +On a computer, the differential builder can be prepared automatically with a +compatible Rust toolchain. On a Pi without a builder, the default is a full +update made by Python without compiling. This requires destination storage +that supports full updates, such as the Ikoka Stick's QSPI flash. + +An internal-flash nRF52 destination that requires differential installation +cannot use that full fallback. Prepare its bundle on a computer instead: + +1. On the Pi, inspect the connected radios: + + ```bash + python3 tools/lora_ota/ota_wizard.py --check + ``` + + Copy the JSON file shown after **Preparation request:** to the computer. + It contains public board information and the exact running firmware hash, + without the admin password. + +2. On the computer, prepare the update using the request, new firmware, and + the exact firmware currently running on the destination: + + ```bash + python3 tools/lora_ota/ota_wizard.py --prepare-only \ + --request DEVICE.json --firmware NEW_FIRMWARE.zip \ + --base RUNNING_FIRMWARE.zip --output update.ota.zip + ``` + + No radio is opened. If the compatible differential builder and all input + files are already present, this also works without network access. Add + `--motatool /path/to/motatool` to use an existing builder. + +3. Copy `update.ota.zip` back to the Pi and run: + + ```bash + python3 tools/lora_ota/ota_wizard.py --firmware update.ota.zip + ``` + + The live destination is checked again before installation. A changed base + hash or a different board stops the differential update. + +For an offline full bundle on a destination that supports it, add `--full` +to the preparation command. This path needs neither the differential builder +nor USB dependencies. + +## Useful options + +| Option | Purpose | +| --- | --- | +| `--check` | Authenticate and inspect OTA support without downloading or installing firmware | +| `--port /dev/ttyUSB0` | Choose a USB port; Windows example: `--port COM5` | +| `--mode repeater` or `--mode companion` | Select the source type if automatic detection is unsuitable | +| `--target PUBLIC_KEY` | Choose a destination by its full 64-character key; it must still answer direct discovery | +| `--firmware FILE` | Use a local firmware image or prepared bundle | +| `--base FILE` | Supply the exact running firmware for differential preparation | +| `--full` | Choose a full application transfer when the destination supports it | +| `--build` | Build custom Full firmware from this checkout | +| `--version 1.18.0.3-custom` | Set the custom build version; it must be newer than the running version | +| `--no-bootloader` | Update application firmware only | +| `--cache DIRECTORY` | Choose where packages, public requests, and recovery records are saved | +| `--yes` | Accept the displayed plan; required choices and credentials must still be supplied | + +For unattended use, provide `MESHCORE_ADMIN_PASSWORD` in the process environment +and pass the port, source type, and target key. The wizard does not write that +password into its preferences, recovery records, or custom build environment. + +## If the connection is interrupted + +Keep the same source radio connected and run the wizard again. It checks the +saved source and destination public keys before restoring settings from an +unfinished update. To restore settings and exit: + +```bash +python3 tools/lora_ota/ota_wizard.py --recover --target PUBLIC_KEY +``` + +Keep the recovery record until the wizard reports that restoration is +complete. A lost install reply is followed by a running-image check; the +wizard does not blindly repeat an install command. Temporary radio windows +use the existing main-channel settings, so an interrupted process does not +leave the destination on a separate update frequency. + +The cache is normally `~/.cache/meshcore/ota` on Linux, +`~/Library/Caches/MeshCore/ota` on macOS, or `%LOCALAPPDATA%\MeshCore\ota` on +Windows. It contains firmware, public preparation records, selected board +keys, and restoration settings. Trusted signer keys added for an update remain +installed on the destination. + +For the individual commands and a manual board-by-board example, see +[Station G2 to Ikoka Stick](ota_station_g2_to_ikoka_stick.md). For routed +updates and explicit multi-node setup, use the +[scripted LoRa OTA guide](lora_ota_automation.md). diff --git a/mkdocs.yml b/mkdocs.yml index bd9545c08..6303c01b1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -64,6 +64,7 @@ nav: - upgrading_1.17.1.8_to_1.17.1.9.md - ota_user_guide.md - ota_easy.md + - ota_wizard.md - lora_ota_automation.md - ota_station_g2_to_ikoka_stick.md - ota_nrf52_bootloader_update.md diff --git a/tools/lora_ota/ota_wizard.py b/tools/lora_ota/ota_wizard.py new file mode 100644 index 000000000..4ee6f4546 --- /dev/null +++ b/tools/lora_ota/ota_wizard.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""Interactive, direct LoRa updates through one USB repeater or Full Companion.""" +from __future__ import annotations + +import argparse +import asyncio +import getpass +import importlib.util +import importlib.metadata +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import time +import venv + +import lora_ota as ota +from ota_wizard_catalog import GitHubCatalog, ImageCache, download_asset, load_application +from ota_wizard_packages import (find_builder, find_running_image, install_builder, + prepare_application, prepare_bootloader, small_host) +from ota_wizard_session import DirectSession +from ota_wizard_bundle import (load_bundle, load_ready_application, load_request, + save_bundle, save_request) + +ROOT = Path(__file__).resolve().parents[2] + + +def cache_directory() -> Path: + if os.name == "nt": + return Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData/Local")) / "MeshCore/ota" + if sys.platform == "darwin": + return Path.home() / "Library/Caches/MeshCore/ota" + return Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "meshcore/ota" + + +def ask(prompt: str, default: str | None = None) -> str: + if not sys.stdin.isatty(): + raise ota.OtaError("This choice needs an interactive terminal; supply the corresponding command-line option.") + value = input(prompt + (f" [{default}]" if default is not None else "") + ": ").strip() + return value or (default if default is not None else "") + + +def select(prompt: str, choices: list[str], default: int = 0) -> int: + for index, value in enumerate(choices, 1): + print(f" {index}. {value}") + while True: + value = ask(prompt, str(default + 1)) + if value.isdigit() and 1 <= int(value) <= len(choices): + return int(value) - 1 + print("Enter one of the numbers above.") + + +def arguments(argv=None): + parser = argparse.ArgumentParser(description=__doc__, + epilog="No passwords are saved. All updater transmissions use a direct, zero-hop route.") + parser.add_argument("--port", help="USB port; selected automatically when only one radio is connected") + parser.add_argument("--mode", choices=("auto", "repeater", "companion"), default="auto") + parser.add_argument("--target", help="complete destination public key; otherwise discover nearby repeaters") + parser.add_argument("--firmware", type=Path, help="custom application DFU ZIP, BIN, HEX, full mOTA, or prepared OTA bundle") + parser.add_argument("--base", type=Path, help="original running firmware, checked against the destination hash") + parser.add_argument("--full", action="store_true", help="use a full update when the destination supports it") + parser.add_argument("--build", action="store_true", help="build custom Full firmware from this source checkout") + parser.add_argument("--source-tree", type=Path, default=ROOT) + parser.add_argument("--version", help="version for a custom build; default is newer than the running version") + parser.add_argument("--motatool", help="existing differential builder executable") + parser.add_argument("--cache", type=Path, default=cache_directory()) + parser.add_argument("--no-bootloader", action="store_true", help="update only application firmware") + parser.add_argument("--no-setup", action="store_true", help="do not create a private Python dependency environment") + parser.add_argument("--recover", action="store_true", help="restore settings from an interrupted update, then exit") + parser.add_argument("--yes", action="store_true", help="accept the displayed update plan; does not guess missing credentials") + parser.add_argument("--runtime-ready", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--check", action="store_true", help="inspect source and destination without preparing or installing an update") + parser.add_argument("--prepare-only", action="store_true", help="prepare a portable update bundle on a computer without opening radios") + parser.add_argument("--request", type=Path, help="destination preparation request saved by a previous live inspection") + parser.add_argument("--output", type=Path, help="output .zip for --prepare-only") + return parser.parse_args(argv) + + +def ensure_dependencies(args, argv) -> None: + required = ("serial", "meshcore", "cryptography") + missing = [name for name in required if importlib.util.find_spec(name) is None] + if "meshcore" not in missing: + try: + if importlib.metadata.version("meshcore") != "2.3.9.1": + missing.append("tested meshcore version") + except importlib.metadata.PackageNotFoundError: + missing.append("meshcore package metadata") + if not missing: + return + if args.no_setup or args.runtime_ready: + raise ota.OtaError("Install the USB updater dependencies: python -m pip install -r " + + str(Path(__file__).with_name("requirements-wizard.txt"))) + print("First run: preparing USB and update-verification support in a private Python environment.") + environment = args.cache / "python" / f"{sys.version_info.major}.{sys.version_info.minor}" + python = environment / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + if not python.is_file(): + try: + venv.EnvBuilder(with_pip=True).create(environment) + except Exception as exc: + raise ota.OtaError("Python's venv support is required. On Raspberry Pi OS, install python3-venv.") from exc + log = environment / "setup.log" + with log.open("w", encoding="utf-8") as output: + completed = subprocess.run([str(python), "-m", "pip", "install", "--disable-pip-version-check", + "--only-binary=:all:", "-r", + str(Path(__file__).with_name("requirements-wizard.txt"))], + stdout=output, stderr=subprocess.STDOUT, timeout=600) + if completed.returncode: + raise ota.OtaError(f"USB support could not be installed without compiling dependencies. Details: {log}") + os.execv(str(python), [str(python), str(Path(__file__).resolve()), "--runtime-ready", *argv]) + + +def choose_port(explicit: str | None, remembered: dict | None = None) -> str: + if explicit: + return explicit + from serial.tools import list_ports + ports = [port for port in list_ports.comports() if port.vid is not None] + if not ports: + raise ota.OtaError("Connect the source repeater or Full Companion by USB, then run this command again.") + if len(ports) == 1: + return ports[0].device + default_port = next((index for index, port in enumerate(ports) + if port.device == (remembered or {}).get("port")), 0) + index = select("USB radio", [f"{port.description} — {port.device}" for port in ports], default_port) + return ports[index].device + + +def remembered_choices(cache: Path) -> dict: + try: + value = json.loads(ota.read_bounded_file(cache / "choices.json", 8192, "remembered choices")) + return value if isinstance(value, dict) else {} + except (OSError, ValueError, ota.OtaError): + return {} + + +def detect_mode(port: str) -> str: + """A harmless text probe avoids feeding Binary commands to a repeater CLI.""" + import serial + connection = serial.Serial(port=None, baudrate=115200, timeout=.1, write_timeout=2) + connection.dtr, connection.rts = True, False + connection.port = port + try: + connection.open() + time.sleep(.75) + connection.write(b"\rver\rota status\r") + deadline = time.monotonic() + 3 + data = bytearray() + while time.monotonic() < deadline and len(data) < 16384: + data.extend(connection.read(1024)) + text = data.decode("utf-8", "replace") + if "companion" in text.lower() or "ota seeder" in text.lower(): + return "companion" + if re.search(r"(?m)(?:->\s*|^\s*>?\s*)v\d+\.\d+", text): + return "repeater" + return "companion" # A Binary Companion does not answer text probes. + finally: + connection.close() + + +def save_preferences(path: Path, **values) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps(values, indent=2) + "\n", encoding="utf-8") + temporary.replace(path) + + +def user_progress(message: str) -> None: + """Keep raw wire diagnostics in recovery records, not the normal UI.""" + if message.startswith("OTA "): + count = re.search(r"\b(\d+)/(\d+)\b", message) + if "ready to install" in message.lower(): + print("Download complete; checking the package before installation.") + elif count: + have, total = map(int, count.groups()) + print(f"Downloading: {100 * have // total if total else 0}% ({have}/{total} blocks)") + else: + print("Waiting for the destination's download status…") + else: + print(message) + + +def custom_build(args, destination, target_env: str) -> Path: + source = args.source_tree.resolve() + if not (source / "build.sh").is_file(): + raise ota.OtaError("Custom builds need a MeshCore source checkout; set --source-tree to that directory.") + if not re.fullmatch(r"[A-Za-z0-9_]+", target_env): + raise ota.OtaError("The detected firmware target cannot be used as a build environment.") + version = args.version or ota.format_version(destination.version + 1) + "-custom" + packed_version = ota.extract_reply_version(version) + if packed_version is None or packed_version <= destination.version: + raise ota.OtaError("The custom build version must be newer than the running version.") + output = args.cache.resolve() / "custom" / (time.strftime("%Y%m%d-%H%M%S") + "-" + target_env) + output.mkdir(parents=True, exist_ok=False) + # build.sh holds the repository's build lock and runs one PlatformIO process. + command = ["bash", "build.sh", "build-firmware", target_env, + "--build-profile", "full", "--radio-preset", "target", + "--profile", "cascade", "--firmware-version", version] + if destination.platform == "esp32": + command += ["--full-exact"] + environment = {key: value for key, value in os.environ.items() + if key not in ("MESHCORE_ADMIN_PASSWORD", "MESHCORE_OTA_PASSWORD")} + environment["OUTPUT_DIR"] = output.as_posix() + if os.name == "nt" and shutil.which("wsl.exe"): + def linux_path(path: Path) -> str: + mapped = subprocess.run(["wsl.exe", "-e", "wslpath", "-a", str(path)], + capture_output=True, text=True, timeout=15) + if mapped.returncode or not mapped.stdout.strip().startswith("/"): + raise ota.OtaError("Custom builds need a working WSL installation. Run this updater inside WSL, or choose a firmware file.") + return mapped.stdout.strip() + command = ["wsl.exe", "--cd", linux_path(source), "--exec", "env", + "-u", "MESHCORE_ADMIN_PASSWORD", "-u", "MESHCORE_OTA_PASSWORD", + "OUTPUT_DIR=" + linux_path(output), *command] + elif not shutil.which("bash"): + raise ota.OtaError("Custom builds need Bash and the firmware build tools. Use a published or local firmware file instead.") + log = output / "build.log" + print(f"Building custom Full firmware. This may download board tools; progress is saved to {log}.") + with log.open("w", encoding="utf-8") as stream: + completed = subprocess.run(command, cwd=source, env=environment, + stdout=stream, stderr=subprocess.STDOUT, timeout=7200) + if completed.returncode: + raise ota.OtaError(f"The custom build failed; its full log is in {log}.") + candidates = list(output.glob("*.zip")) + [path for path in output.glob("*.bin") if "merged" not in path.name] + for candidate in candidates: + try: + load_application(candidate, target_id=destination.target_id, hw_id=destination.hw_id) + return candidate + except ota.OtaError: + continue + raise ota.OtaError("The custom build did not produce a matching application image.") + + +async def run(args) -> int: + args.cache.mkdir(parents=True, exist_ok=True, mode=0o700) + if args.target and not re.fullmatch(r"[0-9A-Fa-f]{64}", args.target): + raise ota.OtaError("--target needs the complete destination public key.") + remembered = remembered_choices(args.cache) + port = choose_port(args.port, remembered) + mode = args.mode if args.mode != "auto" else await asyncio.to_thread(detect_mode, port) + if mode == "companion": + from ota_wizard_companion import CompanionTransport + link = CompanionTransport(port) + else: + from ota_wizard_repeater import RepeaterTransport + link = RepeaterTransport(port) + print(f"Connecting to the USB {mode} on {port}…") + await link.open() + session = None + try: + key = args.target + if not args.recover: + print("Looking for directly reachable repeaters and rooms…") + nearby = await link.discover() + if not nearby: + raise ota.OtaError("No direct destinations replied. Check both radios use the same channel, then retry.") + if key: + if not any(item["public_key"].lower() == key.lower() for item in nearby): + raise ota.OtaError("The selected public key did not answer direct discovery.") + else: + default_target = next((index for index, item in enumerate(nearby) + if item["public_key"].lower() == remembered.get("target")), 0) + selected = 0 if len(nearby) == 1 else select("Destination", [ + f"{'Room' if item.get('type') == 3 else 'Repeater'} — {item['public_key'][:12].upper()}" + for item in nearby], default_target) + key = nearby[selected]["public_key"] + elif not key: + key = remembered.get("target") or ask("Destination public key from the recovery record") + print(f"Destination: {key[:12].upper()} (zero-hop direct)") + password = os.environ.get("MESHCORE_ADMIN_PASSWORD") + if password is None: + if not sys.stdin.isatty(): + raise ota.OtaError("Set MESHCORE_ADMIN_PASSWORD for a noninteractive run.") + password = getpass.getpass("Destination admin password [Enter for default]: ") or "password" + journal = args.cache / "sessions" / (key.lower() + ".json") + previous = None + if journal.is_file(): + previous = json.loads(ota.read_bounded_file(journal, 64 * 1024, "recovery record")) + if not isinstance(previous, dict) or previous.get("format") != 1: + raise ota.OtaError("The interrupted update record has an unsupported format.") + if previous.get("phase") not in ("finished", "prepared", "inspecting"): + source_key = previous.get("source_key") + if (previous.get("target_key") != key.lower() or not isinstance(source_key, str) + or not isinstance(link.source_key, str) or source_key.lower() != link.source_key.lower()): + raise ota.OtaError("The unfinished recovery record belongs to a different source or destination.") + saved = previous.get("settings") + if not isinstance(saved, dict) or set(saved) != {"source", "target"}: + raise ota.OtaError("The interrupted update record has incomplete saved settings.") + for settings in saved.values(): + if (not isinstance(settings, dict) or not all( + isinstance(settings.get(field), str) and len(settings[field]) <= 2048 + for field in ("radio", "rxps", "ota"))): + raise ota.OtaError("The interrupted update record has invalid saved settings.") + session = DirectSession(link, key, password, journal, progress=user_progress) + if previous and previous.get("phase") not in ("finished", "prepared", "inspecting"): + print("Restoring the settings saved by the interrupted update first…") + session.saved = previous["settings"] + session.active = True + session._state = previous + failures = await session.cleanup() + if failures: + raise ota.OtaError("Some recovery settings could not be restored; see " + str(journal)) + session = DirectSession(link, key, password, journal, progress=user_progress) + if args.recover: + print("Recovery is complete.") + return 0 + destination = await session.inspect() + request_file = args.cache / "requests" / (key.lower() + ".json") + save_request(request_file, destination) + print(f"{destination.hw_id}: {destination.version_text}") + if args.check: + print("Direct authentication and OTA capability checks passed. No update was installed.") + print(f"Preparation request: {request_file}") + return 0 + await session.snapshot() + catalog = GitHubCatalog() + print("Checking published firmware for this exact board and role…") + try: + asset = await asyncio.to_thread(catalog.choose_asset, target_id=destination.target_id) + except ota.OtaError as exc: + asset = None + if not args.firmware and not args.build: + print(str(exc)) + choice = 2 if args.build else 1 if args.firmware else 0 + if not args.yes and not args.firmware and not args.build: + options = ["Published GitHub firmware (recommended)", "Custom firmware file"] + if not small_host(): + options.append("Build custom Full firmware from source") + default_choice = remembered.get("firmware_choice", 0 if asset else 1) + if remembered.get("target") != key.lower() or default_choice not in range(len(options)): + default_choice = 0 if asset else 1 + choice = select("Firmware", options, default_choice) + if choice == 0: + if not asset: + raise ota.OtaError("No published firmware matched this board; supply --firmware or choose a custom build.") + firmware = await asyncio.to_thread(download_asset, asset, args.cache / "downloads") + elif choice == 1: + default_file = remembered.get("firmware_file") if remembered.get("target") == key.lower() else None + firmware = args.firmware or Path(ask("Custom application ZIP, BIN, HEX, full mOTA, or prepared bundle", default_file)) + else: + if not asset: + raise ota.OtaError("A custom build needs the exact target environment; use a local firmware file for this board.") + if not args.version: + args.version = ota.format_version(max(destination.version, asset.version) + 1) + "-custom" + firmware = await asyncio.to_thread(custom_build, args, destination, asset.target_env) + prepared = await asyncio.to_thread(load_bundle, firmware, destination) + if prepared: + image, application = prepared + if args.full and not application.info.is_full: + application = await asyncio.to_thread(prepare_application, image, destination, args.cache, full=True) + else: + image = await asyncio.to_thread(load_application, firmware, + target_id=destination.target_id, hw_id=destination.hw_id) + application = await asyncio.to_thread(load_ready_application, firmware, image, destination) + ImageCache(args.cache).remember(image) + if image.fw_version > destination.version and application is None: + builder = find_builder(args.cache, args.motatool) + if not args.full and not builder and not small_host(): + builder = await asyncio.to_thread(install_builder, args.cache) + base = None + if not args.full and builder: + base = await asyncio.to_thread(find_running_image, destination, args.cache, catalog, + supplied=args.base) + if not builder: + print("Using a full update: no differential compiler setup is needed on this host.") + if not builder and destination.platform == "nrf52" and not destination.external_storage: + raise ota.OtaError( + "This internal-flash radio requires a differential package. Prepare it on a computer with " + f"--prepare-only --request {request_file} --firmware NEW_FIRMWARE --base RUNNING_FIRMWARE " + "--output update.ota.zip, then select that ZIP on the Pi.") + application = await asyncio.to_thread(prepare_application, image, destination, args.cache, + base=base, builder=builder, full=args.full) + elif image.fw_version <= destination.version: + application = None + print("The selected published/application version is already installed or older; keeping the running firmware.") + bootloader = None + if not args.no_bootloader and destination.platform == "nrf52": + print("Checking the exact-board bootloader update…") + bootloader = await asyncio.to_thread(prepare_bootloader, destination, args.cache) + if not application and not bootloader: + print("No newer compatible update is needed.") + return 0 + print("\nReady to update:") + print(f" Destination: {destination.hw_id} — {key[:12].upper()}") + if bootloader: + print(f" Bootloader: {bootloader.info.version}, signed exact-board image") + if application: + print(f" Firmware: {application.info.version}, {application.info.kind}, {len(application.blob):,} bytes") + if bootloader or application and application.signer: + signer = bootloader.signer if bootloader else application.signer + print(f" Trusted update signer: {signer[:16]}") + print(" Direct link only. Original radio and power-saving settings will be restored.") + if not args.yes and ask("Start", "Y").lower() not in ("y", "yes"): + print("Update cancelled.") + return 0 + for package in (bootloader, application): + if package: + destination = await session.install(package, destination) + print(f"Verified {package.info.kind} installation.") + save_preferences(args.cache / "choices.json", port=port, mode=mode, + target=key.lower(), hw=destination.hw_id, firmware_choice=choice, + firmware_file=str(firmware) if choice == 1 else None) + print("Update verified successfully.") + return 0 + finally: + cleanup_failed = False + if session: + failures = await session.cleanup() + if failures: + cleanup_failed = True + print(f"Some settings still need recovery. Record: {session.journal}", file=sys.stderr) + for command in failures: + print(" " + command, file=sys.stderr) + await link.close() + if cleanup_failed and sys.exc_info()[0] is None: + raise ota.OtaError("The update completed but settings recovery is still needed; see the saved record.") + + +def prepare_without_radios(args) -> int: + if not args.request or not args.firmware or not args.output: + raise ota.OtaError("Use --prepare-only --request DEVICE.json --firmware FILE --output update.ota.zip.") + destination = load_request(args.request) + image = load_application(args.firmware, target_id=destination.target_id, hw_id=destination.hw_id) + package = load_ready_application(args.firmware, image, destination) + if package is None: + catalog = GitHubCatalog() + builder = find_builder(args.cache, args.motatool) + if not builder and not args.full: + builder = install_builder(args.cache) + base = None + if builder and not args.full: + base = find_running_image(destination, args.cache, catalog, supplied=args.base) + package = prepare_application(image, destination, args.cache, base=base, builder=builder, full=args.full) + save_bundle(args.output, image, package) + print(f"Prepared {package.info.kind}: {args.output} ({len(package.blob):,} transfer bytes)") + print("Copy this ZIP to the Pi and select it as the custom firmware file.") + return 0 + + +def main(argv=None) -> int: + argv = sys.argv[1:] if argv is None else argv + args = arguments(argv) + try: + if args.prepare_only: + return prepare_without_radios(args) + ensure_dependencies(args, argv) + return asyncio.run(run(args)) + except KeyboardInterrupt: + print("\nStopped. Check the recovery record if the destination was unreachable during cleanup.", file=sys.stderr) + return 130 + except (ota.OtaError, RuntimeError, OSError, ValueError, subprocess.TimeoutExpired) as exc: + print(f"Update stopped: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/lora_ota/ota_wizard_bundle.py b/tools/lora_ota/ota_wizard_bundle.py new file mode 100644 index 000000000..809a796fd --- /dev/null +++ b/tools/lora_ota/ota_wizard_bundle.py @@ -0,0 +1,158 @@ +"""Portable, public-only preparation records and PC-to-Pi update bundles.""" +from __future__ import annotations + +from dataclasses import asdict, fields +import hashlib +import json +from pathlib import Path +import zipfile +import re + +import lora_ota as ota +from ota_wizard_session import Destination, Package, validate_package + + +def load_ready_application(path: Path, image: ota.EndFInfo, + destination: Destination) -> Package | None: + """Keep a supplied full mOTA intact, including its verified signing key.""" + if Path(path).suffix.lower() != ".mota": + return None + blob = ota.read_bounded_file(path, 64 * 1024 * 1024, "application mOTA") + info = ota.parse_mota(blob) + if info.is_bootloader or not info.is_full: + raise ota.OtaError("Choose a full application mOTA, not a delta or bootloader package.") + # The complete image, rather than caller-supplied EndF fields, determines + # the expected post-install body hash and binds every manifest identity. + expected = ota.parse_endf(image.image) + if (info.payload != expected.image + or info.image_hash != hashlib.sha256(expected.image).digest() + or info.image_size != len(expected.image) + or info.target_id != expected.target_id or info.hw_id != expected.hw_id + or info.fw_version != expected.fw_version): + raise ota.OtaError("The supplied mOTA does not match the selected complete firmware.") + lib = ota.bootloader_library() + try: + parsed = lib.parse_container(blob) + signer = parsed.manifest.signer_pubkey.hex().upper() if parsed.manifest.is_signed else None + package = Package(blob, expected.body_hash, signer) + validate_package(package, destination) + except ImportError as exc: + raise ota.OtaError("Install Python cryptography to verify this signed application mOTA.") from exc + except (ValueError, TypeError) as exc: + raise ota.OtaError("Invalid supplied application mOTA: " + str(exc)) from exc + return package + + +def save_request(path: Path, destination: Destination) -> None: + value = asdict(destination) + value["base_hash"] = destination.base_hash.hex() + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps({"format": 1, "destination": value}, indent=2) + "\n", encoding="utf-8") + temporary.replace(path) + + +def load_request(path: Path) -> Destination: + try: + value = json.loads(ota.read_bounded_file(path, 64 * 1024, "preparation request")) + if not isinstance(value, dict) or value.get("format") != 1 or not isinstance(value.get("destination"), dict): + raise ValueError("incompatible request format") + data = dict(value["destination"]) + if set(data) != {field.name for field in fields(Destination)}: + raise ValueError("missing or unknown destination fields") + if (not isinstance(data["public_key"], str) + or not re.fullmatch(r"[0-9A-Fa-f]{64}", data["public_key"]) + or int(data["public_key"], 16) == 0): + raise ValueError("invalid destination public key") + if not isinstance(data["hw_id"], str) or not re.fullmatch(r"[A-Za-z0-9_.-]{1,32}", data["hw_id"]): + raise ValueError("invalid board identity") + for name in ("target_id", "version"): + if type(data[name]) is not int or not 0 < data[name] <= 0xFFFFFFFF: + raise ValueError("invalid " + name) + if data["platform"] not in ("esp32", "nrf52"): + raise ValueError("unsupported platform") + if data["max_block_size"] not in (1024, 2048) or type(data["max_block_size"]) is not int: + raise ValueError("unsupported block size") + if type(data["codecs"]) is not int or not 0 < data["codecs"] <= 0xFF: + raise ValueError("invalid codec mask") + if any(type(data[name]) is not bool for name in ("external_storage", "sd_storage")): + raise ValueError("invalid storage flags") + if not isinstance(data["version_text"], str) or len(data["version_text"]) > 512: + raise ValueError("invalid version text") + if not isinstance(data["base_hash"], str) or not re.fullmatch(r"[0-9a-fA-F]{16}", data["base_hash"]): + raise ValueError("invalid running body hash") + data["base_hash"] = bytes.fromhex(data["base_hash"]) + if data["boot_version"] is not None and (type(data["boot_version"]) is not int or not 0 < data["boot_version"] <= 0xFFFFFFFF): + raise ValueError("invalid bootloader version") + if data["bootloader"] is not None: + boot = data["bootloader"] + if not isinstance(boot, dict) or set(boot) != {field.name for field in fields(ota.BootloaderStatus)}: + raise ValueError("invalid bootloader identity") + for name in ("board_id", "target_id", "crc32", "apply_abi", "storage_flags"): + if type(boot[name]) is not int or not 0 <= boot[name] <= 0xFFFFFFFF: + raise ValueError("invalid bootloader field") + if not isinstance(boot["device_name"], str) or not re.fullmatch(r"[A-Za-z0-9_]{1,32}", boot["device_name"]): + raise ValueError("invalid bootloader name") + lib = ota.bootloader_library() + if (lib.bootloader_target_id(boot["board_id"], boot["device_name"]) != boot["target_id"] + or lib.bootloader_hw_id(boot["board_id"], boot["device_name"]).rstrip(b"\0").decode() != boot["hw_id"]): + raise ValueError("inconsistent bootloader identity") + data["bootloader"] = ota.BootloaderStatus(**boot) + return Destination(**data) + except (ValueError, KeyError, TypeError, AttributeError) as exc: + raise ota.OtaError("Invalid OTA preparation request: " + str(exc)) from exc + + +def save_bundle(path: Path, image: ota.EndFInfo, package: Package) -> None: + info = package.info + if (info.is_bootloader or info.image_hash != hashlib.sha256(image.image).digest() + or info.target_id != image.target_id or info.hw_id != image.hw_id + or info.fw_version != image.fw_version or info.image_size != len(image.image)): + raise ota.OtaError("The prepared package and complete new firmware do not agree.") + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(".tmp") + with zipfile.ZipFile(temporary, "w", compression=zipfile.ZIP_STORED) as archive: + archive.writestr("ota-bundle.json", json.dumps({"format": 1, + "package_sha256": hashlib.sha256(package.blob).hexdigest()})) + archive.writestr("application.mota", package.blob) + archive.writestr("firmware.bin", image.image) + temporary.replace(path) + + +def load_bundle(path: Path, destination: Destination) -> tuple[ota.EndFInfo, Package] | None: + try: + return _load_bundle(path, destination) + except (zipfile.BadZipFile, KeyError, ValueError, TypeError, UnicodeError) as exc: + raise ota.OtaError("Invalid prepared OTA bundle: " + str(exc)) from exc + + +def _load_bundle(path: Path, destination: Destination) -> tuple[ota.EndFInfo, Package] | None: + if path.suffix.lower() != ".zip": + return None + ota.read_bounded_file(path, 64 * 1024 * 1024, "OTA bundle") + with zipfile.ZipFile(path) as archive: + if "ota-bundle.json" not in archive.namelist(): + return None + names = archive.namelist() + if len(names) != 3 or set(names) != {"ota-bundle.json", "application.mota", "firmware.bin"}: + raise ota.OtaError("The OTA preparation bundle has unexpected members.") + metadata = json.loads(ota.read_zip_member(archive, archive.getinfo("ota-bundle.json"))) + blob = ota.read_zip_member(archive, archive.getinfo("application.mota")) + image = ota.parse_endf(ota.read_zip_member(archive, archive.getinfo("firmware.bin"))) + if (not isinstance(metadata, dict) or metadata.get("format") != 1 + or metadata.get("package_sha256") != hashlib.sha256(blob).hexdigest()): + raise ota.OtaError("The prepared OTA bundle's checksum does not match.") + info = ota.parse_mota(blob) + if (info.image_hash != hashlib.sha256(image.image).digest() or info.image_size != len(image.image) + or info.target_id != image.target_id or info.hw_id != image.hw_id + or info.fw_version != image.fw_version): + raise ota.OtaError("The complete firmware does not match the prepared package.") + lib = ota.bootloader_library() + problems = lib.verify(lib.parse_container(blob)) + if problems: + raise ota.OtaError("Prepared OTA package integrity failed: " + "; ".join(problems)) + parsed = lib.parse_container(blob) + signer = parsed.manifest.signer_pubkey.hex().upper() if parsed.manifest.is_signed else None + package = Package(blob, image.body_hash, signer) + validate_package(package, destination) + return image, package diff --git a/tools/lora_ota/ota_wizard_catalog.py b/tools/lora_ota/ota_wizard_catalog.py new file mode 100644 index 000000000..d9e1d8c1d --- /dev/null +++ b/tools/lora_ota/ota_wizard_catalog.py @@ -0,0 +1,466 @@ +"""Published firmware, bounded downloads, and exact-image cache for the OTA UI. + +This module never opens a radio or builds firmware. Download names narrow the +choice; the application's EndF identity remains authoritative after download. +""" +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import tempfile +from typing import Callable +from urllib.parse import quote, urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener +import zipfile + +import lora_ota as ota + +DEFAULT_REPOSITORY = "mikecarper/MeshCore" +MAX_DOWNLOAD_BYTES = 64 * 1024 * 1024 +MAX_CATALOG_BYTES = 16 * 1024 * 1024 +MAX_ZIP_MEMBERS = 2048 +_FAMILY_PREFIXES = ("repeater-room", "utility", "logging", "lora-ota", "full-profiles", "companion") +_SAFE_FILENAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._+-]{0,239}") +_VERSION = re.compile(r"(?:^|-)v(\d+)\.(\d+)\.(\d+)(?:\.(\d+))?", re.I) + + +@dataclass(frozen=True) +class ReleaseAsset: + name: str + url: str + size: int + release_tag: str + version: int + target_env: str + profile: str + kind: str + digest: str | None = None + embedded_bootloader: bool = False + + @property + def target_id(self) -> int: + return int.from_bytes(hashlib.sha256(self.target_env.encode()).digest()[:4], "little") + + @property + def version_text(self) -> str: + return ota.format_version(self.version) + + +@dataclass(frozen=True) +class UpdatePlan: + kind: str + reason: str + + +def choose_update_strategy(*, platform: str, external_storage: bool, + exact_base: bool, delta_supported: bool, + low_power: bool = False, can_build_delta: bool | None = None, + prebuilt_delta: bool = False) -> UpdatePlan: + """Choose a compatible transfer without silently inventing a delta base. + + ``prebuilt_delta`` means its target and base were independently checked. + A ready builder can be used on any host; otherwise small hosts default to + full packaging, which needs no Rust or firmware compiler. + """ + if platform not in ("esp32", "nrf52"): + return UpdatePlan("unavailable", "This platform has no qualified LoRa installer.") + if delta_supported and prebuilt_delta: + return UpdatePlan("delta", "A prepared delta matches the running firmware.") + can_build = not low_power if can_build_delta is None else can_build_delta + if delta_supported and exact_base and can_build: + return UpdatePlan("delta", "The exact running image is available for a differential update.") + if platform == "esp32" or external_storage: + reason = ("The exact running image is unavailable; use a full update." + if not exact_base else + "A full update avoids installing or compiling a delta builder on this host." + if delta_supported and not can_build else + "This installer supports a full application update.") + return UpdatePlan("full", reason) + return UpdatePlan("unavailable", "Internal-flash nRF52 needs a compatible delta and its exact running image.") + + +def _repository(value: str) -> str: + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", value): + raise ota.OtaError("GitHub repository must be OWNER/REPOSITORY") + return value + + +def _version(value: str) -> int | None: + match = _VERSION.search(value) + if not match: + return None + parts = tuple(int(item or 0) for item in match.groups()) + if any(item > 255 for item in parts): + return None + return sum(item << shift for item, shift in zip(parts, (24, 16, 8, 0))) + + +def _release_family(tag: str) -> str | None: + if re.match(r"^v\d", tag, re.I): + return tag + prefixes = "|".join(re.escape(value) for value in _FAMILY_PREFIXES) + match = re.fullmatch(rf"(?:{prefixes})(?:-(?:[2-9]|[1-9][0-9]+))?-(v\d.*)", tag) + return match.group(1) if match else None + + +def _is_prerelease(release: dict) -> bool: + return bool(release.get("prerelease")) or bool(re.search( + r"(?:^|[-.])(?:alpha|beta|preview|rc)(?:[-.0-9]|$)", + str(release.get("tag_name", "")), re.I)) + + +def _valid_asset_url(url: str, repository: str, tag: str, name: str) -> bool: + expected = f"https://github.com/{repository}/releases/download/{quote(tag, safe='')}/{quote(name, safe='')}" + return url == expected + + +def _safe_redirect(url: str, *, api: bool = False) -> bool: + try: + parsed = urlsplit(url) + host = (parsed.hostname or "").lower() + return (parsed.scheme == "https" and not parsed.username and not parsed.password + and parsed.port in (None, 443) + and (host == "api.github.com" if api else + host == "github.com" or host.endswith(".githubusercontent.com"))) + except ValueError: + return False + + +class _GithubRedirects(HTTPRedirectHandler): + def __init__(self, *, api: bool): + super().__init__() + self.api = api + + def redirect_request(self, req, fp, code, msg, headers, newurl): + if not _safe_redirect(newurl, api=self.api): + raise ota.OtaError("GitHub redirected the download to an unexpected host") + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def _open(url: str, *, api: bool = False, opener: Callable | None = None): + request = Request(url, headers={"User-Agent": "MeshCore-OTA-Wizard", + "Accept": "application/vnd.github+json" if api else "application/octet-stream"}) + try: + response = (opener or build_opener(_GithubRedirects(api=api)).open)(request, timeout=30) + except (OSError, ValueError) as exc: + raise ota.OtaError(f"Could not download GitHub firmware: {exc}") from exc + final_url = response.geturl() + if not _safe_redirect(final_url, api=api): + response.close() + raise ota.OtaError("GitHub returned an unexpected download host") + return response + + +def parse_release_asset(asset: dict, release_tag: str, repository: str = DEFAULT_REPOSITORY) -> ReleaseAsset | None: + name = asset.get("name", "") + if not isinstance(name, str) or not _SAFE_FILENAME.fullmatch(name): + return None + suffix = Path(name).suffix.lower().lstrip(".") + if suffix not in ("zip", "bin", "hex", "mota"): + return None + stem = name[:-(len(suffix) + 1)] + version = _version(stem) + match = _VERSION.search(stem) + if version is None or match is None: + return None + published_target = stem[:match.start()].rstrip("-") + if published_target.endswith("-merged") or stem.endswith("-merged"): + return None + published_target = re.sub(r"-ota$", "", published_target) + embedded = bool(re.search(r"-with-bootloader-", published_target)) + profile_match = re.search(r"-(full|reduced)(?:-usb-wifi|-logging)?$", published_target, re.I) + profile = profile_match.group(1).lower() if profile_match else "standard" + target_env = published_target[:profile_match.start()] if profile_match else published_target + target_env = re.sub(r"-with-bootloader-.*$", "", target_env) + if not re.fullmatch(r"[A-Za-z0-9_.-]+", target_env): + return None + if not re.search(r"_(?:(?:repeater|repeatr|room_server|room_svr|sensor)(?:$|[_-])|companion_radio_)", target_env, re.I): + return None + url = asset.get("browser_download_url", "") + if not isinstance(url, str) or not _valid_asset_url(url, _repository(repository), release_tag, name): + return None + size = asset.get("size") + if not isinstance(size, int) or isinstance(size, bool) or not 0 < size <= MAX_DOWNLOAD_BYTES: + return None + digest = asset.get("digest") or None + if digest is not None and not re.fullmatch(r"sha256:[0-9a-fA-F]{64}", str(digest)): + return None + return ReleaseAsset(name, url, size, release_tag, version, target_env, profile, + suffix, digest.lower() if digest else None, embedded) + + +class GitHubCatalog: + def __init__(self, repository: str = DEFAULT_REPOSITORY, *, opener: Callable | None = None): + self.repository = _repository(repository) + self.opener = opener + self._assets: list[ReleaseAsset] | None = None + self._all_assets: list[ReleaseAsset] = [] + + def fetch_assets(self, *, include_prerelease: bool = False, max_pages: int = 3) -> list[ReleaseAsset]: + if not 1 <= max_pages <= 10: + raise ota.OtaError("Catalog page limit must be between 1 and 10") + releases: list[dict] = [] + for page in range(1, max_pages + 1): + # Release families contain thousands of per-board assets. Small + # pages keep GitHub's embedded asset metadata bounded on a Pi. + url = f"https://api.github.com/repos/{self.repository}/releases?per_page=10&page={page}" + with _open(url, api=True, opener=self.opener) as response: + raw = response.read(MAX_CATALOG_BYTES + 1) + if len(raw) > MAX_CATALOG_BYTES: + raise ota.OtaError("GitHub release catalog is unexpectedly large") + try: + rows = json.loads(raw) + except (ValueError, UnicodeError) as exc: + raise ota.OtaError("GitHub returned an invalid release catalog") from exc + if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows): + raise ota.OtaError("GitHub returned an invalid release catalog") + releases.extend(row for row in rows if not row.get("draft") + and (include_prerelease or not _is_prerelease(row))) + if len(rows) < 10: + break + mains = [row for row in releases if re.match(r"^v\d", str(row.get("tag_name", "")), re.I) + and _version(row["tag_name"]) is not None] + if not mains: + raise ota.OtaError("No published main firmware release was found") + main = max(mains, key=lambda row: (_version(row["tag_name"]), row.get("published_at") or "")) + family = main["tag_name"] + result: list[ReleaseAsset] = [] + historical: list[ReleaseAsset] = [] + for release in releases: + tag = str(release.get("tag_name", "")) + release_family = _release_family(tag) + if release_family is None: + continue + rows = release.get("assets", []) + if not isinstance(rows, list): + raise ota.OtaError("GitHub returned an invalid release asset list") + for row in rows: + if isinstance(row, dict): + candidate = parse_release_asset(row, tag, self.repository) + if candidate is not None: + historical.append(candidate) + if release_family == family: + result.append(candidate) + self._all_assets = historical + self._assets = result + return list(result) + + def candidates(self, target_env: str | None = None, *, target_id: int | None = None, + profile: str = "full", include_embedded: bool = False) -> list[ReleaseAsset]: + if target_env is None and target_id is None: + raise ota.OtaError("Select an exact firmware target") + assets = self.fetch_assets() if self._assets is None else self._assets + result = [row for row in assets if (target_env is None or row.target_env == target_env) + and (target_id is None or row.target_id == target_id) + and (include_embedded or not row.embedded_bootloader)] + # Prefer a declared Full profile; older releases used the bare target. + exact = [row for row in result if row.profile == profile] + result = exact or [row for row in result if row.profile == "standard"] + preference = {"mota": 0, "zip": 1, "bin": 2, "hex": 3} + return sorted(result, key=lambda row: (-row.version, preference[row.kind], row.name)) + + def choose_asset(self, target_env: str | None = None, **kwargs) -> ReleaseAsset: + choices = self.candidates(target_env, **kwargs) + if not choices: + raise ota.OtaError("No published application matches this exact board, role, and profile") + return choices[0] + + def base_candidates(self, target_env: str | None = None, *, target_id: int | None = None, + fw_version: int | None = None, limit: int = 8) -> list[ReleaseAsset]: + """Find possible old images; their downloaded EndF body hash MUST be checked.""" + if target_env is None and target_id is None: + raise ota.OtaError("Select an exact firmware target") + if not 1 <= limit <= 32: + raise ota.OtaError("Base candidate limit must be between 1 and 32") + if self._assets is None: + self.fetch_assets() + preference = {"mota": 0, "zip": 1, "bin": 2, "hex": 3} + matches = [row for row in self._all_assets + if not row.embedded_bootloader + and (target_env is None or row.target_env == target_env) + and (target_id is None or row.target_id == target_id) + and (fw_version is None or row.version == fw_version)] + return sorted(matches, key=lambda row: (-row.version, preference[row.kind], row.name))[:limit] + + +def _atomic_bytes(path: Path, data: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(prefix=".ota-", suffix=".tmp", dir=path.parent) + try: + with os.fdopen(fd, "wb") as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def download_asset(asset: ReleaseAsset, cache_dir: Path, *, opener: Callable | None = None) -> Path: + """Download exactly one public release asset, verifying GitHub's SHA256 when supplied.""" + if not _SAFE_FILENAME.fullmatch(asset.name) or not 0 < asset.size <= MAX_DOWNLOAD_BYTES: + raise ota.OtaError("Invalid release asset name or size") + parsed = urlsplit(asset.url) + parts = parsed.path.split("/") + if len(parts) != 7 or not _valid_asset_url(asset.url, "/".join(parts[1:3]), asset.release_tag, asset.name): + raise ota.OtaError("Release download URL does not match its asset") + if asset.digest and not re.fullmatch(r"sha256:[0-9a-fA-F]{64}", asset.digest): + raise ota.OtaError("Unsupported release asset digest") + cache_dir = Path(cache_dir) + cache_dir.mkdir(parents=True, exist_ok=True) + key = hashlib.sha256(asset.url.encode()).hexdigest()[:16] + path = cache_dir / f"{key}-{asset.name}" + metadata = path.with_name(path.name + ".json") + if path.is_file() and not path.is_symlink(): + data = ota.read_bounded_file(path, MAX_DOWNLOAD_BYTES, "cached release asset") + digest = hashlib.sha256(data).hexdigest() + try: + saved = json.loads(ota.read_bounded_file(metadata, 4096, "cached download metadata")) + except (OSError, ValueError, ota.OtaError): + saved = {} + if not isinstance(saved, dict): + saved = {} + expected = asset.digest.split(":", 1)[1] if asset.digest else saved.get("sha256") + if len(data) == asset.size and digest == expected and saved.get("url") == asset.url: + return path + data = bytearray() + try: + with _open(asset.url, opener=opener) as response: + while True: + block = response.read(65536) + if not block: + break + data.extend(block) + if len(data) > asset.size or len(data) > MAX_DOWNLOAD_BYTES: + raise ota.OtaError("Release download exceeded its declared size") + except OSError as exc: + raise ota.OtaError(f"Could not finish GitHub firmware download: {exc}") from exc + if len(data) != asset.size: + raise ota.OtaError("Release download is incomplete") + digest = hashlib.sha256(data).hexdigest() + if asset.digest and digest.lower() != asset.digest.split(":", 1)[1].lower(): + raise ota.OtaError("Release download SHA256 does not match GitHub") + _atomic_bytes(path, bytes(data)) + _atomic_bytes(metadata, json.dumps({"url": asset.url, "sha256": digest}).encode()) + return path + + +def _check_identity(identity: ota.EndFInfo, target_id: int | None, hw_id: str | None) -> ota.EndFInfo: + if target_id is not None and identity.target_id != target_id: + raise ota.OtaError(f"Application target {identity.target_id:08X} does not match {target_id:08X}") + if hw_id is not None and identity.hw_id != hw_id: + raise ota.OtaError(f"Application hardware {identity.hw_id!r} does not match {hw_id!r}") + return identity + + +def _application_bytes(raw: bytes, suffix: str) -> ota.EndFInfo: + if suffix == ".mota": + info = ota.parse_mota(raw) + if info.is_bootloader or not info.is_full: + raise ota.OtaError("An application image needs a full application mOTA, not a delta or bootloader") + raw = info.payload + elif suffix == ".hex": + raw = ota.parse_intel_hex(raw) + return ota.parse_endf(raw) + + +def load_application(path: Path, *, target_id: int | None = None, + hw_id: str | None = None) -> ota.EndFInfo: + """Read a validated raw application, including the BIN inside a Nordic DFU ZIP.""" + path = Path(path) + suffix = path.suffix.lower() + if suffix not in (".zip", ".bin", ".hex", ".mota"): + raise ota.OtaError("Choose an application BIN, HEX, DFU ZIP, or full mOTA") + if suffix != ".zip": + return _check_identity(_application_bytes(ota.read_bounded_file( + path, MAX_DOWNLOAD_BYTES, "application file"), suffix), target_id, hw_id) + ota.read_bounded_file(path, MAX_DOWNLOAD_BYTES, "application ZIP") + try: + with zipfile.ZipFile(path) as archive: + members = archive.infolist() + if len(members) > MAX_ZIP_MEMBERS: + raise ota.OtaError("Application ZIP contains too many members") + names = [member.filename for member in members] + if len(set(names)) != len(names): + raise ota.OtaError("Application ZIP contains duplicate member names") + manifest_members = [member for member in members if member.filename == "manifest.json"] + if manifest_members: + manifest = json.loads(ota.read_zip_member(archive, manifest_members[0])) + app = manifest.get("manifest", {}).get("application") + if not isinstance(app, dict) or not isinstance(app.get("bin_file"), str): + raise ota.OtaError("DFU ZIP has no application image") + name = app["bin_file"] + if (PurePosixPath(name).is_absolute() or ".." in PurePosixPath(name).parts + or "\\" in name or ":" in name): + raise ota.OtaError("DFU ZIP has an unsafe application member") + member = archive.getinfo(name) + return _check_identity(ota.parse_endf(ota.read_zip_member(archive, member)), target_id, hw_id) + matches: dict[bytes, ota.EndFInfo] = {} + for member in members: + member_suffix = Path(member.filename).suffix.lower() + if member_suffix not in (".bin", ".hex", ".mota") or member.is_dir(): + continue + try: + identity = _check_identity(_application_bytes( + ota.read_zip_member(archive, member), member_suffix), target_id, hw_id) + matches[hashlib.sha256(identity.image).digest()] = identity + except ota.OtaError: + continue + if len(matches) != 1: + raise ota.OtaError("Application ZIP must contain one unambiguous matching firmware image") + return next(iter(matches.values())) + except (OSError, KeyError, ValueError, TypeError, zipfile.BadZipFile) as exc: + raise ota.OtaError(f"Invalid application ZIP: {exc}") from exc + + +class ImageCache: + def __init__(self, cache_dir: Path): + self.directory = Path(cache_dir) / "images" + + def remember(self, identity: ota.EndFInfo) -> Path: + # Reparse callers' records rather than trusting manually constructed metadata. + identity = ota.parse_endf(identity.image) + path = self.directory / f"{identity.target_id:08X}" / f"{identity.body_hash.hex().upper()}.bin" + fingerprint = path.with_suffix(".sha256") + if path.is_file() and not path.is_symlink(): + try: + previous = load_application(path, target_id=identity.target_id) + digest = ota.read_bounded_file(fingerprint, 128, "cached image fingerprint").decode().strip() + except (ota.OtaError, UnicodeError): + previous = None + digest = "" + if (previous is not None and hashlib.sha256(previous.image).hexdigest() == digest + and previous.image != identity.image): + raise ota.OtaError("Cached body hash already identifies a different complete image") + _atomic_bytes(path, identity.image) + _atomic_bytes(fingerprint, hashlib.sha256(identity.image).hexdigest().encode()) + return path + + def store(self, path: Path, *, target_id: int | None = None, hw_id: str | None = None) -> Path: + return self.remember(load_application(path, target_id=target_id, hw_id=hw_id)) + + def find(self, base_hash: bytes | str, target_id: int, hw_id: str | None = None, + *, fw_version: int | None = None) -> Path | None: + if isinstance(base_hash, str): + if not re.fullmatch(r"[0-9a-fA-F]{16}", base_hash): + raise ota.OtaError("Running base hash must have 16 hexadecimal characters") + base_hash = bytes.fromhex(base_hash) + if len(base_hash) != 8: + raise ota.OtaError("Running base hash must contain eight bytes") + path = self.directory / f"{target_id:08X}" / f"{base_hash.hex().upper()}.bin" + if not path.is_file() or path.is_symlink(): + return None + try: + identity = load_application(path, target_id=target_id, hw_id=hw_id) + digest = ota.read_bounded_file(path.with_suffix(".sha256"), 128, + "cached image fingerprint").decode().strip() + except (ota.OtaError, UnicodeError): + return None + return path if (hashlib.sha256(identity.image).hexdigest() == digest and + identity.body_hash == base_hash and + (fw_version is None or identity.fw_version == fw_version)) else None diff --git a/tools/lora_ota/ota_wizard_companion.py b/tools/lora_ota/ota_wizard_companion.py new file mode 100644 index 000000000..00f2b6c73 --- /dev/null +++ b/tools/lora_ota/ota_wizard_companion.py @@ -0,0 +1,550 @@ +"""One-owner USB Companion transport for the interactive direct OTA updater. + +The SDK's Binary connection carries both admin commands and legacy mOTA radio +frames. USB folder mode is deliberately never entered: it excludes Binary +Companion traffic. Imports of the SDK and mOTA implementation are lazy. +""" +from __future__ import annotations + +import asyncio +from copy import deepcopy +import hashlib +import math +import re +import secrets +import string +import time +import zlib + +import lora_ota as ota + + +class CompanionError(ota.OtaError): + pass + + +def _key(value): + value = value.hex() if isinstance(value, bytes) else str(value) + if not re.fullmatch(r"[0-9a-fA-F]{64}", value): + raise CompanionError("A complete 32-byte public key is required") + return value.lower() + + +def _kind(event): + value = getattr(event, "type", None) + return getattr(value, "value", value) + + +def _payload(event): + value = getattr(event, "payload", {}) + return value if isinstance(value, dict) else {} + + +def raw_rx_bytes(payload): + """SDK RX_LOG_DATA.raw_hex includes SNR/RSSI; payload may be parser-modified.""" + try: + if "raw_hex" in payload: + raw = payload["raw_hex"] + raw = bytes.fromhex(raw) if isinstance(raw, str) else bytes(raw) + return raw[2:] if len(raw) >= 5 else b"" + raw = payload.get("payload", b"") + return bytes.fromhex(raw) if isinstance(raw, str) else bytes(raw) + except (TypeError, ValueError): + return b"" + + +def packet_airtime(length, radio, preamble=16): + """Conservative explicit-header LoRa airtime, seconds, including CRC.""" + _, bw, sf, cr = radio[:4] + symbol = 2 ** int(sf) / (float(bw) * 1000) + de = int(symbol >= .016) + payload_symbols = 8 + max(0, math.ceil( + (8 * length - 4 * sf + 28 + 16) / (4 * (sf - 2 * de)))) * cr + return (preamble + 4.25 + payload_symbols) * symbol + + +def _deflate(data): + compressor = zlib.compressobj(9, zlib.DEFLATED, -15) + return compressor.compress(data) + compressor.flush() + + +class CompanionTransport: + def __init__(self, port, baud=115200, *, command_timeout=15, + remote_timeout=60, discovery_seconds=4, sdk_factory=None, + event_types=None, source_factory=None): + self.port, self.baud = port, baud + self.command_timeout = command_timeout + self.remote_timeout = remote_timeout + self.discovery_seconds = discovery_seconds + self._sdk_factory, self._event_types = sdk_factory, event_types + self._source_factory = source_factory + self._sdk = None + self._wire_lock = asyncio.Lock() + self._admin_lock = asyncio.Lock() + self._discovery_lock = asyncio.Lock() + self._subscriptions = [] + self._saved_paths = {} + self._authenticated = set() + self._guarded_key = None + self._discovered = {} + self._discovery_tag = None + self._login_waiter = self._reply_waiter = None + self._used_correlations = set() + self._timestamps = {} + self._messages_ready = asyncio.Event() + self._message_task = None + self._raw = asyncio.Queue(maxsize=64) + self._serve_task = None + self._serve_stop = asyncio.Event() + self._disconnected = False + + @property + def self_info(self): + return dict(getattr(self._sdk, "self_info", {}) or {}) + + @property + def source_key(self): + value = self.self_info.get("public_key", "") + try: + return _key(value) + except CompanionError: + return None + + async def reset_remote_session(self, public_key): + # The receiver loses login sessions on reboot. Preserve the used + # command correlations and outbound timestamps so delayed replies + # cannot satisfy a fresh request after authenticating again. + async with self._admin_lock: + key = _key(public_key) + self._authenticated.discard(key) + if self._guarded_key == key: + await self._guard_mode(key, 3) + self._guarded_key = None + + async def open(self): + if self._sdk is not None: + return self + if self._sdk_factory is None: + try: + from meshcore import MeshCore, EventType + except ImportError as exc: + raise CompanionError("Install the meshcore Python package for USB Companion support") from exc + self._sdk_factory, self._event_types = MeshCore.create_serial, EventType + if self._event_types is None: + raise CompanionError("Companion event definitions are unavailable") + try: + self._sdk = await self._sdk_factory( + self.port, baudrate=self.baud, only_error=True, + default_timeout=self.command_timeout, auto_reconnect=False) + if self._sdk is None: + raise CompanionError("No Binary Companion response on this USB port") + for name in ("DISCOVER_RESPONSE", "RX_LOG_DATA", "CONTACT_MSG_RECV", + "LOGIN_SUCCESS", "LOGIN_FAILED", "MESSAGES_WAITING", "DISCONNECTED"): + self._subscriptions.append(self._sdk.subscribe( + getattr(self._event_types, name), self._on_event)) + await self._call("get_contacts") + self._disconnected = False + self._message_task = asyncio.create_task(self._fetch_messages()) + self._messages_ready.set() + return self + except BaseException: + await self.close() + raise + + async def close(self): + self._serve_stop.set() + task = self._serve_task + if task is not None and task is not asyncio.current_task(): + try: + await asyncio.wait_for(asyncio.shield(task), 5) + except (asyncio.TimeoutError, CompanionError, ConnectionError): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + if self._message_task is not None: + self._message_task.cancel() + await asyncio.gather(self._message_task, return_exceptions=True) + self._message_task = None + error = None + if self._sdk is not None: + try: + await self.restore_contact_path() + except (CompanionError, TimeoutError, ConnectionError) as exc: + error = exc + for subscription in self._subscriptions: + self._sdk.unsubscribe(subscription) + self._subscriptions.clear() + await self._sdk.disconnect() + self._sdk = None + if error is not None: + raise error + + async def _call(self, method, *args, **kwargs): + if self._sdk is None or self._disconnected: + raise ConnectionError("USB Companion is disconnected") + function = getattr(self._sdk.commands, method, None) + if function is None: + raise CompanionError(f"Installed Companion SDK does not support {method}") + async with self._wire_lock: + try: + result = await asyncio.wait_for(function(*args, **kwargs), self.command_timeout) + except asyncio.TimeoutError as exc: + raise TimeoutError(f"USB Companion timed out during {method}") from exc + except OSError as exc: + raise ConnectionError("USB Companion connection failed") from exc + if result is None: + raise TimeoutError(f"USB Companion did not answer {method}") + if _kind(result) == "command_error": + reason = str(_payload(result).get("reason", "")).lower() + if reason in ("timeout", "no_event_received") or reason.startswith("timeout "): + raise TimeoutError(f"USB Companion timed out during {method}") + # Neither arbitrary SDK payloads nor command arguments enter errors. + raise CompanionError(f"USB Companion rejected {method}") + return result + + def _on_event(self, event): + kind, payload = _kind(event), _payload(event) + if kind == "disconnected": + self._disconnected = True + for waiting in (self._login_waiter, self._reply_waiter): + if waiting is not None and not waiting[-1].done(): + waiting[-1].set_exception(ConnectionError("USB Companion disconnected")) + elif kind == "messages_waiting": + self._messages_ready.set() + elif kind == "discover_response": + public_key = str(payload.get("pubkey", "")).lower() + tag = payload.get("tag") + if isinstance(tag, bytes): + tag = tag.hex() + if (tag != self._discovery_tag or not re.fullmatch(r"[0-9a-f]{64}", public_key) + or payload.get("node_type") not in (2, 3) + or payload.get("path_len") not in (0, 64, 128)): + return + self._discovered[public_key] = dict( + public_key=public_key, type=payload["node_type"], path="", + rssi=payload.get("RSSI"), snr=payload.get("SNR")) + elif kind == "rx_log_data" and self._serve_task is not None: + raw = raw_rx_bytes(payload) + if 3 <= len(raw) <= 255 and not self._raw.full(): + self._raw.put_nowait(raw) + elif kind in ("login_success", "login_failed") and self._login_waiter is not None: + key, future = self._login_waiter + if payload.get("pubkey_prefix", "").lower() != key[:12] or future.done(): + return + if (kind == "login_failed" or not payload.get("is_admin", False) + or payload.get("acl_permissions", 0) & 7 != 3): + future.set_exception(CompanionError("The selected repeater rejected the admin login")) + else: + future.set_result(payload) + elif kind == "contact_message" and self._reply_waiter is not None: + key, correlation, future = self._reply_waiter + text = payload.get("text", "") + if (not future.done() and payload.get("pubkey_prefix", "").lower() == key[:12] + and payload.get("txt_type") == 1 and payload.get("path_len") == 255 + and isinstance(text, str) and text.startswith(correlation)): + future.set_result(text[len(correlation):]) + + async def _fetch_messages(self): + while True: + try: + try: + await asyncio.wait_for(self._messages_ready.wait(), 1) + except asyncio.TimeoutError: + if self._reply_waiter is None: + continue + self._messages_ready.clear() + for _ in range(32): + result = await self._call("get_msg", timeout=2) + if _kind(result) == "no_more_messages": + break + except (CompanionError, TimeoutError, ConnectionError): + if self._disconnected: + return + await asyncio.sleep(.2) + + async def discover(self): + async with self._discovery_lock: + tag = secrets.randbits(32) or 1 + self._discovery_tag = tag.to_bytes(4, "little").hex() + self._discovered = {} + try: + await self._call("send_node_discover_req", 0x0C, prefix_only=False, tag=tag) + await asyncio.sleep(self.discovery_seconds) + contacts = getattr(self._sdk, "contacts", {}) + results = [] + for key, item in self._discovered.items(): + contact = contacts.get(key, {}) + results.append(dict(item, name=contact.get("adv_name", key[:12]))) + return sorted(results, key=lambda item: (item["name"], item["public_key"])) + finally: + self._discovery_tag = None + + async def _direct_contact(self, public_key): + key = _key(public_key) + contacts = getattr(self._sdk, "contacts", {}) + if any(other.lower() != key and other.lower().startswith(key[:12]) for other in contacts): + raise CompanionError("Contact key prefix is ambiguous; refusing remote administration") + if key not in self._saved_paths: + original = deepcopy(contacts.get(key)) + if original is None and key not in self._discovered: + raise CompanionError("Discover the selected repeater on this direct link first") + self._saved_paths[key] = original + contact = deepcopy(contacts.get(key)) + if contact is None: + discovered = self._discovered[key] + contact = dict(public_key=key, type=discovered["type"], flags=0, + out_path="", out_path_len=0, out_path_hash_mode=0, + adv_name=discovered.get("name", key[:12]), last_advert=0, + adv_lat=0, adv_lon=0) + # Reassert the direct path before every operation: background adverts or + # another client must not silently restore a routed/flood path. + contact.update(out_path="", out_path_len=0, out_path_hash_mode=0) + await self._call("update_contact", contact) + contacts[key] = contact + return contact + + async def restore_contact_path(self, public_key=None): + keys = [_key(public_key)] if public_key else list(self._saved_paths) + if self._guarded_key in keys: + await self._guard_mode(self._guarded_key, 3) + self._guarded_key = None + for key in keys: + if key not in self._saved_paths: + continue + original = self._saved_paths[key] + contacts = getattr(self._sdk, "contacts", {}) + if original is None: + await self._call("remove_contact", key) + contacts.pop(key, None) + else: + contact = deepcopy(contacts.get(key, original)) + for field in ("out_path", "out_path_len", "out_path_hash_mode"): + contact[field] = original[field] + await self._call("update_contact", contact) + contacts[key] = contact + self._saved_paths.pop(key) + self._authenticated.discard(key) + + async def local_command(self, text): + if text.strip().lower() == "get radio": + return "> " + ",".join(str(value) for value in await self.get_radio()) + result = await self._call("run_cli_command", text) + reply = _payload(result).get("text") + if not isinstance(reply, str): + raise CompanionError("Companion did not return a local CLI reply") + # Firmware CLI failures are data, just like remote CLI replies. The + # session layer distinguishes unsupported capabilities from failures; + # replacing every ERR with a generic message loses that information. + return reply + + async def get_radio(self): + result = await self._call("send_appstart") + info = _payload(result) + try: + return (float(info["radio_freq"]), float(info["radio_bw"]), + int(info["radio_sf"]), int(info["radio_cr"])) + except (KeyError, TypeError, ValueError) as exc: + raise CompanionError("Companion did not report its radio settings") from exc + + async def set_radio(self, radio): + if len(radio) != 4: + raise CompanionError("Radio settings require frequency, bandwidth, SF and CR") + await self._call("set_radio", *radio) + actual = await self.get_radio() + if any(abs(float(a) - float(b)) > .002 for a, b in zip(actual, radio)): + raise CompanionError("Companion radio readback does not match the requested settings") + + async def remote_command(self, public_key, password, text): + key = _key(public_key) + if not isinstance(password, str) or len(password.encode("utf-8")) > 15: + raise CompanionError("Admin passwords must fit in 15 UTF-8 bytes") + if not isinstance(text, str) or not text or any(c in text for c in "\r\n\0"): + raise CompanionError("Remote commands must be one nonempty line") + if len(text.encode("utf-8")) > 155: + raise CompanionError("Remote command is too long") + async with self._admin_lock: + return await self._remote_command(key, password, text) + + async def _remote_command(self, key, password, text, renewed=False): + # Caller owns the admin lock; a verified explicit login-required reply + # permits one fresh login. Silence never invalidates authentication. + contact = await self._direct_contact(key) + try: + await self._guard_mode(key, 1) + except CompanionError as exc: + raise CompanionError("The USB radio needs current Full Companion firmware with the zero-hop OTA admin guard") from exc + self._guarded_key = key + if key not in self._authenticated: + # The library's send_login_sync accepts any login-success + # event. Register our own key-filtered future before sending. + future = asyncio.get_running_loop().create_future() + self._login_waiter = (key, future) + try: + await self._prepare_login_clock(key) + await self._call("send_login", contact, password) + await asyncio.wait_for(future, self.remote_timeout) + except asyncio.TimeoutError as exc: + raise TimeoutError("Timed out waiting for the selected repeater's admin login") from exc + finally: + self._login_waiter = None + if not future.done(): + future.cancel() + elif not future.cancelled(): + future.exception() + # A zero-hop PATH teaches this server its empty return route. + # PATH has lower RF priority than CLI; wait for actual TX drain + # before a fresh full-key proof, and retry only this read-only proof. + for attempt in range(3): + try: + await self._teach_return_path(key) + proof = await self._command_reply(key, contact, "get public.key") + if proof.strip().removeprefix(">").strip().lower() != key: + raise CompanionError("The direct destination did not prove its complete public key") + break + except TimeoutError: + if attempt == 2: + raise + self._authenticated.add(key) + reply = await self._command_reply(key, contact, text) + if (not renewed and re.search( + r"\b(?:login required|not logged in|authentication required|session expired)\b", + reply, re.IGNORECASE)): + self._authenticated.discard(key) + await self._guard_mode(key, 3) + self._guarded_key = None + return await self._remote_command(key, password, text, renewed=True) + return reply + + async def _prepare_login_clock(self, key): + now = int(time.time()) + result = await self._call("get_time") + current = _payload(result).get("time") + if not isinstance(current, int) or isinstance(current, bool) or not 0 <= current <= 0xFFFFFFFF: + raise CompanionError("The Companion did not report a valid RTC time before login") + if current > now + 600: + raise CompanionError("The Companion clock is over ten minutes ahead; correct it before OTA login") + if current < now: + requested = now + 2 + await self._call("set_time", requested) + result = await self._call("get_time") + current = _payload(result).get("time") + if (not isinstance(current, int) or isinstance(current, bool) + or not requested <= current <= now + 600): + raise CompanionError("Could not verify the Companion clock advance before OTA login") + # sendLogin uses the source RTC, while subsequent CLI packets use host + # timestamps. Their floor must follow the login, even when the source + # is a little ahead of the computer or was just advanced above it. + self._timestamps[key] = max(self._timestamps.get(key, 0), current) + + async def _guard_mode(self, key, mode): + # Reserved modes of CMD52 are rejected by stock firmware without RF. + # The public SDK send method preserves the connection's single owner. + await self._call("send", b"\x34" + bytes([mode]) + bytes.fromhex(key), + [self._event_types.OK, self._event_types.ERROR]) + + async def _teach_return_path(self, key): + await self._guard_mode(key, 2) + deadline = time.monotonic() + self.remote_timeout + while time.monotonic() < deadline: + status = await self.local_command("get ota.direct") + match = re.fullmatch(r"\s*>?\s*ota\.direct v=1 active=1 busy=([01])\s*", status) + if match is None: + raise CompanionError("Companion did not confirm the active zero-hop admin guard") + if match[1] == "0": + return + await asyncio.sleep(min(.1, max(.001, self.remote_timeout / 10))) + raise TimeoutError("Timed out waiting for the direct return path to finish transmitting") + + async def _command_reply(self, key, contact, text): + alphabet = string.ascii_letters + string.digits + if len(self._used_correlations) >= len(alphabet) ** 2: + raise CompanionError("Remote command correlation space exhausted; reconnect") + while True: + correlation = "".join(secrets.choice(alphabet) for _ in range(2)) + "|" + if correlation not in self._used_correlations: + self._used_correlations.add(correlation) + break + future = asyncio.get_running_loop().create_future() + self._reply_waiter = (key, correlation, future) + timestamp = max(int(time.time()), self._timestamps.get(key, 0) + 1) + self._timestamps[key] = timestamp + try: + await self._call("send_cmd", contact, correlation + text, timestamp=timestamp) + self._messages_ready.set() + reply = await asyncio.wait_for(future, self.remote_timeout) + except asyncio.TimeoutError as exc: + raise TimeoutError("Timed out waiting for the selected repeater's command reply") from exc + finally: + self._reply_waiter = None + if not future.done(): + future.cancel() + elif not future.cancelled(): + future.exception() + return reply + + async def serve(self, blob, stop_event): + """Serve one validated package concurrently with Binary admin commands. + + The caller must first open the source's same-tuple TempRadio window. + CMD65's OK is queue admission only, so send one frame at a time with a + two-airtime gap rather than enqueueing an entire requested block batch. + """ + if self._serve_task is not None: + raise CompanionError("A Companion OTA source is already running") + try: + from .mota_kiss_direct import MotaSource, decode_radio + except ImportError: + from mota_kiss_direct import MotaSource, decode_radio + radio = await self.get_radio() + temporary = await self.local_command("get tempradio") + if not re.search(r"\bactive\b", temporary, re.IGNORECASE): + raise CompanionError("Open the source Companion's same-channel TempRadio window before serving") + preamble_match = re.search(r"\bpreamble=(\d+)\b", temporary) + preamble = int(preamble_match[1]) if preamble_match else 32 + identity = str(self.self_info.get("public_key", "")).encode() + seeder_id = hashlib.sha256(identity + hashlib.sha256(blob).digest()).digest()[:4] + source = (self._source_factory or MotaSource)(blob, seeder_id, compressor=_deflate) + await asyncio.to_thread(source.prepare) + self._serve_task = asyncio.current_task() + self._serve_stop.clear() + while not self._raw.empty(): + self._raw.get_nowait() + next_advert = 0.0 + next_transmit = 0.0 + try: + while not stop_event.is_set() and not self._serve_stop.is_set(): + if self._disconnected: + raise ConnectionError("USB Companion disconnected while serving") + now = time.monotonic() + if now >= next_advert: + replies = [source.advert()] + next_advert = now + 30 + else: + try: + raw = await asyncio.wait_for(self._raw.get(), .2) + except asyncio.TimeoutError: + continue + incoming = decode_radio(raw) + if incoming is None or incoming.wide or incoming.observed_path_bytes: + continue + replies = source.replies(raw) + for frame in replies: + if stop_event.is_set() or self._serve_stop.is_set(): + break + parsed = decode_radio(frame) + if parsed is None or parsed.wide or parsed.observed_path_bytes: + raise CompanionError("OTA source produced an invalid direct frame") + # Even discovery and HAVE are non-forwardable. The reused + # legacy source normally floods those frames. + direct = b"\x32\0" + parsed.message + delay = max(0.0, next_transmit - time.monotonic()) + while delay > 0 and not (stop_event.is_set() or self._serve_stop.is_set()): + await asyncio.sleep(min(delay, .2)) + delay = max(0.0, next_transmit - time.monotonic()) + if stop_event.is_set() or self._serve_stop.is_set(): + break + await self._call("send_raw_packet", direct, priority=1) + source.note_sent(direct) + next_transmit = time.monotonic() + max(.25, 2 * packet_airtime(len(direct), radio, preamble) + .1) + return dict(source.stats) + finally: + self._serve_task = None diff --git a/tools/lora_ota/ota_wizard_packages.py b/tools/lora_ota/ota_wizard_packages.py new file mode 100644 index 000000000..a7402bf5a --- /dev/null +++ b/tools/lora_ota/ota_wizard_packages.py @@ -0,0 +1,210 @@ +"""Package preparation for the interactive updater; no radio I/O.""" +from __future__ import annotations + +import hashlib +import io +import json +import os +from pathlib import Path +import platform +import re +import shutil +import subprocess +import zipfile + +import lora_ota as ota +from ota_wizard_catalog import GitHubCatalog, ImageCache, ReleaseAsset, _open, download_asset, load_application +from ota_wizard_session import Destination, Package, full_package, validate_package + +OFFICIAL_BOOT_KEY = "272564CC588D3D122285A15E6E2566D2ABE7177BB7EA1D1E41B23B29F0F85D2D" +BOOT_REPOSITORY = "mikecarper/Adafruit_nRF52_Bootloader_OTAFIX" +MOTATOOL_REV = "8c38369e7d35ad50cf74261869676d52dd24adf7" + + +def small_host() -> bool: + # ARM also includes Apple Silicon and Windows workstations. Only Linux + # exposes the board/resource evidence used for this low-power default. + if platform.system() != "Linux": + return False + try: + if "raspberry pi" in Path("/proc/device-tree/model").read_text().lower(): + return True + except OSError: + pass + if not platform.machine().lower().startswith(("arm", "aarch")): + return False + try: + memory = re.search(r"^MemTotal:\s+(\d+)\s+kB\s*$", + Path("/proc/meminfo").read_text(), re.M) + cores = os.cpu_count() + return bool(memory and int(memory[1]) < 4 * 1024 * 1024 + and cores is not None and cores <= 4) + except OSError: + return False + + +def find_builder(cache: Path, explicit: str | None = None) -> str | None: + if explicit: + found = shutil.which(explicit) or (explicit if Path(explicit).is_file() else None) + if not found: + raise ota.OtaError("The selected differential builder does not exist.") + return str(found) + found = shutil.which("motatool") + cached = cache / "motatool" / MOTATOOL_REV / "bin" / ("motatool.exe" if os.name == "nt" else "motatool") + return found or (str(cached) if cached.is_file() else None) + + +def install_builder(cache: Path, progress=print) -> str | None: + """A workstation can prepare the pinned tool in a private cache once.""" + if small_host() or not shutil.which("cargo") or not shutil.which("rustc"): + return None + try: + for tool in ("cargo", "rustc"): + reply = subprocess.run([tool, "--version"], capture_output=True, text=True, timeout=10) + version = re.search(r"\b(\d+)\.(\d+)\.(\d+)", reply.stdout) + if reply.returncode or not version or tuple(map(int, version.groups())) < (1, 86, 0): + return None + except (OSError, subprocess.TimeoutExpired): + return None + prefix = cache / "motatool" / MOTATOOL_REV + prefix.mkdir(parents=True, exist_ok=True) + progress("Preparing the differential builder once. This downloads dependencies and can take several minutes.") + log = prefix / "build.log" + environment = {key: value for key, value in os.environ.items() + if key not in ("MESHCORE_ADMIN_PASSWORD", "MESHCORE_OTA_PASSWORD")} + try: + with log.open("w", encoding="utf-8") as output: + result = subprocess.run(["cargo", "install", "--git", "https://github.com/mikecarper/motatool.git", + "--rev", MOTATOOL_REV, "--locked", "--bin", "motatool", + "--root", str(prefix)], env=environment, + stdout=output, stderr=subprocess.STDOUT, timeout=3600) + except (OSError, subprocess.TimeoutExpired): + progress(f"The differential builder was unavailable; setup details are in {log}.") + return None + if result.returncode: + progress(f"The differential builder was unavailable; setup details are in {log}.") + return None + return find_builder(cache) + + +def find_running_image(destination: Destination, cache: Path, catalog: GitHubCatalog | None, + *, supplied: Path | None = None, progress=print) -> Path | None: + images = ImageCache(cache) + if supplied: + identity = load_application(supplied, target_id=destination.target_id, hw_id=destination.hw_id) + if identity.body_hash != destination.base_hash or identity.fw_version != destination.version: + raise ota.OtaError("The supplied base is not the exact running firmware.") + return images.remember(identity) + found = images.find(destination.base_hash, destination.target_id, destination.hw_id, + fw_version=destination.version) + if found: + progress("Found the exact running image in the local cache.") + return found + if catalog: + for asset in catalog.base_candidates(target_id=destination.target_id, + fw_version=destination.version, limit=4): + try: + candidate = download_asset(asset, cache / "downloads") + identity = load_application(candidate, target_id=destination.target_id, + hw_id=destination.hw_id) + except (ota.OtaError, OSError): + continue + if identity.body_hash == destination.base_hash and identity.fw_version == destination.version: + progress("Found and hash-checked the published running image.") + return images.remember(identity) + return None + + +def prepare_application(image: ota.EndFInfo, destination: Destination, cache: Path, + *, base: Path | None = None, builder: str | None = None, + full: bool = False, progress=print) -> Package: + block = min(2048, destination.max_block_size) + result = full_package(image, block) + if base and builder and not full: + identity = load_application(base, target_id=destination.target_id, hw_id=destination.hw_id) + if identity.body_hash != destination.base_hash or identity.fw_version != destination.version: + raise ota.OtaError("The differential base is not the exact running firmware.") + inputs = cache / "packages" / f"{destination.target_id:08X}" + inputs.mkdir(parents=True, exist_ok=True) + new_path = ImageCache(cache).remember(image) + output = inputs / f"{destination.base_hash.hex()}-{image.body_hash.hex()}-delta.mota" + command = [builder, "build", "--base", str(base), "--fw", str(new_path), + "--patch-type", "in-place" if destination.platform == "nrf52" else "sequential", + "--block-size", str(block), "--out", str(output)] + progress("Preparing a differential update from the exact running image…") + # Never accept a previous invocation's output after a failed build. + output.unlink(missing_ok=True) + try: + completed = subprocess.run(command, capture_output=True, text=True, timeout=3600) + if not completed.returncode and output.is_file(): + blob = ota.read_bounded_file(output, 64 * 1024 * 1024, "differential package") + differential = Package(blob, image.body_hash) + validate_package(differential, destination) + info = differential.info + if (info.is_full or info.is_bootloader or info.fw_version != image.fw_version + or info.image_size != len(image.image) + or info.image_hash != hashlib.sha256(image.image).digest()): + raise ota.OtaError("The differential builder returned a different new image.") + if len(blob) < len(result.blob) or not destination.external_storage and destination.platform == "nrf52": + result = differential + except (ota.OtaError, OSError, ValueError, subprocess.TimeoutExpired): + # Full images are the portable fallback. validate_package below + # refuses it on internal-flash nRF52, where a delta is mandatory. + pass + if result.info.is_full: + progress("A full update is smaller or the differential builder could not prepare a compatible delta.") + validate_package(result, destination) + return result + + +def prepare_bootloader(destination: Destination, cache: Path) -> Package | None: + """Select by installed boot identity, then verify against the pinned publisher key.""" + boot = destination.bootloader + if not boot or boot.apply_abi < 3 or not boot.storage_flags & 0x08: + return None + with _open(f"https://api.github.com/repos/{BOOT_REPOSITORY}/releases/latest", api=True) as response: + raw = response.read(2 * 1024 * 1024 + 1) + if len(raw) > 2 * 1024 * 1024: + raise ota.OtaError("The bootloader release catalog is too large.") + release = json.loads(raw) + tag = release.get("tag_name", "") + if not re.fullmatch(r"v\d+\.\d+\.\d+-OTAFIX\d+\.\d+\.\d+", tag): + raise ota.OtaError("The current bootloader release has an unsupported identity.") + assets = [row for row in release.get("assets", []) + if re.fullmatch(r"OTAFIX-\d+\.\d+\.\d+-bootloader-mota\.zip", row.get("name", ""))] + if len(assets) != 1: + raise ota.OtaError("The official bootloader release does not have one normal signed bundle.") + row = assets[0] + archive_path = download_asset(ReleaseAsset(row["name"], row["browser_download_url"], row["size"], + tag, 0, "bootloader", "standard", "zip", row.get("digest")), + cache / "downloads") + with zipfile.ZipFile(archive_path) as archive: + inventory = json.loads(ota.read_zip_member(archive, archive.getinfo("manifest.json"))) + key = ota.read_zip_member(archive, archive.getinfo("OTAFIX_MOTA_SIGNING_PUBLIC_KEY.txt")).decode().strip() + if key != OFFICIAL_BOOT_KEY or inventory.get("signing_public_key") != OFFICIAL_BOOT_KEY: + raise ota.OtaError("The bootloader release signing key does not match the pinned publisher.") + matches = [entry for entry in inventory.get("packages", []) + if str(entry.get("hardware_id")) == boot.hw_id + and int(str(entry.get("target_id", "0")), 0) == boot.target_id] + if len(matches) != 1: + raise ota.OtaError("No normal bootloader package matches this exact installed identity.") + row = matches[0] + filename = row.get("file", "") + if Path(filename).name != filename: + raise ota.OtaError("Invalid bootloader bundle member.") + blob = ota.read_zip_member(archive, archive.getinfo("mota/" + filename)) + if hashlib.sha256(blob).hexdigest().lower() != str(row.get("sha256", "")).lower(): + raise ota.OtaError("Bootloader package checksum does not match its release inventory.") + lib = ota.bootloader_library() + problems = lib.verify(lib.parse_container(blob), expect_pub=bytes.fromhex(OFFICIAL_BOOT_KEY)) + if problems: + raise ota.OtaError("Bootloader signature or integrity check failed: " + "; ".join(problems)) + package = Package(blob, signer=OFFICIAL_BOOT_KEY) + if destination.boot_version is not None and package.info.fw_version <= destination.boot_version: + return None + if destination.boot_version is None: + # Unknown versions require an operator's explicit diagnosis rather than + # guessing whether a protected/newer bootloader should be replaced. + return None + validate_package(package, destination) + return package diff --git a/tools/lora_ota/ota_wizard_repeater.py b/tools/lora_ota/ota_wizard_repeater.py new file mode 100644 index 000000000..04285e53e --- /dev/null +++ b/tools/lora_ota/ota_wizard_repeater.py @@ -0,0 +1,459 @@ +"""One USB repeater link for direct admin commands and mOTA file serving. + +Only the reader task consumes serial bytes. It answers device-initiated seeder +requests while command callers await their echoed, line-framed CLI replies. +Requires repeater firmware with ``ota.peer`` and shared-text seeder control. +""" +from __future__ import annotations + +import asyncio +from contextlib import suppress +import math +import re +import struct +import time +import zlib + +import lora_ota as ota + + +class RepeaterError(ota.OtaError): + pass + + +def _key(value: str) -> str: + if not isinstance(value, str) or not re.fullmatch(r"[0-9a-fA-F]{64}", value): + raise RepeaterError("A complete destination public key is required") + return value.lower() + + +def _one_line(value: str, limit: int) -> bytes: + if not isinstance(value, str) or not value or any(ord(char) < 32 or ord(char) == 127 for char in value): + raise RepeaterError("Commands must be one nonempty line without control characters") + encoded = value.encode("utf-8") + if len(encoded) > limit: + raise RepeaterError("The command is too long for the USB repeater") + return encoded + + +class SeederData: + """The read-only subset of MotaSeederProto.h, with independent raw DEFLATE.""" + def __init__(self, blob: bytes): + self.info = ota.parse_mota(bytes(blob)) + self.blob = self.info.blob + self._compressed: dict[int, bytes | None] = {} + + def request(self, op: int, args: bytes) -> tuple[int, bytes]: + info = self.info + if op == 1 and not args: + return 0, b"\x01" + if op == 2 and args == b"\0": + blocks = math.ceil(info.payload_size / info.block_size) + return 0, struct.pack("<4sIIBBIIIIIBBH", info.merkle_root, + info.target_id, info.fw_version, info.codec_id, info.flags, + len(self.blob), 8 + ota.MOTA_FIXED_MANIFEST_SIZE, blocks, + info.payload_offset, info.payload_size, info.block_size.bit_length() - 1, + 1, 0) + if op == 3 and len(args) == 7: + index, offset, length = struct.unpack("= blocks or length > 190 or (not length and offset): + return 1, b"" + if block not in self._compressed: + start = block * info.block_size + raw = info.payload[start:start + info.block_size] + compressor = zlib.compressobj(9, zlib.DEFLATED, -15) + encoded = compressor.compress(raw) + compressor.flush() + self._compressed[block] = encoded if len(encoded) < len(raw) else None + encoded = self._compressed[block] + if encoded is not None and offset + length <= len(encoded): + return 0, struct.pack(" self.frame_timeout: + self.buffer.clear() + self.line.clear() + self.pending_since = None + self.buffer.extend(data) + while self.buffer: + if self.buffer[:1] == b"M" and len(self.buffer) < 3: + self.pending_since = self.pending_since or now + break + if self.buffer[:2] == b"MS" and self.buffer[2] in self.HEADER_LENGTHS: + op = self.buffer[2] + header = self.HEADER_LENGTHS[op] + needed = 3 + header + 1 + if len(self.buffer) < needed: + self.pending_since = self.pending_since or now + break + if op == 6: + length = int.from_bytes(self.buffer[11:13], "little") + if length > 512: + del self.buffer[:3] + self.line.clear() + self.pending_since = None + continue + needed += length + if len(self.buffer) < needed: + self.pending_since = self.pending_since or now + break + frame = bytes(self.buffer[:needed]) + del self.buffer[:needed] + self.line.clear() + self.pending_since = None + checksum = 0 + for byte in frame[2:]: + checksum ^= byte + if checksum == 0: + events.append(("request", op, frame[3:-1])) + continue + byte = self.buffer.pop(0) + self.pending_since = None + if byte == 10: + line = self.line.decode("utf-8", "replace").strip("\r\n") + self.line.clear() + if line: + events.append(("line", line)) + else: + self.line.append(byte) + if len(self.line) > 512: + self.line.clear() + return events + + +class RepeaterTransport: + def __init__(self, port, baud=115200, *, command_timeout=12, + remote_timeout=40, poll_seconds=.25, serial_factory=None, + open_settle=.75): + self.port, self.baud = port, baud + self.command_timeout, self.remote_timeout = command_timeout, remote_timeout + self.poll_seconds, self.open_settle = poll_seconds, open_settle + self._serial_factory = serial_factory + self._serial = self._reader_task = None + self._decoder = StreamDecoder() + self._lines = asyncio.Queue(maxsize=64) + self._write_lock, self._command_lock, self._admin_lock = (asyncio.Lock() for _ in range(3)) + self._reader_failed = asyncio.Event() + self._failure = None + self._source_key = None + self._authenticated_key = None + self._last_peer_use = 0 + self._clock_checked = False + self._seeder = None + self._serve_task = self._serve_stop = None + + @property + def source_key(self): + return self._source_key + + async def open(self): + if self._serial is not None: + return self + factory = self._serial_factory + if factory is None: + try: + import serial + except ImportError as exc: + raise RepeaterError("Install pyserial for a USB repeater connection") from exc + factory = serial.Serial + def create(): + connection = factory(port=None, baudrate=self.baud, timeout=.1, + write_timeout=3, rtscts=False, dsrdtr=False) + connection.dtr, connection.rts = True, False + connection.port = self.port + connection.open() + return connection + try: + self._serial = await asyncio.to_thread(create) + self._reader_failed.clear() + self._failure = None + self._reader_task = asyncio.create_task(self._read_loop()) + await asyncio.sleep(self.open_settle) + reply = await self.local_command("get public.key") + match = re.search(r"\b[0-9a-fA-F]{64}\b", reply) + if match is None: + raise RepeaterError("The USB repeater did not report its complete public key") + self._source_key = match[0].lower() + status = await self.local_command("ota.peer status") + if not re.fullmatch(r"OTApeer v=1 session=[01] busy=[01] peers=\d+", status): + raise RepeaterError("This repeater needs firmware with direct USB OTA administration (ota.peer)") + return self + except BaseException: + await self.close() + raise + + async def _write(self, data: bytes): + if self._serial is None or self._failure is not None: + raise ConnectionError("The USB repeater is disconnected") + async with self._write_lock: + def send(): + remaining = memoryview(data) + while remaining: + count = self._serial.write(remaining) + if not count: + raise OSError("USB write did not make progress") + remaining = remaining[count:] + try: + await asyncio.to_thread(send) + except OSError as exc: + raise ConnectionError("Could not write to the USB repeater") from exc + + async def _read_loop(self): + try: + while self._serial is not None: + raw = await asyncio.to_thread(self._serial.read, 512) + if not raw and not self._serial.is_open: + raise ConnectionError("USB repeater disconnected") + for event in self._decoder.feed(raw): + if event[0] == "request": + _, op, args = event + status, payload = self._seeder.request(op, args) if self._seeder else ( + (0, b"\0") if op == 1 else (1, b"")) + frame = b"ms" + bytes((op, status)) + payload + checksum = 0 + for byte in frame: + checksum ^= byte + await self._write(frame + bytes((checksum,))) + else: + if self._lines.full(): + self._lines.get_nowait() + self._lines.put_nowait(event[1]) + except asyncio.CancelledError: + raise + except Exception as exc: + self._failure = exc + self._reader_failed.set() + if self._lines.full(): + self._lines.get_nowait() + self._lines.put_nowait(None) + + @staticmethod + def _echo_matches(line: str, command: str) -> bool: + if line.strip() == command.strip(): + return True + if command.startswith("ota.peer login "): + prefix = " ".join(command.split(" ", 3)[:3]) + " " + return line.startswith(prefix) and bool(re.fullmatch(r"\*+", line[len(prefix):])) + return False + + async def local_command(self, text): + encoded = _one_line(text, 240) + async with self._command_lock: + while not self._lines.empty(): + self._lines.get_nowait() + if self._failure is not None: + raise ConnectionError("The USB repeater is disconnected") from self._failure + await self._write(encoded + b"\r\n") + deadline, echo_seen = time.monotonic() + self.command_timeout, False + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("The USB repeater did not complete the command; its result is unknown") + line = await asyncio.wait_for(self._lines.get(), remaining) + if line is None: + raise ConnectionError("The USB repeater is disconnected") from self._failure + if self._echo_matches(line, text): + echo_seen = True + continue + if echo_seen: + match = re.match(r"\s*->\s?(.*)$", line) + if match: + return match[1].strip() + + async def _check_clock(self): + if self._clock_checked: + return + now = int(time.time()) + start, end = ota.parse_source_clock_window(await self.local_command("clock"), "USB repeater") + if start > now + 600: + raise RepeaterError("The repeater clock is over ten minutes ahead; correct it before OTA login") + if end < now: + reply = await self.local_command(f"time {now + 2}") + if not reply.startswith("OK"): + raise RepeaterError("Could not advance the repeater clock for authenticated OTA") + self._clock_checked = True + + async def _job(self, command: str) -> str: + queued = await self.local_command(command) + match = re.fullmatch(r"OTApeer job=([0-9a-fA-F]{8}) state=pending", queued) + if match is None: + raise RepeaterError(queued or "The repeater did not start the direct request") + job, deadline = match[1].upper(), time.monotonic() + self.remote_timeout + offset, total, result, result_state = 0, None, bytearray(), None + while time.monotonic() < deadline: + reply = await self.local_command("ota.peer result" + (f" {offset}" if offset else "")) + page = re.fullmatch(r"OTApeer job=([0-9a-fA-F]{8}) state=(pending|ok|error) bytes=(\d+) off=(\d+) data=([0-9a-fA-F]+|-)", reply) + if page is None or page[1].upper() != job: + raise RepeaterError("The repeater's direct request changed; its result is unknown") + state, size, page_offset = page[2], int(page[3]), int(page[4]) + if state == "pending": + await asyncio.sleep(self.poll_seconds) + continue + if size > 160 or page_offset != offset or (total is not None and (total != size or result_state != state)): + raise RepeaterError("The repeater returned inconsistent direct reply pages") + try: + chunk = b"" if page[5] == "-" else bytes.fromhex(page[5]) + except ValueError as exc: + raise RepeaterError("The repeater returned an invalid direct reply") from exc + if len(chunk) > 48 or offset + len(chunk) > size or (not chunk and offset < size): + raise RepeaterError("The repeater returned an invalid direct reply length") + total, result_state = size, state + result.extend(chunk) + offset += len(chunk) + if offset == total: + text = result.decode("utf-8", "replace") + if state == "error": + if "timed out" in text.lower() or "unknown" in text.lower(): + raise ota.TransmissionError(text) + raise RepeaterError(text) + return text + raise ota.TransmissionError("No completed direct reply; the command result is unknown") + + async def discover(self): + async with self._admin_lock: + return await self._discover_locked() + + async def _discover_locked(self): + self._authenticated_key = None + await self._job("ota.peer discover") + status = await self.local_command("ota.peer status") + count = re.search(r"\bpeers=(\d+)\b", status) + if count is None or int(count[1]) > 8: + raise RepeaterError("Invalid direct discovery result") + result = [] + for index in range(int(count[1])): + reply = await self.local_command(f"ota.peer peers {index}") + peer = re.fullmatch(r"OTApeer peer=(\d+) key=([0-9a-fA-F]{64}) type=([23]) rssi=(-?\d+) snr=(-?\d+)", reply) + if peer is None or int(peer[1]) != index: + raise RepeaterError("The repeater returned an invalid direct peer") + result.append({"public_key": peer[2].lower(), "type": int(peer[3]), + "rssi": int(peer[4]), "snr": int(peer[5])}) + return sorted(result, key=lambda item: item["rssi"], reverse=True) + + async def _login_locked(self, key, password): + # Native login binds its packet layout to a freshly discovered board + # role. A remembered/manual key must prove it is still directly heard. + peers = await self._discover_locked() + if key not in {item["public_key"] for item in peers}: + raise RepeaterError("The selected destination did not answer fresh direct discovery") + await self._job(f"ota.peer login {key} {password}") + self._authenticated_key = key + + async def remote_command(self, public_key, password, text): + key = _key(public_key) + _one_line(text, 154) + if (not isinstance(password, str) or len(password.encode("utf-8")) > 63 + or any(ord(char) < 32 or ord(char) == 127 for char in password)): + raise RepeaterError("Admin passwords must fit in 63 UTF-8 bytes without control characters") + async with self._admin_lock: + await self._check_clock() + if key != self._authenticated_key or time.monotonic() - self._last_peer_use > 240: + await self._login_locked(key, password) + command = f"ota.peer cmd {key} {text}" + try: + reply = await self._job(command) + except RepeaterError as exc: + if str(exc) != "ERR peer login required": + raise + await self._login_locked(key, password) + reply = await self._job(command) + self._last_peer_use = time.monotonic() + return reply + + async def get_radio(self): + reply = (await self.local_command("get radio")).lstrip("> ") + fields = reply.split(",") + try: + values = float(fields[0]), float(fields[1]), int(fields[2]), int(fields[3]) + ota.parse_temp_radio(",".join(map(str, values)) + ",240") + return values + except (ValueError, IndexError, ota.OtaError) as exc: + raise RepeaterError("The repeater did not report valid radio parameters") from exc + + async def reset_remote_session(self, public_key): + """Drop the peer timestamp floor after a target reboot, retaining USB serving.""" + _key(public_key) + async with self._admin_lock: + reply = await self.local_command("ota.peer close") + if reply != "OTApeer closed": + raise RepeaterError(reply or "Could not reset the direct admin session") + self._authenticated_key = None + self._last_peer_use = 0 + + async def set_radio(self, radio): + if len(radio) != 4: + raise RepeaterError("Radio settings need frequency, bandwidth, SF and CR") + text = ",".join(map(str, radio)) + ota.parse_temp_radio(text + ",240") + reply = await self.local_command("set radio " + text) + if not reply.startswith("OK"): + raise RepeaterError(reply) + actual = await self.get_radio() + if any(abs(float(a) - float(b)) > .002 for a, b in zip(actual, radio)): + raise RepeaterError("Saved repeater radio settings do not match the requested tuple") + + async def serve(self, blob, stop_event): + if self._serve_task is not None: + raise RepeaterError("The USB repeater is already serving an update") + self._seeder = SeederData(blob) + self._serve_task, self._serve_stop = asyncio.current_task(), stop_event + stopped = failed = None + try: + reply = await self.local_command("ota folder on") + if not reply.startswith("OK folder attached"): + raise RepeaterError(reply or "Could not attach the OTA package") + stopped = asyncio.create_task(stop_event.wait()) + failed = asyncio.create_task(self._reader_failed.wait()) + await asyncio.wait((stopped, failed), return_when=asyncio.FIRST_COMPLETED) + if self._failure is not None: + raise ConnectionError("The USB repeater disconnected while serving") from self._failure + finally: + for task in (stopped, failed): + if task: + task.cancel() + with suppress(asyncio.CancelledError): + await task + if self._serial is not None and self._failure is None: + with suppress(TimeoutError, ConnectionError, RepeaterError): + await self.local_command("ota folder off") + self._seeder = self._serve_task = self._serve_stop = None + + async def close(self): + if self._serve_stop is not None: + self._serve_stop.set() + if self._serve_task is not asyncio.current_task(): + with suppress(Exception): + await asyncio.wait_for(asyncio.shield(self._serve_task), self.command_timeout + 1) + if self._serial is not None and self._reader_task is not None and self._failure is None: + with suppress(Exception): + await self.local_command("ota.peer close") + connection, self._serial = self._serial, None + if connection is not None: + with suppress(Exception): + await asyncio.to_thread(connection.close) + if self._reader_task is not None: + self._reader_task.cancel() + with suppress(asyncio.CancelledError): + await self._reader_task + self._reader_task = None + self._source_key = self._authenticated_key = None + self._clock_checked = False diff --git a/tools/lora_ota/ota_wizard_session.py b/tools/lora_ota/ota_wizard_session.py new file mode 100644 index 000000000..92c6aca48 --- /dev/null +++ b/tools/lora_ota/ota_wizard_session.py @@ -0,0 +1,499 @@ +"""Direct OTA transactions shared by the interactive USB transports.""" +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +import hashlib +import json +from pathlib import Path +import re +import time +from typing import Callable + +import lora_ota as ota + + +@dataclass(frozen=True) +class Destination: + public_key: str + target_id: int + hw_id: str + base_hash: bytes + version: int + version_text: str + platform: str + external_storage: bool + max_block_size: int + codecs: int + bootloader: ota.BootloaderStatus | None = None + boot_version: int | None = None + sd_storage: bool = False + + +@dataclass(frozen=True) +class Package: + blob: bytes + body_hash: bytes | None = None + signer: str | None = None + + @property + def info(self) -> ota.MotaInfo: + return ota.parse_mota(self.blob) + + +def cli_value(reply: str) -> str: + return reply.strip().removeprefix(">").strip() + + +def temp_state(reply: str) -> str: + value = cli_value(reply) + if re.fullmatch(r"(?i)(?:TempRadio\s+)?(?:inactive|off)", value): + return "off" + if re.match(r"(?i)^(?:TempRadio\s+)?pending\b", value): + return "pending" + if re.match(r"(?i)^TempRadio active:", value): + return "active" + radio_tuple(value) + return "active" + + +def checked(reply: str) -> str: + value = cli_value(reply) + if not value or re.match(r"(?i)^(?:err(?:or)?\b|unknown\b|unsupported\b)", value): + raise ota.OtaError(value or "The radio returned no reply.") + return value + + +def radio_tuple(reply: str) -> tuple[float, float, int, int]: + values = cli_value(reply).split(",") + try: + result = (float(values[0]), float(values[1]), int(values[2]), int(values[3])) + except (ValueError, IndexError) as exc: + raise ota.OtaError("Could not read the radio frequency and modulation.") from exc + ota.parse_temp_radio(",".join(map(str, result)) + ",240") + return result + + +def full_package(image: ota.EndFInfo, block_size: int) -> Package: + """A Pi can package a full image without a Rust or firmware compiler.""" + lib = ota.bootloader_library() + manifest = lib.build_manifest( + target_id=image.target_id, fw_version=image.fw_version, + image_size=len(image.image), payload=image.image, + block_size=block_size, image_hash=hashlib.sha256(image.image).digest(), + codec_id=lib.CODEC_FULL, is_full=True, hw_id=image.hw_id) + blob = lib.build_container(manifest, image.image) + problems = lib.verify(lib.parse_container(blob)) + if problems: + raise ota.OtaError("Invalid full package: " + "; ".join(problems)) + return Package(blob, image.body_hash) + + +def validate_package(package: Package, destination: Destination) -> None: + info = package.info + library = ota.bootloader_library() + try: + expected_signer = bytes.fromhex(package.signer) if package.signer else None + parsed = library.parse_container(package.blob) + if expected_signer is not None and (len(expected_signer) != 32 or not parsed.manifest.is_signed): + raise ValueError("The claimed signing key requires a signed container") + problems = library.verify(parsed, expect_pub=expected_signer) + except (ValueError, TypeError) as exc: + raise ota.OtaError("Invalid package signing key or container") from exc + if problems: + raise ota.OtaError("Package verification failed: " + "; ".join(problems)) + if info.block_size > destination.max_block_size: + raise ota.OtaError("The package's block size exceeds this receiver's limit.") + if info.is_bootloader: + boot = destination.bootloader + if destination.platform != "nrf52" or not boot or boot.apply_abi < 3: + raise ota.OtaError("This bootloader cannot update itself over LoRa.") + if destination.codecs != 0x5: + raise ota.OtaError("Bootloader self-update requires FULL|INPLACE codecs 0x5.") + if (info.target_id != boot.target_id or info.hw_id != boot.hw_id + or info.bootloader_storage != boot.storage_flags): + raise ota.OtaError("Bootloader package does not match the installed identity and storage.") + if not package.signer: + raise ota.OtaError("Bootloader installation needs a pinned trusted signing key.") + if destination.boot_version is None or info.fw_version <= destination.boot_version: + raise ota.OtaError("Choose a bootloader newer than the installed version.") + else: + if not isinstance(package.body_hash, bytes) or len(package.body_hash) != 8: + raise ota.OtaError("The application package needs its expected post-install body hash.") + if info.target_id != destination.target_id or info.hw_id != destination.hw_id: + raise ota.OtaError("The firmware package is for a different board or role.") + if info.fw_version <= destination.version: + raise ota.OtaError("Choose firmware newer than the running version.") + if not info.is_full and info.base_hash != destination.base_hash: + raise ota.OtaError("The delta does not match the exact running firmware.") + if destination.platform == "nrf52": + if not destination.codecs & (1 << info.codec_id): + raise ota.OtaError("The installed bootloader does not support this package codec.") + if info.is_full and not destination.external_storage: + raise ota.OtaError("Internal-flash nRF52 needs an exact-base in-place delta.") + if not info.is_full and info.codec_id != ota.MOTA_CODEC_IN_PLACE: + raise ota.OtaError("nRF52 needs an in-place differential package.") + elif not info.is_full and info.codec_id != ota.MOTA_CODEC_SEQUENTIAL: + raise ota.OtaError("ESP32 needs a sequential differential package.") + if destination.sd_storage and not package.signer: + raise ota.OtaError("This SD installer requires a trusted signed application package.") + + +class DirectSession: + """One target, zero-hop routing, bounded operations, and durable recovery state.""" + + def __init__(self, transport, public_key: str, password: str, journal: Path, + *, progress: Callable[[str], None] = print, poll_seconds: float = 5, + transfer_timeout: float = 4 * 60 * 60, reboot_timeout: float = 300): + if not re.fullmatch(r"[0-9a-fA-F]{64}", public_key): + raise ota.OtaError("Choose a destination with a complete public key.") + self.link, self.key, self.password = transport, public_key.lower(), password + self.journal, self.progress = Path(journal), progress + self.poll_seconds, self.transfer_timeout = poll_seconds, transfer_timeout + self.reboot_timeout = reboot_timeout + self.saved: dict[str, dict[str, str]] = {} + self.active = False + self._serve_stop = None + self._serve_task = None + self._state: dict = {"format": 1, "target_key": self.key, + "source_key": getattr(transport, "source_key", None), + "phase": "inspecting"} + + async def remote(self, command: str, *, retries: int = 2) -> str: + # Only callers of read-only/idempotent commands choose retries. Install + # and pull are sent once and reconciled from device state if uncertain. + for attempt in range(retries + 1): + try: + return checked(await self.link.remote_command(self.key, self.password, command)) + except (TimeoutError, asyncio.TimeoutError, ConnectionError, ota.TransmissionError) as exc: + if attempt == retries: + raise ota.TransmissionError("No reply from the selected direct destination.") from exc + await asyncio.sleep(min(1 + attempt, 3)) + raise AssertionError("unreachable") + + async def local(self, command: str) -> str: + return checked(await self.link.local_command(command)) + + def save(self, phase: str, **fields) -> None: + self._state.update(phase=phase, settings=self.saved, **fields) + self.journal.parent.mkdir(parents=True, exist_ok=True) + temporary = self.journal.with_suffix(".tmp") + temporary.write_text(json.dumps(self._state, indent=2) + "\n", encoding="utf-8") + temporary.replace(self.journal) + + async def inspect(self) -> Destination: + identity = re.search(r"\b[0-9A-Fa-f]{64}\b", await self.remote("get public.key")) + if not identity or identity[0].lower() != self.key: + raise ota.OtaError("The responding radio did not prove the selected public key.") + status = await self.remote("ota status") + self_status = await self.remote("ota self") + version_text = await self.remote("ver") + version = ota.extract_reply_version(version_text) + target = re.search(r"\btarget:([0-9a-fA-F]{8})\b", status) + hardware = re.search(r"\bhw=([^ |]+)", status) + body = re.search(r"\bbase_hash=([0-9a-fA-F]{16})\b", self_status) + if not target or not hardware or not body or version is None: + raise ota.OtaError("The destination must report valid firmware, hardware, target, and body hash.") + combined = status + " " + self_status + nrf = "bootloader:" in combined or bool(re.search(r"\bbl:", combined)) + external = bool(re.search(r"(?:QSPI|SD) apply OK|\bbl:(?:QSPI|SD)\b", combined)) + if re.search(r"(?:QSPI|SD) store:ERR|storage unsafe:|NO .*apply|bl:NONE", combined): + raise ota.OtaError("The destination's storage or bootloader is not ready for LoRa updates.") + boot, boot_version, codecs = None, None, 0x7 if not nrf else 0 + if nrf: + try: + boot = ota.parse_bootloader_status(await self.remote("ota bootloader status", retries=0)) + except ota.OtaError: + pass # Application-only bootloaders can still support firmware OTA. + try: + boot_reply = await self.remote("get bootloader.ver", retries=0) + match = re.search(r"OTAFIX(\d+)\.(\d+)\.(\d+)(?:-preview\.(\d+))?", boot_reply) + if match: + major, minor, patch = map(int, match.groups()[:3]) + preview = int(match[4]) if match[4] else 255 + boot_version = (major << 24) | (minor << 16) | (patch << 8) | preview + except ota.OtaError: + pass + caps = re.search(r"abi=(\d+)\s+codecs=0x([0-9a-fA-F]+)", combined) + if caps and int(caps[1]) >= 3: + codecs = int(caps[2], 16) + elif boot and boot.apply_abi >= 3: + codecs = 0x5 + if not codecs: + raise ota.OtaError("The installed nRF52 bootloader did not confirm supported OTA codecs.") + return Destination(self.key, int(target[1], 16), hardware[1], bytes.fromhex(body[1]), + version, version_text, "nrf52" if nrf else "esp32", external, + ota.parse_target_max_block_size(status, self_status), codecs, + boot, boot_version, "SD apply OK" in combined or "bl:SD" in combined) + + async def snapshot(self) -> None: + source_key = getattr(self.link, "source_key", None) + if not isinstance(source_key, str) or not re.fullmatch(r"[0-9a-fA-F]{64}", source_key): + identity = re.search(r"\b[0-9A-Fa-f]{64}\b", await self.local("get public.key")) + if identity is None: + raise ota.OtaError("The USB source did not prove its complete public key.") + source_key = identity[0] + self._state["source_key"] = source_key.lower() + for label, command in (("source", self.local), ("target", self.remote)): + temp = temp_state(await command("get tempradio")) + if temp.lower() != "off": + raise ota.OtaError(f"The {label} already has a temporary radio window. Finish it before updating.") + config = await command("ota config") + radio = await command("get radio") + try: + rxps = await command("get radio.rxps.config") + except RuntimeError as exc: + if "unsupported" in str(exc).lower(): + rxps = "unsupported" + else: + rxps = await command("get radio.rxps") + if rxps != "unsupported": + ota.parse_rxps_settings(rxps, label) + self.saved[label] = {"radio": radio, "rxps": rxps, "ota": config} + source, target = (radio_tuple(self.saved[label]["radio"]) for label in ("source", "target")) + if any(abs(a - b) > .001 for a, b in zip(source, target)): + raise ota.OtaError("The source and destination must start on the same saved radio settings.") + self.save("prepared") + + async def _wait_window(self, command) -> None: + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + reply = temp_state(await command("get tempradio")) + if reply.lower() not in ("pending", "off"): + return + await asyncio.sleep(1) + raise ota.OtaError("The temporary OTA window did not become active.") + + async def begin(self) -> None: + if not self.saved: + await self.snapshot() + self.active = True + self.save("opening OTA windows") + for label, command in (("target", self.remote), ("source", self.local)): + commands = ["ota config hops 0"] + if "mode=seeder-only" not in self.saved[label]["ota"]: + commands[:0] = ["ota config autofetch off", "ota config autoinstall off"] + if self.saved[label]["rxps"] != "unsupported": + commands.append("set radio.rxps off") + for text in commands: + await command(text) + if self.saved[label]["rxps"] != "unsupported": + current = await self._read_rxps(command, label) + if current.enabled: + raise ota.OtaError(f"The {label} did not disable RX power saving.") + # Keep the existing RF tuple so both admin and file traffic stay + # reachable even if the process is interrupted before cleanup. + values = radio_tuple(self.saved["target"]["radio"]) + await command("tempradio " + ",".join(map(str, values)) + ",240") + await self._wait_window(command) + self.save("OTA windows active") + + async def _stop_serving(self) -> None: + if self._serve_task: + self._serve_stop.set() + try: + await asyncio.wait_for(self._serve_task, timeout=15) + except (asyncio.TimeoutError, asyncio.CancelledError): + self._serve_task.cancel() + await asyncio.gather(self._serve_task, return_exceptions=True) + finally: + self._serve_task = self._serve_stop = None + + async def _start_serving(self, package: Package) -> None: + await self._stop_serving() + self._serve_stop = asyncio.Event() + self._serve_task = asyncio.create_task(self.link.serve(package.blob, self._serve_stop)) + await asyncio.sleep(.2) + if self._serve_task.done(): + await self._serve_task + raise ota.OtaError("The file server stopped before starting the transfer.") + + async def _ready(self, info: ota.MotaInfo) -> None: + deadline = time.monotonic() + self.transfer_timeout + last = None + while time.monotonic() < deadline: + if self._serve_task.done(): + await self._serve_task + raise ota.OtaError("The USB file server stopped during the transfer.") + status, complete = await self._download_state(info) + if status != last: + self.progress(status) + last = status + if complete: + return + await asyncio.sleep(self.poll_seconds) + raise ota.OtaError("The transfer timed out. The recovery record retains the package ID.") + + async def _download_state(self, info: ota.MotaInfo) -> tuple[str, bool]: + status = await self.remote("ota status") + download = re.search(r"\bdownload:\s*([^|]+)", status, re.I) + section = download[1] if download else "" + mid = re.search(r"\bid=([0-9A-Fa-f]{8})\b", section) + if mid is None: + # Long hardware names can truncate the status reply before id=. + # ota stats also has a self id=; only inspect its FETCH section. + stats = await self.remote("ota stats") + fetch = re.search(r"\bfetch\s+([^|]+)", stats, re.I) + section = fetch[1] if fetch else "" + mid = re.search(r"\bid=([0-9A-Fa-f]{8})\b", section) + status += " | " + stats + if mid is None: + raise ota.OtaError("The destination did not confirm a live download and manifest ID.") + if mid[1].lower() != info.manifest_id.lower(): + raise ota.OtaError("The destination's download is not the selected package.") + if re.match(r"failed\b", section, re.I): + raise ota.OtaError("The destination reported a failed download: " + status) + complete = bool(re.match(r"(?:ready to install|done)\b", section, re.I)) + return status, complete + + async def _installed(self, package: Package, before: Destination) -> Destination: + info, deadline = package.info, time.monotonic() + self.reboot_timeout + while time.monotonic() < deadline: + try: + reset_session = getattr(self.link, "reset_remote_session", None) + if reset_session: + await reset_session(self.key) + current = await self.inspect() + if info.is_bootloader: + if current.boot_version == info.fw_version: + if (current.target_id != before.target_id or current.base_hash != before.base_hash + or current.hw_id != before.hw_id + or not current.bootloader or not before.bootloader + or (current.bootloader.board_id, current.bootloader.device_name, + current.bootloader.storage_flags) != + (before.bootloader.board_id, before.bootloader.device_name, + before.bootloader.storage_flags)): + raise ota.OtaError("The application changed during the bootloader-only update.") + return current + elif (current.version == info.fw_version and package.body_hash == current.base_hash + and current.target_id == info.target_id and current.hw_id == info.hw_id): + return current + except (ota.OtaError, TimeoutError, ConnectionError): + pass + await asyncio.sleep(self.poll_seconds) + raise ota.OtaError("Could not confirm the installed version and body hash after reboot.") + + async def install(self, package: Package, before: Destination) -> Destination: + validate_package(package, before) + info = package.info + if not self.active: + await self.begin() + # A bootloader reboot clears the destination's RAM-only lease. + if temp_state(await self.remote("get tempradio")) == "off": + values = radio_tuple(self.saved["target"]["radio"]) + await self.remote("tempradio " + ",".join(map(str, values)) + ",240") + await self._wait_window(self.remote) + if package.signer: + await self.remote("ota key add " + package.signer) + if not info.is_bootloader and not info.is_full: + live = await self.inspect() + if live.base_hash != info.base_hash: + raise ota.OtaError("The destination's firmware changed after delta preparation.") + await self._start_serving(package) + self.save("downloading", package_mid=info.manifest_id, package_hash=info.image_hash.hex(), + package_kind=info.kind, expected_body=package.body_hash.hex() if package.body_hash else None) + # Discovery may need more than one advertisement. It is read-only and + # always bound to the exact selected MID before a pull is issued. + for _ in range(12): + if info.manifest_id.lower() in (await self.remote("ota ls")).lower(): + break + await asyncio.sleep(self.poll_seconds) + else: + raise ota.OtaError("The selected package was not discovered over the direct link.") + try: + await self.remote(f"ota pull {info.manifest_id} flash", retries=0) + except ota.TransmissionError: + await self._download_state(info) + await self._ready(info) + if info.is_bootloader: + staged = await self.remote("ota bootloader status") + expected = f"staged:ready mid={info.manifest_id} hash={info.image_hash[:8].hex()}" + if expected.lower() not in staged.lower(): + raise ota.OtaError("The staged bootloader MID/hash does not match the verified package.") + command = f"ota bootloader install {info.manifest_id} {info.image_hash[:8].hex().upper()}" + else: + command = "ota install" + self.save("installing") + try: + await self.remote(command, retries=0) + except ota.TransmissionError: + self.progress("The install reply was lost; checking the running image before taking further action.") + current = await self._installed(package, before) + await self._stop_serving() + self.save("installed") + return current + + async def cleanup(self) -> list[str]: + errors = [] + try: + await self._stop_serving() + except Exception: + errors.append("Stop the USB file server before restoring settings.") + if self.active: + for label, command in (("target", self.remote), ("source", self.local)): + if label not in self.saved: + continue + settings = self.saved[label] + commands = ["normalradio"] + if settings["rxps"] != "unsupported": + try: + rxps = ota.parse_rxps_settings(settings["rxps"], label) + current = await self._read_rxps(command, label) + if current != rxps: + commands.append(ota.rxps_restore_command(rxps)) + if not rxps.enabled: + commands.append("set radio.rxps off") + except Exception: + errors.append(f"Restore the recorded {label} RXPS settings from {self.journal}.") + for key in ("hops", "autofetch", "autoinstall"): + if key != "hops" and "mode=seeder-only" in settings["ota"]: + continue + match = re.search(rf"\b{key}=([A-Za-z0-9]+)", settings["ota"]) + if match: + commands.append(f"ota config {key} {match[1]}") + for text in commands: + try: + await command(text) + except Exception: + errors.append(f"{label}: {text}") + try: + await self._verify_restored(command, label, settings) + except Exception: + errors.append(f"Verify and restore the recorded {label} radio/RXPS/OTA settings from {self.journal}.") + restore_path = getattr(self.link, "restore_contact_path", None) + if restore_path: + try: + await restore_path() + except Exception: + errors.append("Restore the Companion's recorded destination contact route.") + self.save("needs cleanup" if errors else "finished", recovery_commands=errors) + self.active = bool(errors) + return errors + + async def _read_rxps(self, command, label: str) -> ota.RxpsSettings: + try: + reply = await command("get radio.rxps.config") + except ota.OtaError as exc: + if not re.search(r"unknown|unrecognized|not supported", str(exc), re.I): + raise + reply = await command("get radio.rxps") + return ota.parse_rxps_settings(reply, label) + + async def _verify_restored(self, command, label: str, settings: dict) -> None: + deadline = time.monotonic() + 45 + while temp_state(await command("get tempradio")) != "off": + if time.monotonic() >= deadline: + raise ota.OtaError(f"The {label} did not return to normal radio.") + await asyncio.sleep(1) + current_radio = radio_tuple(await command("get radio")) + if any(abs(a - b) > .001 for a, b in zip(current_radio, radio_tuple(settings["radio"]))): + raise ota.OtaError(f"The {label} radio settings changed.") + if settings["rxps"] != "unsupported": + if await self._read_rxps(command, label) != ota.parse_rxps_settings(settings["rxps"], label): + raise ota.OtaError(f"The {label} RXPS settings did not restore exactly.") + actual = await command("ota config") + for key in ("hops", "autofetch", "autoinstall"): + previous = re.search(rf"\b{key}=([A-Za-z0-9]+)", settings["ota"]) + current = re.search(rf"\b{key}=([A-Za-z0-9]+)", actual) + if previous is None or current is None or previous[1] != current[1]: + raise ota.OtaError(f"The {label} OTA policy did not restore exactly.") diff --git a/tools/lora_ota/requirements-wizard.txt b/tools/lora_ota/requirements-wizard.txt new file mode 100644 index 000000000..60f5e0713 --- /dev/null +++ b/tools/lora_ota/requirements-wizard.txt @@ -0,0 +1,4 @@ +# USB control and signed bootloader verification; wheels only in the private setup. +meshcore==2.3.9.1 +pyserial==3.5 +cryptography==50.0.1 diff --git a/tools/lora_ota/test_ota_wizard.py b/tools/lora_ota/test_ota_wizard.py new file mode 100644 index 000000000..a9aa35456 --- /dev/null +++ b/tools/lora_ota/test_ota_wizard.py @@ -0,0 +1,225 @@ +"""User-facing choices, recovery identity, and compiler-free offline preparation.""" +import asyncio +from contextlib import redirect_stderr, redirect_stdout +import io +import json +import os +from pathlib import Path +import sys +import tempfile +from types import ModuleType, SimpleNamespace +import unittest +from unittest import mock + +import lora_ota as ota +import ota_wizard as wizard +import ota_wizard_bundle as bundles +from test_ota_wizard_bundle import KEY, destination, image, signed_package + +SOURCE = '22' * 32 + + +class WizardTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory(prefix='meshcore-wizard-ui-') + self.addCleanup(self.directory.cleanup) + self.cache = Path(self.directory.name) + + def test_explicit_port_does_not_probe_or_prompt(self): + with mock.patch.object(wizard, 'select', side_effect=AssertionError('unexpected prompt')): + self.assertEqual(wizard.choose_port('/dev/ttyUSB0'), '/dev/ttyUSB0') + + def test_one_usb_device_is_selected_without_prompting(self): + ports = SimpleNamespace(comports=lambda: [SimpleNamespace( + device='/dev/ttyUSB0', description='Station G2', vid=0x10C4)]) + serial = ModuleType('serial') + tools = ModuleType('serial.tools') + tools.list_ports = ports + serial.tools = tools + with mock.patch.dict(sys.modules, {'serial': serial, 'serial.tools': tools}), \ + mock.patch.object(wizard, 'select', side_effect=AssertionError('unexpected prompt')): + self.assertEqual(wizard.choose_port(None), '/dev/ttyUSB0') + + def test_menu_retries_invalid_selection_and_preserves_default(self): + with mock.patch.object(wizard, 'ask', side_effect=['bad', '3', '2']), redirect_stdout(io.StringIO()): + self.assertEqual(wizard.select('Firmware', ['Published', 'Local']), 1) + + def test_windows_custom_build_maps_paths_into_wsl_and_excludes_credentials(self): + source = self.cache/'source checkout' + source.mkdir() + (source/'build.sh').write_text('# fixture only\n') + args = wizard.arguments(['--build', '--source-tree', str(source), '--cache', str(self.cache)]) + builds = [] + def process(command, **kwargs): + if command[:4] == ['wsl.exe', '-e', 'wslpath', '-a']: + mapped = '/mnt/source checkout' if Path(command[4]) == source else '/mnt/output cache' + return SimpleNamespace(returncode=0, stdout=mapped+'\n') + builds.append((command, kwargs)) + output = Path(kwargs['env']['OUTPUT_DIR']) + (output/'firmware.bin').write_bytes(image().image) + return SimpleNamespace(returncode=0) + fake_os = SimpleNamespace(name='nt', environ={ + 'MESHCORE_ADMIN_PASSWORD': 'never inherit', 'MESHCORE_OTA_PASSWORD': 'nor this', + 'PATH': 'tool path'}) + with mock.patch.object(wizard, 'os', fake_os), \ + mock.patch.object(wizard.shutil, 'which', return_value='wsl.exe'), \ + mock.patch.object(wizard.subprocess, 'run', side_effect=process), \ + redirect_stdout(io.StringIO()): + built = wizard.custom_build(args, destination(), 'ikoka_stick_nrf_33dbm_repeater') + self.assertEqual(built.read_bytes(), image().image) + self.assertEqual(len(builds), 1) + command, kwargs = builds[0] + self.assertEqual(command[:5], ['wsl.exe', '--cd', '/mnt/source checkout', '--exec', 'env']) + self.assertIn('OUTPUT_DIR=/mnt/output cache', command) + self.assertIn('full', command) + self.assertNotIn('MESHCORE_ADMIN_PASSWORD', kwargs['env']) + self.assertNotIn('MESHCORE_OTA_PASSWORD', kwargs['env']) + self.assertNotIn('never inherit', repr(command)) + + def test_prepare_only_uses_local_files_without_usb_network_or_compiler(self): + request, firmware, output = (self.cache/name for name in + ('device.json', 'new.bin', 'update.ota.zip')) + bundles.save_request(request, destination()) + firmware.write_bytes(image().image) + argv = ['--prepare-only', '--full', '--request', str(request), + '--firmware', str(firmware), '--output', str(output), '--cache', str(self.cache)] + with mock.patch.object(wizard, 'ensure_dependencies', side_effect=AssertionError('USB bootstrap')), \ + mock.patch.object(wizard, 'choose_port', side_effect=AssertionError('USB probe')), \ + mock.patch('ota_wizard_catalog._open', side_effect=AssertionError('network')), \ + mock.patch.object(wizard.subprocess, 'run', side_effect=AssertionError('compiler')), \ + redirect_stdout(io.StringIO()): + self.assertEqual(wizard.main(argv), 0) + prepared_image, package = bundles.load_bundle(output, destination()) + self.assertEqual(prepared_image.image, image().image) + self.assertTrue(package.info.is_full) + + def test_prepare_only_rejects_wrong_board_before_writing_bundle(self): + request, firmware, output = (self.cache/name for name in + ('device.json', 'new.bin', 'update.ota.zip')) + bundles.save_request(request, destination()) + firmware.write_bytes(image(hw='wrong_board').image) + with redirect_stderr(io.StringIO()), redirect_stdout(io.StringIO()): + result = wizard.main(['--prepare-only', '--full', '--request', str(request), + '--firmware', str(firmware), '--output', str(output), + '--cache', str(self.cache)]) + self.assertEqual(result, 1) + self.assertFalse(output.exists()) + + def test_internal_flash_destination_does_not_silently_fall_back_to_full(self): + from dataclasses import replace + request, firmware, output = (self.cache/name for name in + ('device.json', 'new.bin', 'update.ota.zip')) + bundles.save_request(request, replace(destination(), external_storage=False)) + firmware.write_bytes(image().image) + with mock.patch.object(wizard, 'find_builder', return_value=None), \ + mock.patch.object(wizard, 'install_builder', return_value=None), \ + redirect_stderr(io.StringIO()), redirect_stdout(io.StringIO()): + result = wizard.main(['--prepare-only', '--request', str(request), + '--firmware', str(firmware), '--output', str(output), + '--cache', str(self.cache)]) + self.assertEqual(result, 1) + self.assertFalse(output.exists()) + + def test_prepare_only_preserves_signed_full_input_even_with_full_option(self): + from dataclasses import replace + live = replace(destination(), sd_storage=True) + original = signed_package() + request, firmware, output = (self.cache/name for name in + ('device.json', 'signed-full.mota', 'update.ota.zip')) + bundles.save_request(request, live) + firmware.write_bytes(original.blob) + for full in ([], ['--full']): + argv = ['--prepare-only', '--request', str(request), '--firmware', str(firmware), + '--output', str(output), '--cache', str(self.cache), *full] + with self.subTest(full=bool(full)), \ + mock.patch.object(wizard, 'GitHubCatalog', side_effect=AssertionError('network')), \ + mock.patch.object(wizard, 'find_builder', side_effect=AssertionError('compiler')), \ + mock.patch.object(wizard, 'prepare_application', side_effect=AssertionError('repackage')), \ + redirect_stdout(io.StringIO()): + self.assertEqual(wizard.main(argv), 0) + _, prepared = bundles.load_bundle(output, live) + self.assertEqual(prepared, original) + + def link(self): + return SimpleNamespace(source_key=SOURCE, open=mock.AsyncMock(), close=mock.AsyncMock(), + discover=mock.AsyncMock(return_value=[{'public_key': KEY, 'type': 2}])) + + def run_options(self, *more): + return wizard.arguments(['--port', '/dev/ttyUSB0', '--mode', 'repeater', '--target', KEY, + '--cache', str(self.cache), '--yes', *more]) + + def fake_transport(self, link): + module = ModuleType('ota_wizard_repeater') + module.RepeaterTransport = mock.Mock(return_value=link) + return mock.patch.dict(sys.modules, {'ota_wizard_repeater': module}) + + def test_check_does_not_prepare_download_or_install_and_saves_no_password(self): + link = self.link() + session = SimpleNamespace(inspect=mock.AsyncMock(return_value=destination()), + snapshot=mock.AsyncMock(), cleanup=mock.AsyncMock(return_value=[]), + journal=self.cache/'session.json') + with self.fake_transport(link), \ + mock.patch.object(wizard, 'DirectSession', return_value=session), \ + mock.patch.object(wizard, 'GitHubCatalog', side_effect=AssertionError('network')), \ + mock.patch.dict(os.environ, {'MESHCORE_ADMIN_PASSWORD': 'DoNotPersistThis'}), \ + redirect_stdout(io.StringIO()): + self.assertEqual(asyncio.run(wizard.run(self.run_options('--check'))), 0) + session.snapshot.assert_not_awaited() + link.close.assert_awaited_once() + for file in self.cache.rglob('*.json'): + self.assertNotIn('DoNotPersistThis', file.read_text()) + + def test_live_full_option_retains_supplied_signed_mota_for_sd_destination(self): + from dataclasses import replace + live = replace(destination(), sd_storage=True) + original = signed_package() + firmware = self.cache/'signed-full.mota' + firmware.write_bytes(original.blob) + link = self.link() + session = SimpleNamespace(inspect=mock.AsyncMock(return_value=live), + snapshot=mock.AsyncMock(), install=mock.AsyncMock(return_value=live), + cleanup=mock.AsyncMock(return_value=[]), journal=self.cache/'session.json') + catalog = SimpleNamespace(choose_asset=mock.Mock(side_effect=ota.OtaError('offline'))) + with self.fake_transport(link), \ + mock.patch.object(wizard, 'DirectSession', return_value=session), \ + mock.patch.object(wizard, 'GitHubCatalog', return_value=catalog), \ + mock.patch.object(wizard, 'prepare_application', side_effect=AssertionError('repackage')), \ + mock.patch.object(wizard, 'find_builder', side_effect=AssertionError('compiler')), \ + mock.patch.dict(os.environ, {'MESHCORE_ADMIN_PASSWORD': 'not saved'}), \ + redirect_stdout(io.StringIO()): + self.assertEqual(asyncio.run(wizard.run(self.run_options( + '--firmware', str(firmware), '--full', '--no-bootloader'))), 0) + session.install.assert_awaited_once_with(original, live) + link.close.assert_awaited_once() + + def test_unfinished_recovery_cannot_change_a_different_source(self): + link = self.link() + journal = self.cache/'sessions'/(KEY+'.json') + journal.parent.mkdir() + journal.write_text(json.dumps({'format': 1, 'phase': 'armed', 'target_key': KEY, + 'source_key': '33'*32, 'settings': {}})) + with self.fake_transport(link), \ + mock.patch.object(wizard, 'DirectSession') as session, \ + mock.patch.dict(os.environ, {'MESHCORE_ADMIN_PASSWORD': 'not saved'}), \ + redirect_stdout(io.StringIO()), self.assertRaisesRegex(ota.OtaError, 'different source'): + asyncio.run(wizard.run(self.run_options('--recover'))) + session.assert_not_called() + link.close.assert_awaited_once() + + def test_invalid_recovery_identity_is_rejected_as_a_user_error(self): + link = self.link() + journal = self.cache/'sessions'/(KEY+'.json') + journal.parent.mkdir() + journal.write_text(json.dumps({'format': 1, 'phase': 'armed', 'target_key': KEY, + 'source_key': None, 'settings': {}})) + with self.fake_transport(link), \ + mock.patch.object(wizard, 'DirectSession') as session, \ + mock.patch.dict(os.environ, {'MESHCORE_ADMIN_PASSWORD': 'not saved'}), \ + redirect_stdout(io.StringIO()), self.assertRaises(ota.OtaError): + asyncio.run(wizard.run(self.run_options('--recover'))) + session.assert_not_called() + link.close.assert_awaited_once() + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/lora_ota/test_ota_wizard_bundle.py b/tools/lora_ota/test_ota_wizard_bundle.py new file mode 100644 index 000000000..ff7f7b105 --- /dev/null +++ b/tools/lora_ota/test_ota_wizard_bundle.py @@ -0,0 +1,231 @@ +"""Portable request and bundle checks without serial devices or networking.""" +from dataclasses import replace +import io +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +import warnings +import zipfile + +import lora_ota as ota +import ota_wizard_bundle as bundles +from ota_wizard_session import Destination, Package, full_package + +KEY = '11' * 32 +HW = 'ikoka_stick_nrf_33dbm' +TARGET = 0xE5331F02 + + +def image(version=0x01120002, target=TARGET, hw=HW, body=b'new firmware' * 32): + lib = ota.bootloader_library() + return ota.parse_endf(body + lib.build_endf(body, lib.FwIdent(version, target, hw))) + + +def destination(): + old = image(0x01120001, body=b'original application' * 16) + return Destination(KEY, TARGET, HW, old.body_hash, old.fw_version, + 'v1.18.0.1', 'nrf52', True, 2048, 5) + + +def signed_package(new_image=None): + try: + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + except ImportError as exc: + raise unittest.SkipTest('cryptography is required for signature fixtures') from exc + new_image = image() if new_image is None else new_image + lib = ota.bootloader_library() + manifest = lib.build_manifest(target_id=new_image.target_id, fw_version=new_image.fw_version, + image_size=len(new_image.image), payload=new_image.image, block_size=2048, + image_hash=hashlib.sha256(new_image.image).digest(), codec_id=lib.CODEC_FULL, + is_full=True, hw_id=new_image.hw_id, + sign_priv=Ed25519PrivateKey.from_private_bytes(bytes(range(32)))) + return Package(lib.build_container(manifest, new_image.image), new_image.body_hash, + manifest.signer_pubkey.hex().upper()) + + +class WizardBundleTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory(prefix='meshcore-wizard-bundle-') + self.addCleanup(self.directory.cleanup) + self.path = Path(self.directory.name) + self.destination = destination() + self.image = image() + self.package = full_package(self.image, 2048) + + def bundle(self): + file = self.path/'update.ota.zip' + bundles.save_bundle(file, self.image, self.package) + return file + + def rewrite(self, path, transform): + with zipfile.ZipFile(path) as archive: + members = {name: archive.read(name) for name in archive.namelist()} + transform(members) + with zipfile.ZipFile(path, 'w') as archive: + for name, data in members.items(): + archive.writestr(name, data) + + def test_request_round_trip_preserves_only_public_destination_data(self): + file = self.path/'device.json' + bundles.save_request(file, self.destination) + self.assertEqual(bundles.load_request(file), self.destination) + raw = json.loads(file.read_text()) + self.assertEqual(raw['destination']['public_key'], KEY) + self.assertEqual(raw['destination']['base_hash'], self.destination.base_hash.hex()) + self.assertNotIn('password', file.read_text().lower()) + self.assertNotIn('private_key', file.read_text().lower()) + + def test_full_bundle_round_trip_validates_firmware_and_package(self): + loaded_image, loaded_package = bundles.load_bundle(self.bundle(), self.destination) + self.assertEqual(loaded_image.image, self.image.image) + self.assertEqual(loaded_package.blob, self.package.blob) + self.assertEqual(loaded_package.body_hash, self.image.body_hash) + with zipfile.ZipFile(self.path/'update.ota.zip') as archive: + self.assertEqual(set(archive.namelist()), + {'ota-bundle.json', 'application.mota', 'firmware.bin'}) + + def test_request_round_trip_binds_exact_bootloader_identity(self): + lib = ota.bootloader_library() + board, name = 0x28860044, 'XIAO_DFU' + boot = ota.BootloaderStatus(board, lib.bootloader_target_id(board, name), name, + lib.bootloader_hw_id(board, name).rstrip(b'\0').decode(), + 0x6F9BE280, 3, 14) + expected = replace(self.destination, bootloader=boot, boot_version=0x0204000B) + file = self.path/'request.json' + bundles.save_request(file, expected) + self.assertEqual(bundles.load_request(file), expected) + value = json.loads(file.read_text()) + value['destination']['bootloader']['target_id'] ^= 1 + file.write_text(json.dumps(value)) + with self.assertRaisesRegex(ota.OtaError, 'identity'): + bundles.load_request(file) + + def test_normal_firmware_zip_is_not_mistaken_for_prepared_bundle(self): + file = self.path/'application.zip' + with zipfile.ZipFile(file, 'w') as archive: + archive.writestr('manifest.json', '{}') + self.assertIsNone(bundles.load_bundle(file, self.destination)) + + def test_changed_package_checksum_is_rejected(self): + file = self.bundle() + self.rewrite(file, lambda members: members.__setitem__( + 'application.mota', members['application.mota'][:-1] + b'X')) + with self.assertRaisesRegex(ota.OtaError, 'checksum'): + bundles.load_bundle(file, self.destination) + + def test_changed_complete_firmware_identity_is_rejected(self): + file = self.bundle() + self.rewrite(file, lambda members: members.__setitem__( + 'firmware.bin', image(body=b'different firmware' * 32).image)) + with self.assertRaisesRegex(ota.OtaError, 'does not match'): + bundles.load_bundle(file, self.destination) + + def test_recomputed_outer_checksum_does_not_hide_corrupted_container(self): + file = self.bundle() + def corrupt(members): + blob = bytearray(members['application.mota']) + blob[-1] ^= 1 + members['application.mota'] = bytes(blob) + metadata = json.loads(members['ota-bundle.json']) + metadata['package_sha256'] = hashlib.sha256(blob).hexdigest() + members['ota-bundle.json'] = json.dumps(metadata) + self.rewrite(file, corrupt) + with self.assertRaises(ota.OtaError): + bundles.load_bundle(file, self.destination) + + def test_malformed_bundle_metadata_is_reported_as_a_user_error(self): + for metadata in (b'[]', b'null', b'"text"', b'{broken'): + file = self.bundle() + self.rewrite(file, lambda members: members.__setitem__('ota-bundle.json', metadata)) + with self.subTest(metadata=metadata), self.assertRaises(ota.OtaError): + bundles.load_bundle(file, self.destination) + + def test_wrong_live_board_and_non_upgrade_are_rejected(self): + file = self.bundle() + for candidate in (replace(self.destination, target_id=TARGET ^ 1), + replace(self.destination, hw_id='another_board'), + replace(self.destination, version=self.image.fw_version)): + with self.subTest(candidate=candidate), self.assertRaises(ota.OtaError): + bundles.load_bundle(file, candidate) + + def test_unexpected_and_duplicate_members_are_rejected(self): + for member in ('extra.json', 'firmware.bin'): + file = self.bundle() + with warnings.catch_warnings(), zipfile.ZipFile(file, 'a') as archive: + warnings.simplefilter('ignore', UserWarning) + archive.writestr(member, b'not another input') + with self.subTest(member=member), self.assertRaisesRegex(ota.OtaError, 'members'): + bundles.load_bundle(file, self.destination) + + def test_save_refuses_a_package_for_a_different_complete_image(self): + with self.assertRaises(ota.OtaError): + bundles.save_bundle(self.path/'wrong.zip', image(body=b'other'), self.package) + + def test_invalid_request_fields_fail_as_user_errors(self): + file = self.path/'request.json' + for field, value in (('public_key', 'short'), ('target_id', -1), + ('platform', 'unknown'), ('base_hash', 'abcd'), + ('max_block_size', 4096)): + bundles.save_request(file, self.destination) + contents = json.loads(file.read_text()) + contents['destination'][field] = value + file.write_text(json.dumps(contents)) + with self.subTest(field=field), self.assertRaises(ota.OtaError): + bundles.load_request(file) + + def test_signed_full_input_preserves_signature_and_is_usable_by_sd_installer(self): + original = signed_package(self.image) + path = self.path/'signed-full.mota' + path.write_bytes(original.blob) + package = bundles.load_ready_application(path, self.image, + replace(self.destination, sd_storage=True)) + self.assertEqual(package, original) + self.assertEqual(package.blob, path.read_bytes()) + self.assertEqual(package.body_hash, self.image.body_hash) + + def test_unsigned_full_input_remains_rejected_for_sd(self): + path = self.path/'unsigned-full.mota' + path.write_bytes(self.package.blob) + self.assertEqual(bundles.load_ready_application(path, self.image, self.destination), self.package) + with self.assertRaisesRegex(ota.OtaError, 'trusted signed'): + bundles.load_ready_application(path, self.image, replace(self.destination, sd_storage=True)) + + def test_supplied_mota_is_bound_to_exact_image_target_hardware_version_and_codec(self): + original = signed_package(self.image) + path = self.path/'signed-full.mota' + path.write_bytes(original.blob) + with self.assertRaisesRegex(ota.OtaError, 'selected complete firmware'): + bundles.load_ready_application(path, image(body=b'another complete image'), self.destination) + for live in (replace(self.destination, target_id=TARGET ^ 1), + replace(self.destination, hw_id='different_board'), + replace(self.destination, version=self.image.fw_version), + replace(self.destination, codecs=4)): + with self.subTest(live=live), self.assertRaises(ota.OtaError): + bundles.load_ready_application(path, self.image, live) + + def test_corrupted_signature_and_non_full_input_are_rejected(self): + original = signed_package(self.image) + path = self.path/'signed-full.mota' + broken = bytearray(original.blob) + broken[8 + 129] ^= 1 # Fixed-layout manifest signature; payload remains intact. + path.write_bytes(broken) + with self.assertRaisesRegex(ota.OtaError, 'verification failed'): + bundles.load_ready_application(path, self.image, self.destination) + lib = ota.bootloader_library() + manifest = lib.build_manifest(target_id=TARGET, fw_version=self.image.fw_version, + image_size=len(self.image.image), payload=b'patch', block_size=2048, + image_hash=hashlib.sha256(self.image.image).digest(), codec_id=lib.CODEC_DETOOLS_INPLACE, + is_full=False, hw_id=HW, base_hash=self.destination.base_hash) + path.write_bytes(lib.build_container(manifest, b'patch')) + with self.assertRaisesRegex(ota.OtaError, 'full application'): + bundles.load_ready_application(path, self.image, self.destination) + + def test_raw_input_does_not_require_a_ready_package(self): + self.assertIsNone(bundles.load_ready_application(self.path/'not-opened.bin', + self.image, self.destination)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/lora_ota/test_ota_wizard_catalog.py b/tools/lora_ota/test_ota_wizard_catalog.py new file mode 100644 index 000000000..f0a5a9c8d --- /dev/null +++ b/tools/lora_ota/test_ota_wizard_catalog.py @@ -0,0 +1,244 @@ +"""Offline release selection, download, and exact-base checks for the OTA UI.""" +from __future__ import annotations + +import hashlib +import io +import json +from pathlib import Path +import struct +import tempfile +import unittest +from unittest import mock +from urllib.parse import quote +import zipfile + +import lora_ota as ota +import ota_wizard_catalog as catalog +from test_lora_ota import firmware, mota_blob + + +ENV = "ikoka_stick_nrf_33dbm_repeater" +HW = "ikoka_stick_nrf_33dbm" +TARGET = int.from_bytes(hashlib.sha256(ENV.encode()).digest()[:4], "little") +FAMILY = "v1.18.0.1-halo-keymind-cascade-dev-03214a16" +OLD_FAMILY = "v1.17.1.9-halo-keymind-cascade-dev-43f474a8" +VERSION = 0x01120001 + + +def asset_row(name=None, tag=FAMILY, data=b"firmware", digest=True): + name = name or f"{ENV}-full-ota-{FAMILY}.zip" + return {"name": name, "size": len(data), + "browser_download_url": f"https://github.com/mikecarper/MeshCore/releases/download/{quote(tag, safe='')}/{quote(name, safe='')}", + "digest": "sha256:" + hashlib.sha256(data).hexdigest() if digest else None} + + +def release(tag, assets=(), **kwargs): + return {"tag_name": tag, "draft": False, "prerelease": False, + "assets": list(assets), "published_at": "2026-10-20T00:00:00Z", **kwargs} + + +class Response(io.BytesIO): + def __init__(self, data, url): + super().__init__(data) + self.url = url + + def geturl(self): + return self.url + + +class CatalogTest(unittest.TestCase): + def test_current_release_family_includes_chunk_ten_but_not_board_previews(self): + latest = asset_row() + old = asset_row(f"{ENV}-full-ota-{OLD_FAMILY}.zip", OLD_FAMILY) + reduced = asset_row(f"{ENV}-reduced-ota-{FAMILY}.zip", f"lora-ota-10-{FAMILY}") + rows = [release("w12-v1.19.0.1", [asset_row()]), + release("v1.19.0.1-preview.2", [asset_row()]), + release(OLD_FAMILY, [old], published_at="2026-10-25T00:00:00Z"), + release(FAMILY), release(f"lora-ota-2-{FAMILY}", [ + {**latest, "browser_download_url": latest["browser_download_url"].replace( + f"/download/{FAMILY}/", f"/download/lora-ota-2-{FAMILY}/")}]), + release(f"lora-ota-10-{FAMILY}", [reduced])] + calls = [] + def opener(request, timeout): + calls.append(request.full_url) + return Response(json.dumps(rows).encode(), request.full_url) + source = catalog.GitHubCatalog(opener=opener) + current = source.fetch_assets() + self.assertEqual(len(current), 2) + self.assertEqual(source.choose_asset(target_id=TARGET).profile, "full") + self.assertEqual(source.choose_asset(ENV, profile="reduced").profile, "reduced") + bases = source.base_candidates(target_id=TARGET, fw_version=0x01110109) + self.assertEqual([row.name for row in bases], [old["name"]]) + self.assertEqual(len(calls), 1) + + def test_exact_target_role_profile_and_embedded_variants(self): + full = catalog.parse_release_asset(asset_row(), FAMILY) + embedded = catalog.parse_release_asset(asset_row( + f"{ENV}-with-bootloader-2.4.12-full-ota-{FAMILY}.zip"), FAMILY) + room = catalog.parse_release_asset(asset_row( + f"ikoka_stick_nrf_33dbm_room_server_lora_ota-full-ota-{FAMILY}.zip"), FAMILY) + companion = catalog.parse_release_asset(asset_row( + f"ikoka_stick_nrf_33dbm_companion_radio_full-{FAMILY}.zip"), FAMILY) + self.assertEqual(full.target_id, TARGET) + self.assertTrue(embedded.embedded_bootloader) + self.assertIsNotNone(companion) + source = catalog.GitHubCatalog() + source._assets = [embedded, room, full] + self.assertEqual(source.candidates(target_id=TARGET), [full]) + self.assertEqual(source.candidates(ENV, profile="reduced"), []) + self.assertEqual(source.candidates(ENV, include_embedded=True), [full, embedded]) + with self.assertRaises(ota.OtaError): + source.candidates() + with self.assertRaises(ota.OtaError): + source.choose_asset("another_repeater") + + def test_untrusted_asset_metadata_is_not_selectable(self): + for mutation in ({"browser_download_url": "http://github.com/file.zip"}, + {"browser_download_url": "https://example.com/file.zip"}, + {"size": True}, {"size": 0}, + {"size": catalog.MAX_DOWNLOAD_BYTES + 1}, + {"digest": "sha1:1234"}, + {"name": "../escape.zip"}): + with self.subTest(mutation=mutation): + self.assertIsNone(catalog.parse_release_asset({**asset_row(), **mutation}, FAMILY)) + self.assertIsNone(catalog.parse_release_asset(asset_row( + f"{ENV}-full-ota-{FAMILY}-merged.bin"), FAMILY)) + + def test_invalid_catalog_and_redirect_are_reported_before_selection(self): + for data in (b"not JSON", b"{}", b"[null]"): + with self.subTest(data=data), self.assertRaises(ota.OtaError): + catalog.GitHubCatalog(opener=lambda req, timeout: Response(data, req.full_url)).fetch_assets() + with self.assertRaisesRegex(ota.OtaError, "unexpected"): + catalog.GitHubCatalog(opener=lambda req, timeout: Response(b"[]", "http://127.0.0.1/")).fetch_assets() + with self.assertRaisesRegex(ota.OtaError, "unexpected"): + handler = catalog._GithubRedirects(api=False) + handler.redirect_request(None, None, 302, "redirect", {}, "http://localhost/file") + + def test_bounded_atomic_download_cache_and_sha256(self): + data = b"published application" * 500 + row = catalog.parse_release_asset(asset_row(data=data), FAMILY) + calls = [] + def opener(request, timeout): + calls.append(request.full_url) + return Response(data, "https://release-assets.githubusercontent.com/example?token=public") + with tempfile.TemporaryDirectory() as folder: + path = catalog.download_asset(row, Path(folder), opener=opener) + self.assertEqual(path.read_bytes(), data) + self.assertEqual(catalog.download_asset(row, Path(folder), opener=opener), path) + self.assertEqual(len(calls), 1) + path.write_bytes(b"x" * len(data)) + self.assertEqual(catalog.download_asset(row, Path(folder), opener=opener).read_bytes(), data) + self.assertEqual(len(calls), 2) + self.assertFalse(list(Path(folder).glob("*.tmp"))) + + def test_incomplete_oversize_digest_or_host_failure_leaves_no_download(self): + row = catalog.parse_release_asset(asset_row(data=b"abcd"), FAMILY) + for data, url in ((b"abc", row.url), (b"abcde", row.url), + (b"xxxx", row.url), (b"abcd", "https://github.com.evil.test/file")): + with self.subTest(data=data, url=url), tempfile.TemporaryDirectory() as folder: + with self.assertRaises(ota.OtaError): + catalog.download_asset(row, Path(folder), opener=lambda req, timeout: Response(data, url)) + self.assertEqual(list(Path(folder).iterdir()), []) + + def test_older_github_assets_without_digest_have_checked_local_cache(self): + row = catalog.parse_release_asset(asset_row(digest=False), FAMILY) + with tempfile.TemporaryDirectory() as folder: + opener = mock.Mock(side_effect=lambda req, timeout: Response(b"firmware", req.full_url)) + path = catalog.download_asset(row, Path(folder), opener=opener) + catalog.download_asset(row, Path(folder), opener=opener) + self.assertEqual(opener.call_count, 1) + path.with_name(path.name + ".json").write_text("[]") + catalog.download_asset(row, Path(folder), opener=opener) + self.assertEqual(opener.call_count, 2) + + +class ImageCacheTest(unittest.TestCase): + def setUp(self): + self.image = firmware(b"exact running firmware" * 20, VERSION, TARGET, HW) + + def dfu_zip(self, path, image=None, member="application.bin"): + with zipfile.ZipFile(path, "w") as archive: + archive.writestr("manifest.json", json.dumps({"manifest": {"application": {"bin_file": member}}})) + archive.writestr(member, image or self.image) + + def test_dfu_application_identity_and_persistent_exact_base_cache(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + source = root / "application.zip" + self.dfu_zip(source) + cache = catalog.ImageCache(root / "cache") + identity = catalog.load_application(source, target_id=TARGET, hw_id=HW) + self.assertEqual(identity.image, self.image) + path = cache.store(source, target_id=TARGET, hw_id=HW) + self.assertEqual(cache.find(identity.body_hash, TARGET, HW, fw_version=VERSION), path) + self.assertEqual(catalog.ImageCache(root / "cache").find(identity.body_hash.hex(), TARGET, HW), path) + self.assertIsNone(cache.find(identity.body_hash, TARGET + 1, HW)) + self.assertIsNone(cache.find(identity.body_hash, TARGET, "OtherBoard")) + self.assertIsNone(cache.find(identity.body_hash, TARGET, HW, fw_version=VERSION + 1)) + path.write_bytes(self.image[:-1] + b"!") + self.assertIsNone(cache.find(identity.body_hash, TARGET, HW)) + + def test_wrong_target_hash_and_unsafe_dfu_are_rejected(self): + with tempfile.TemporaryDirectory() as folder: + source = Path(folder) / "application.zip" + self.dfu_zip(source) + with self.assertRaisesRegex(ota.OtaError, "target"): + catalog.load_application(source, target_id=TARGET + 1) + self.dfu_zip(source, self.image[:1] + b"!" + self.image[2:]) + with self.assertRaisesRegex(ota.OtaError, "hash"): + catalog.load_application(source) + self.dfu_zip(source, member="../application.bin") + with self.assertRaisesRegex(ota.OtaError, "unsafe"): + catalog.load_application(source) + + def test_generic_archive_requires_unambiguous_matching_target(self): + with tempfile.TemporaryDirectory() as folder: + source = Path(folder) / "application.zip" + other = firmware(b"other board", VERSION, TARGET + 1, "OtherBoard") + with zipfile.ZipFile(source, "w") as archive: + archive.writestr("target.bin", self.image) + archive.writestr("other.bin", other) + self.assertEqual(catalog.load_application(source, target_id=TARGET).image, self.image) + with self.assertRaisesRegex(ota.OtaError, "unambiguous"): + catalog.load_application(source) + + def test_full_mota_is_a_base_but_delta_is_not(self): + with tempfile.TemporaryDirectory() as folder: + source = Path(folder) / "application.mota" + source.write_bytes(mota_blob(self.image, version=VERSION, target=TARGET, hw=HW)) + self.assertEqual(catalog.load_application(source, target_id=TARGET, hw_id=HW).image, self.image) + source.write_bytes(mota_blob(self.image, full=False, version=VERSION, target=TARGET, hw=HW)) + with self.assertRaisesRegex(ota.OtaError, "full application"): + catalog.load_application(source) + + def test_body_hash_does_not_allow_replacing_a_different_trailer(self): + with tempfile.TemporaryDirectory() as folder: + cache = catalog.ImageCache(Path(folder)) + identity = ota.parse_endf(self.image) + cache.remember(identity) + changed = bytearray(self.image) + struct.pack_into(" 0) + self.guard_busy = max(0, self.guard_busy - 1) + return await self.response("run_cli_command", event("cli_reply", text=f"> ota.direct v=1 active=1 busy={busy}")) + return await self.response("run_cli_command", event("cli_reply", text=self.temporary)) + + async def send(self, data, expected): + assert data[:1] == b"\x34" and len(data) == 34 and data[2:].hex() == KEY + self.guard_modes.append(data[1]) + if not self.guard_supported: + return await self.response("send", event("command_error", reason="unsupported")) + if data[1] == 2: + self.guard_busy = 1 + return await self.response("send") + + async def set_time(self, timestamp): + assert timestamp >= self.clock, "The host must never turn the source clock back" + self.time_sets.append(timestamp) + if not self.ignore_clock_advance: + self.clock = timestamp + return await self.response("set_time") + + async def get_time(self): + return await self.response("get_time", event("current_time", time=self.clock)) + + async def update_contact(self, value): + self.sdk.contacts[value["public_key"]] = deepcopy(value) + return await self.response("update_contact") + + async def remove_contact(self, key): + self.sdk.contacts.pop(key, None) + return await self.response("remove_contact") + + async def send_node_discover_req(self, filter, prefix_only, tag): + assert filter == 0x0C and prefix_only is False + fields = dict(tag=tag.to_bytes(4, "little").hex(), pubkey=KEY, + node_type=2, path_len=0, RSSI=-70, SNR=7) + self.sdk.emit(Types.DISCOVER_RESPONSE, **fields) + for changed in (dict(tag="00000000"), dict(pubkey=OTHER, path_len=1), + dict(pubkey="ab" * 8), dict(pubkey=OTHER, node_type=1)): + self.sdk.emit(Types.DISCOVER_RESPONSE, **(fields | changed)) + return await self.response("send_node_discover_req") + + async def send_login(self, destination, password): + assert destination["out_path_len"] == 0 and destination["out_path"] == "" + self.login_count += 1 + # A wrong radio's success must not authenticate this one; the listener + # is registered before the USB acknowledgement can arrive. + self.sdk.emit(Types.LOGIN_SUCCESS, pubkey_prefix=OTHER[:12], is_admin=True) + self.sdk.emit(Types.LOGIN_SUCCESS if self.login_allowed else Types.LOGIN_FAILED, + pubkey_prefix=KEY[:12], is_admin=self.login_admin, + permissions=1 if self.login_admin else 0, acl_permissions=self.login_acl) + return await self.response("send_login", event("message_sent")) + + async def send_cmd(self, destination, text, timestamp=None): + assert destination["out_path_len"] == 0 + assert self.guard_busy == 0, "CLI overtook the lower-priority return PATH" + self.command_times.append(timestamp) + if self.command_replies: + is_proof = text[3:] == "get public.key" + result = self.proof_key if is_proof else self.next_reply or "> verified" + reply = dict(pubkey_prefix=KEY[:12], txt_type=1, path_len=255, + text=text[:3] + result) + if not is_proof: + self.next_reply = None + stale = ("00|" if text[:3] != "00|" else "11|") + "wrong request" + for changed in (dict(pubkey_prefix=OTHER[:12]), dict(txt_type=0), + dict(path_len=2), dict(path_len=0), dict(text=stale)): + self.sdk.emit(Types.CONTACT_MSG_RECV, **(reply | changed)) + self.sdk.emit(Types.CONTACT_MSG_RECV, **reply) + return await self.response("send_cmd", event("message_sent")) + + async def send_raw_packet(self, raw, priority): + self.raw.append(raw) + return await self.response("send_raw_packet") + + +class SDK: + def __init__(self, contacts=None): + self.contacts = {KEY: contact()} if contacts is None else contacts + self.self_info = dict(public_key="ef" * 32, radio_freq=910.525, + radio_bw=62.5, radio_sf=7, radio_cr=5) + self.commands = Commands(self) + self.subscriptions = [] + self.closed = False + + def subscribe(self, kind, callback): + subscription = (kind, callback) + self.subscriptions.append(subscription) + return subscription + + def unsubscribe(self, subscription): + self.subscriptions.remove(subscription) + + def emit(self, kind, **payload): + for subscribed, callback in list(self.subscriptions): + if kind == subscribed: + callback(event(kind, **payload)) + + async def disconnect(self): + self.closed = True + + +class FakeSource: + last = None + + def __init__(self, blob, seeder_id, compressor): + FakeSource.last = self + self.stats = {"requests": 0} + self.sent = [] + + def prepare(self): + pass + + def advert(self): + return b"\x31\0\x01hostdata" + + def replies(self, raw): + self.stats["requests"] += 1 + return [b"\x31\0\x05reply"] + + def note_sent(self, frame): + self.sent.append(frame) + + +class CompanionTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.sdk = SDK() + self.original = deepcopy(self.sdk.contacts[KEY]) + + async def factory(port, **kwargs): + self.assertEqual(port, "FAKE") + self.assertTrue(kwargs["only_error"]) + self.assertFalse(kwargs["auto_reconnect"]) + return self.sdk + + self.transport = companion.CompanionTransport( + "FAKE", sdk_factory=factory, event_types=Types, + discovery_seconds=0, remote_timeout=.05, + source_factory=FakeSource) + await self.transport.open() + + async def asyncTearDown(self): + await self.transport.close() + + async def test_discovery_uses_fresh_full_key_direct_responses_only(self): + rows = await self.transport.discover() + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["public_key"], KEY) + self.assertEqual(rows[0]["rssi"], -70) + self.assertEqual(rows[0]["name"], "Hilltop") + self.assertEqual(self.sdk.contacts[KEY], self.original) + + async def test_native_settings_and_binary_local_cli_share_wire_lock(self): + self.assertEqual(self.transport.source_key, "ef" * 32) + self.assertEqual(await self.transport.get_radio(), (910.525, 62.5, 7, 5)) + await asyncio.gather(self.transport.get_radio(), self.transport.local_command("get tempradio")) + self.assertEqual(self.sdk.commands.maximum_active, 1) + await self.transport.set_radio((911, 125, 8, 5)) + self.assertEqual(await self.transport.get_radio(), (911, 125, 8, 5)) + + async def test_binary_local_cli_preserves_unsupported_capability_reason(self): + self.sdk.commands.temporary = "ERR RX power saving is unsupported on this radio" + reply = await self.transport.local_command("get radio.rxps.config") + self.assertIn("unsupported on this radio", reply) + + async def test_guest_success_cannot_authenticate_as_admin(self): + self.sdk.commands.login_admin = False + with self.assertRaisesRegex(companion.CompanionError, "admin login"): + await self.transport.remote_command(KEY, "secret", "ver") + self.assertNotIn(KEY, self.transport._authenticated) + self.assertNotIn("send_cmd", self.sdk.commands.calls) + + async def test_old_source_guard_rejection_does_not_emit_a_radio_login(self): + self.sdk.commands.guard_supported = False + with self.assertRaisesRegex(companion.CompanionError, "current Full Companion"): + await self.transport.remote_command(KEY, "secret", "ota pull DEADBEEF flash") + self.assertEqual(self.sdk.commands.login_count, 0) + self.assertNotIn("send_cmd", self.sdk.commands.calls) + + async def test_manager_role_cannot_authenticate_for_ota(self): + self.sdk.commands.login_acl = 4 + with self.assertRaisesRegex(companion.CompanionError, "admin login"): + await self.transport.remote_command(KEY, "secret", "ver") + self.assertNotIn(KEY, self.transport._authenticated) + + async def test_clock_a_little_ahead_never_moves_back_and_cli_follows_login(self): + now = 1800000000 + self.sdk.commands.clock = now + 300 + with mock.patch.object(companion.time, "time", return_value=now): + await self.transport.remote_command(KEY, "secret", "ver") + self.assertEqual(self.sdk.commands.time_sets, []) + self.assertTrue(all(stamp > now + 300 for stamp in self.sdk.commands.command_times)) + self.assertLess(self.sdk.commands.calls.index("get_time"), self.sdk.commands.calls.index("send_login")) + + async def test_stale_clock_is_advanced_and_read_back_before_login(self): + now = 1800000000 + self.sdk.commands.clock = now - 3600 + with mock.patch.object(companion.time, "time", return_value=now): + await self.transport.remote_command(KEY, "secret", "ver") + self.assertEqual(self.sdk.commands.time_sets, [now + 2]) + self.assertEqual(self.sdk.commands.calls.count("get_time"), 2) + self.assertTrue(all(stamp > now + 2 for stamp in self.sdk.commands.command_times)) + + async def test_bad_or_far_future_clock_stops_before_radio_login(self): + now = 1800000000 + for value in (now + 601, None, "1800000000", True, -1): + self.sdk.commands.clock = value + with self.subTest(clock=value), mock.patch.object(companion.time, "time", return_value=now): + with self.assertRaisesRegex(companion.CompanionError, "clock|RTC"): + await self.transport.remote_command(KEY, "secret", "ver") + self.assertEqual(self.sdk.commands.login_count, 0) + self.assertEqual(self.sdk.commands.time_sets, []) + + async def test_unverified_clock_advance_stops_before_radio_login(self): + now = 1800000000 + self.sdk.commands.clock = now - 60 + self.sdk.commands.ignore_clock_advance = True + with mock.patch.object(companion.time, "time", return_value=now): + with self.assertRaisesRegex(companion.CompanionError, "verify.*clock advance"): + await self.transport.remote_command(KEY, "secret", "ver") + self.assertEqual(self.sdk.commands.login_count, 0) + + async def test_wrong_full_key_proof_cannot_run_the_requested_command(self): + self.sdk.commands.proof_key = OTHER + with self.assertRaisesRegex(companion.CompanionError, "complete public key"): + await self.transport.remote_command(KEY, "secret", "ota install") + self.assertNotIn(KEY, self.transport._authenticated) + self.assertEqual(len(self.transport._used_correlations), 1) + + async def test_strict_login_reply_correlation_and_path_restore(self): + for command in ("ver", "ota self"): + self.assertEqual(await self.transport.remote_command(KEY, "secret", command), "> verified") + self.assertEqual(self.sdk.commands.login_count, 1) + self.assertEqual(len(self.transport._used_correlations), 3) + self.assertEqual(self.sdk.contacts[KEY]["out_path_len"], 0) + await self.transport.restore_contact_path() + self.assertEqual(self.sdk.contacts[KEY], self.original) + self.assertEqual(self.sdk.commands.guard_modes, [1, 2, 1, 3]) + + async def test_reboot_reset_reauthenticates_without_reusing_old_correlations(self): + await self.transport.remote_command(KEY, "secret", "ver") + previous = set(self.transport._used_correlations) + await self.transport.reset_remote_session(KEY) + self.assertNotIn(KEY, self.transport._authenticated) + self.assertEqual(self.transport._used_correlations, previous) + self.assertEqual(self.sdk.contacts[KEY]["out_path_len"], 0) + await self.transport.remote_command(KEY, "secret", "ota self") + self.assertEqual(self.sdk.commands.login_count, 2) + self.assertEqual(len(self.transport._used_correlations), 4) + self.assertTrue(previous < self.transport._used_correlations) + + async def test_rejected_login_restores_route_on_close_without_password_error(self): + self.sdk.commands.login_allowed = False + with self.assertRaises(companion.CompanionError) as caught: + await self.transport.remote_command(KEY, "my-password", "ver") + self.assertNotIn("my-password", str(caught.exception)) + await self.transport.close() + self.assertEqual(self.sdk.contacts[KEY], self.original) + + async def test_new_direct_contact_removed_after_session(self): + self.sdk.contacts.clear() + await self.transport.discover() + self.assertEqual(await self.transport.remote_command(KEY, "secret", "ver"), "> verified") + await self.transport.restore_contact_path() + self.assertNotIn(KEY, self.sdk.contacts) + + async def test_ambiguous_prefix_and_unseen_key_refused(self): + with self.assertRaisesRegex(companion.CompanionError, "Discover"): + await self.transport.remote_command(OTHER, "secret", "ver") + collision = KEY[:12] + "01" * 26 + self.sdk.contacts[collision] = contact(collision) + with self.assertRaisesRegex(companion.CompanionError, "ambiguous"): + await self.transport.remote_command(KEY, "secret", "ver") + + async def test_remote_timeout_is_bounded_and_keeps_authenticated_session(self): + await self.transport.remote_command(KEY, "secret", "ver") + self.sdk.commands.command_replies = False + with self.assertRaisesRegex(TimeoutError, "command reply"): + await self.transport.remote_command(KEY, "secret", "ver") + self.assertIn(KEY, self.transport._authenticated) + self.sdk.commands.command_replies = True + self.assertEqual(await self.transport.remote_command(KEY, "secret", "ver"), "> verified") + + async def test_source_requires_temporary_lease(self): + for inactive in ("TempRadio inactive", "TempRadio pending: 910,62.5,7,5", "> off"): + self.sdk.commands.temporary = inactive + with self.assertRaisesRegex(companion.CompanionError, "TempRadio"): + await self.transport.serve(b"package", asyncio.Event()) + self.assertEqual(self.sdk.commands.raw, []) + + async def test_only_explicit_expired_auth_renews_login(self): + await self.transport.remote_command(KEY, "secret", "ver") + self.sdk.commands.next_reply = "ERR login required" + self.assertEqual(await self.transport.remote_command(KEY, "secret", "ver"), "> verified") + self.assertEqual(self.sdk.commands.login_count, 2) + self.sdk.commands.next_reply = "ERR not permitted" + self.assertEqual(await self.transport.remote_command(KEY, "secret", "erase"), "ERR not permitted") + self.assertEqual(self.sdk.commands.login_count, 2) + + async def test_disconnect_has_connection_error_contract(self): + self.sdk.emit(Types.DISCONNECTED) + with self.assertRaises(ConnectionError): + await self.transport.get_radio() + # Restore fixture connectivity so teardown can restore the saved path. + self.transport._disconnected = False + + async def test_source_direct_only_with_simultaneous_remote_admin(self): + stop = asyncio.Event() + with mock.patch.object(companion, "packet_airtime", return_value=0): + serving = asyncio.create_task(self.transport.serve(b"package", stop)) + for _ in range(200): + if self.sdk.commands.raw: + break + await asyncio.sleep(.005) + self.assertTrue(self.sdk.commands.raw) + # Actual SDK raw_hex = SNR/RSSI + original RF frame. Its packet + # parser may have replaced payload, so that field is misleading. + for raw in (b"\x31\x01\x44\x04request", b"\xf2\0\x04request", + b"\x32\0\x04request"): + self.sdk.emit(Types.RX_LOG_DATA, raw_hex=(b"\x10\xba" + raw).hex(), + payload="001122", path_len=0) + self.assertEqual(await self.transport.remote_command(KEY, "secret", "ota status"), "> verified") + for _ in range(200): + if len(self.sdk.commands.raw) >= 2: + break + await asyncio.sleep(.005) + stop.set() + result = await asyncio.wait_for(serving, 2) + self.assertEqual(result["requests"], 1) + self.assertEqual(len(self.sdk.commands.raw), 2) + self.assertTrue(all(frame[:2] == b"\x32\0" for frame in self.sdk.commands.raw)) + self.assertEqual(self.sdk.commands.maximum_active, 1) + + async def test_real_package_source_answers_manifest_using_direct_frames(self): + # Exercise the reused real source, not only the adapter test double. + from mota_kiss_direct import motalib + import hashlib + body = b"real firmware image" * 80 + manifest = motalib.build_manifest( + target_id=0x12345678, fw_version=0x01120001, + image_size=len(body), payload=body, block_size=1024, + image_hash=hashlib.sha256(body).digest(), codec_id=0, + is_full=True, hw_id="test-radio") + blob = motalib.build_container(manifest, body) + self.transport._source_factory = None + stop = asyncio.Event() + with mock.patch.object(companion, "packet_airtime", return_value=0): + serving = asyncio.create_task(self.transport.serve(blob, stop)) + for _ in range(200): + if self.sdk.commands.raw: + break + await asyncio.sleep(.005) + request = b"\x32\0\x04" + manifest.merkle_root + b"\xff\xff" + self.sdk.emit(Types.RX_LOG_DATA, raw_hex=(b"\x10\xba" + request).hex()) + for _ in range(200): + if len(self.sdk.commands.raw) >= 3: + break + await asyncio.sleep(.005) + stop.set() + await asyncio.wait_for(serving, 2) + self.assertEqual(len(self.sdk.commands.raw), 3) + self.assertEqual([frame[2] for frame in self.sdk.commands.raw], [1, 5, 5]) + self.assertTrue(all(frame[:2] == b"\x32\0" for frame in self.sdk.commands.raw)) + + +class PureHelpers(unittest.TestCase): + def test_sdk_rx_shapes_and_malformed_logs(self): + raw = b"\x32\0\x04request" + self.assertEqual(companion.raw_rx_bytes({"raw_hex": "10ba" + raw.hex()}), raw) + self.assertEqual(companion.raw_rx_bytes({"payload": raw.hex()}), raw) + self.assertEqual(companion.raw_rx_bytes({"raw_hex": "badhex"}), b"") + + def test_airtime_increases_with_sf_and_payload(self): + self.assertGreater(companion.packet_airtime(180, (910, 62.5, 8, 5)), + companion.packet_airtime(180, (910, 62.5, 7, 5))) + self.assertGreater(companion.packet_airtime(180, (910, 62.5, 7, 5)), + companion.packet_airtime(20, (910, 62.5, 7, 5))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/lora_ota/test_ota_wizard_packages.py b/tools/lora_ota/test_ota_wizard_packages.py new file mode 100644 index 000000000..487ede11b --- /dev/null +++ b/tools/lora_ota/test_ota_wizard_packages.py @@ -0,0 +1,248 @@ +"""Offline package planning, builder fallback, and pinned bootloader bundles.""" +from __future__ import annotations + +from dataclasses import replace +import hashlib +import io +import json +from pathlib import Path +import subprocess +import tempfile +from types import SimpleNamespace +import unittest +from unittest import mock +import zipfile + +import lora_ota as ota +import ota_wizard_packages as packages +import ota_wizard_session as session +from test_lora_ota import mota_blob +from test_lora_ota_bootloader import boot_blob +from test_ota_wizard_session import (application, boot_status, BOOT_NEW, BOOT_OLD, + HW, KEY, OLD, NEW, TARGET, TEST_SIGNER) + + +def destination(**changes): + current = application(OLD, b"old application" * 600) + result = session.Destination(KEY, TARGET, HW, current.body_hash, OLD, "v1.17.1.9", + "nrf52", True, 2048, 5, + ota.parse_bootloader_status(boot_status()), BOOT_OLD) + return replace(result, **changes) + + +def delta_blob(image, base, **changes): + arguments = dict(full=False, version=image.fw_version, target=image.target_id, + hw=image.hw_id, base_hash=base.body_hash, + codec=ota.MOTA_CODEC_IN_PLACE, payload=b"example patch") + arguments.update(changes) + return mota_blob(image.image, **arguments) + + +class PackageTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.cache = Path(self.temporary.name) + self.new = application() + self.old = application(OLD, b"old application" * 600) + self.base = self.cache / "base.bin" + self.base.write_bytes(self.old.image) + + def test_full_package_works_without_network_or_native_builder(self): + with mock.patch.object(packages.subprocess, "run", side_effect=AssertionError("must not compile")): + result = packages.prepare_application(self.new, destination(), self.cache, progress=lambda _: None) + self.assertTrue(result.info.is_full) + self.assertEqual((result.info.target_id, result.info.hw_id, result.body_hash), + (TARGET, HW, self.new.body_hash)) + self.assertEqual(result.info.image_hash, hashlib.sha256(self.new.image).digest()) + + def test_exact_base_delta_uses_in_place_and_expected_new_image(self): + commands = [] + def build(command, **_): + commands.append(command) + Path(command[command.index("--out") + 1]).write_bytes(delta_blob(self.new, self.old)) + return SimpleNamespace(returncode=0) + with mock.patch.object(packages.subprocess, "run", side_effect=build): + result = packages.prepare_application(self.new, destination(), self.cache, + base=self.base, builder="fake-builder", progress=lambda _: None) + self.assertFalse(result.info.is_full) + self.assertEqual(result.info.base_hash, self.old.body_hash) + self.assertIn("in-place", commands[0]) + self.assertEqual(result.body_hash, self.new.body_hash) + + def test_failed_timed_out_and_malformed_delta_fall_back_to_full(self): + for behavior in ("exit", "timeout", "malformed", "wrong-image", "missing"): + with self.subTest(behavior=behavior): + def build(command, **_): + output = Path(command[command.index("--out") + 1]) + if behavior == "timeout": + raise subprocess.TimeoutExpired(command, 1) + if behavior == "malformed": + output.write_bytes(b"broken package") + if behavior == "wrong-image": + wrong = application(NEW, b"another new image" * 600) + output.write_bytes(delta_blob(wrong, self.old)) + return SimpleNamespace(returncode=1 if behavior == "exit" else 0) + with mock.patch.object(packages.subprocess, "run", side_effect=build): + result = packages.prepare_application(self.new, destination(), self.cache, + base=self.base, builder="fake-builder", progress=lambda _: None) + self.assertTrue(result.info.is_full) + + def test_internal_nrf_rejects_full_fallback(self): + with mock.patch.object(packages.subprocess, "run", return_value=SimpleNamespace(returncode=1)): + with self.assertRaisesRegex(ota.OtaError, "Internal-flash"): + packages.prepare_application(self.new, destination(external_storage=False), self.cache, + base=self.base, builder="fake-builder", progress=lambda _: None) + + def test_explicit_wrong_base_is_rejected_before_builder(self): + self.base.write_bytes(application(OLD, b"different old image" * 600).image) + with mock.patch.object(packages.subprocess, "run") as build: + with self.assertRaisesRegex(ota.OtaError, "exact running firmware"): + packages.prepare_application(self.new, destination(), self.cache, + base=self.base, builder="fake-builder", progress=lambda _: None) + build.assert_not_called() + + def test_stale_delta_file_is_removed_before_build(self): + directory = self.cache / "packages" / f"{TARGET:08X}" + directory.mkdir(parents=True) + output = directory / f"{self.old.body_hash.hex()}-{self.new.body_hash.hex()}-delta.mota" + output.write_bytes(delta_blob(self.new, self.old)) + def build(*_, **__): + self.assertFalse(output.exists()) + return SimpleNamespace(returncode=0) + with mock.patch.object(packages.subprocess, "run", side_effect=build): + result = packages.prepare_application(self.new, destination(), self.cache, + base=self.base, builder="fake-builder", progress=lambda _: None) + self.assertTrue(result.info.is_full) + + def test_target_role_codec_blocksize_and_expected_body_are_required(self): + package = session.full_package(self.new, 1024) + for change, error in ((dict(target_id=TARGET ^ 1), "different board"), + (dict(hw_id="wrong-role"), "different board"), + (dict(max_block_size=512), "block size"), + (dict(codecs=4), "codec")): + with self.subTest(change=change), self.assertRaisesRegex(ota.OtaError, error): + session.validate_package(package, destination(**change)) + with self.assertRaisesRegex(ota.OtaError, "body hash"): + session.validate_package(session.Package(package.blob), destination()) + + def test_published_base_search_skips_bad_candidate_and_checks_hash(self): + good = self.cache / "published.bin" + good.write_bytes(self.old.image) + source = mock.Mock() + source.base_candidates.return_value = ["broken", "valid"] + with mock.patch.object(packages, "download_asset", side_effect=[ota.OtaError("bad ZIP"), good]): + found = packages.find_running_image(destination(), self.cache, source, progress=lambda _: None) + self.assertEqual(ota.parse_endf(found.read_bytes()).body_hash, self.old.body_hash) + source.base_candidates.assert_called_once_with(target_id=TARGET, fw_version=OLD, limit=4) + + def test_raspberry_pi_does_not_install_native_tool(self): + with mock.patch.object(packages, "small_host", return_value=True), \ + mock.patch.object(packages.subprocess, "run") as run: + self.assertIsNone(packages.install_builder(self.cache)) + run.assert_not_called() + + def test_apple_silicon_and_windows_arm_keep_workstation_features(self): + for system in ("Darwin", "Windows"): + with self.subTest(system=system), \ + mock.patch.object(packages.platform, "system", return_value=system), \ + mock.patch.object(packages.platform, "machine", return_value="arm64"), \ + mock.patch.object(packages.Path, "read_text", side_effect=AssertionError("no Linux probing")): + self.assertFalse(packages.small_host()) + + def test_pi_board_model_remains_low_power_even_with_large_ram(self): + with mock.patch.object(packages.platform, "system", return_value="Linux"), \ + mock.patch.object(packages.Path, "read_text", return_value="Raspberry Pi 5 Model B Rev 1.0\0"), \ + mock.patch.object(packages.os, "cpu_count", return_value=8): + self.assertTrue(packages.small_host()) + + def test_linux_arm_fallback_requires_both_low_ram_and_few_cores(self): + for ram_kb, cores, expected in ((2 * 1024 * 1024, 4, True), + (8 * 1024 * 1024, 4, False), + (2 * 1024 * 1024, 8, False), + (2 * 1024 * 1024, None, False)): + with self.subTest(ram=ram_kb, cores=cores), \ + mock.patch.object(packages.platform, "system", return_value="Linux"), \ + mock.patch.object(packages.platform, "machine", return_value="aarch64"), \ + mock.patch.object(packages.Path, "read_text", side_effect=[OSError("no model"), f"MemTotal: {ram_kb} kB\n"]), \ + mock.patch.object(packages.os, "cpu_count", return_value=cores): + self.assertEqual(packages.small_host(), expected) + + def test_native_setup_is_pinned_and_strips_password_environment(self): + calls = [] + def run(command, **kwargs): + calls.append((command, kwargs)) + return SimpleNamespace(returncode=0, stdout="rustc 1.86.0") + with mock.patch.object(packages, "small_host", return_value=False), \ + mock.patch.object(packages.shutil, "which", return_value="tool"), \ + mock.patch.object(packages.subprocess, "run", side_effect=run), \ + mock.patch.dict(packages.os.environ, {"MESHCORE_ADMIN_PASSWORD": "private-password"}): + packages.install_builder(self.cache, progress=lambda _: None) + command, options = calls[-1] + self.assertIn(packages.MOTATOOL_REV, command) + self.assertIn("--locked", command) + self.assertNotIn("MESHCORE_ADMIN_PASSWORD", options["env"]) + self.assertNotIn("private-password", str(calls)) + + def boot_bundle(self, *, storage=14, key=TEST_SIGNER, checksum=True): + blob = boot_blob(storage=storage, version=BOOT_NEW) + info = ota.parse_mota(blob) + path = self.cache / "bootloaders.zip" + inventory = {"signing_public_key": key, "packages": [{ + "hardware_id": info.hw_id, "target_id": f"0x{info.target_id:08X}", + "file": "boot.mota", "sha256": hashlib.sha256(blob).hexdigest() if checksum else "0" * 64}]} + with zipfile.ZipFile(path, "w") as archive: + archive.writestr("manifest.json", json.dumps(inventory)) + archive.writestr("OTAFIX_MOTA_SIGNING_PUBLIC_KEY.txt", key + "\n") + archive.writestr("mota/boot.mota", blob) + release = {"tag_name": "v0.11.0-OTAFIX2.4.12", "assets": [{ + "name": "OTAFIX-2.4.12-bootloader-mota.zip", "size": path.stat().st_size, + "browser_download_url": "https://github.com/mikecarper/Adafruit_nRF52_Bootloader_OTAFIX/releases/download/v0.11.0-OTAFIX2.4.12/OTAFIX-2.4.12-bootloader-mota.zip"}]} + return path, json.dumps(release).encode() + + def prepare_boot(self, dest=None, **bundle_options): + path, release = self.boot_bundle(**bundle_options) + with mock.patch.object(packages, "OFFICIAL_BOOT_KEY", TEST_SIGNER), \ + mock.patch.object(packages, "_open", return_value=io.BytesIO(release)), \ + mock.patch.object(packages, "download_asset", return_value=path): + return packages.prepare_bootloader(dest or destination(), self.cache) + + def test_bootloader_bundle_matches_pinned_key_and_exact_storage(self): + result = self.prepare_boot() + self.assertEqual((result.info.fw_version, result.info.bootloader_storage, result.signer), + (BOOT_NEW, 14, TEST_SIGNER)) + self.assertIsNone(self.prepare_boot(destination(boot_version=BOOT_NEW))) + self.assertIsNone(self.prepare_boot(destination(boot_version=None))) + + def test_bootloader_wrong_key_checksum_and_storage_are_rejected(self): + for options, error in ((dict(key="00" * 32), "pinned publisher"), + (dict(checksum=False), "checksum"), + (dict(storage=10), "exact installed identity")): + with self.subTest(options=options), self.assertRaisesRegex(ota.OtaError, error): + self.prepare_boot(**options) + + def test_bootloader_signature_cannot_be_approved_by_a_different_key(self): + package = session.Package(boot_blob(storage=14, version=BOOT_NEW), signer="00" * 32) + with self.assertRaises(ota.OtaError): + session.validate_package(package, destination()) + + def test_unsigned_sd_application_cannot_claim_a_trusted_signer(self): + full = session.full_package(self.new, 1024) + claimed = session.Package(full.blob, self.new.body_hash, TEST_SIGNER) + with self.assertRaisesRegex(ota.OtaError, "signing key"): + session.validate_package(claimed, destination(sd_storage=True)) + + def test_bootloader_requires_supported_platform_codecs_and_block_size(self): + package = session.Package(boot_blob(storage=14, version=BOOT_NEW), signer=TEST_SIGNER) + for change in (dict(platform="esp32"), dict(codecs=4), dict(max_block_size=512)): + with self.subTest(change=change), self.assertRaises(ota.OtaError): + session.validate_package(package, destination(**change)) + + def test_esp32_rejects_an_in_place_delta(self): + package = session.Package(delta_blob(self.new, self.old), self.new.body_hash) + with self.assertRaisesRegex(ota.OtaError, "sequential"): + session.validate_package(package, destination(platform="esp32")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/lora_ota/test_ota_wizard_repeater.py b/tools/lora_ota/test_ota_wizard_repeater.py new file mode 100644 index 000000000..649a3858e --- /dev/null +++ b/tools/lora_ota/test_ota_wizard_repeater.py @@ -0,0 +1,353 @@ +"""Offline one-port repeater broker tests, including binary/ASCII interleaving.""" +from __future__ import annotations + +import asyncio +from datetime import datetime, timezone +import queue +import struct +import unittest +from unittest import mock +import zlib + +import lora_ota as ota +from ota_wizard_repeater import RepeaterError, RepeaterTransport, SeederData, StreamDecoder +from test_lora_ota import firmware, mota_blob + + +SOURCE = "a" * 64 +TARGET = "b" * 64 + + +def request(op, args=b""): + checksum = op + for byte in args: + checksum ^= byte + return b"MS" + bytes((op,)) + args + bytes((checksum,)) + + +def package(): + image = firmware(b"repeating application bytes" * 100, 0x01120001) + return mota_blob(image, version=0x01120001) + + +class SeederTest(unittest.TestCase): + def test_exact_wire_geometry_read_limits_and_checksum_independent_payload(self): + blob = package() + core = SeederData(blob) + self.assertEqual(core.request(1, b""), (0, b"\1")) + status, desc = core.request(2, b"\0") + self.assertEqual(status, 0) + self.assertEqual(len(desc), 38) + self.assertEqual(desc[:4], core.info.merkle_root) + self.assertEqual(struct.unpack_from(" done\r\n" + for byte in wire: + events.extend(stream.feed(bytes((byte,)))) + self.assertEqual(events, [("line", "LOG MSG text"), + ("request", 3, struct.pack(" done")]) + self.assertEqual(stream.feed(request(1)[:-1] + b"\0"), []) + self.assertEqual(stream.feed(request(1)), [("request", 1, b"")]) + + def test_decoder_expires_old_partial_frame_without_dropping_fresh_data(self): + stream = StreamDecoder(frame_timeout=1) + with mock.patch("ota_wizard_repeater.time.monotonic", return_value=1): + self.assertEqual(stream.feed(b"MS\3"), []) + with mock.patch("ota_wizard_repeater.time.monotonic", return_value=3): + self.assertEqual(stream.feed(request(1)), [("request", 1, b"")]) + + +class FakeSerial: + def __init__(self, **kwargs): + self.is_open = False + self.incoming = queue.Queue() + self.pending = bytearray() + self.commands = [] + self.writes = [] + self.binary = [] + self.port = None + self.job = 0 + self.result = "" + self.result_error = False + self.result_pending = 0 + self.authenticated = False + self.clock = None + self.interleave = False + self.attaching = False + self.stale_job = False + self.reject_login_once = False + self.folder_off = False + self.peer_type = 2 + self.peer_count = 1 + self.peer_key = TARGET + self.command_results = {"ver": "v1.18.0.1", "stats-core": "x" * 150} + + def open(self): + self.is_open = True + + def close(self): + self.is_open = False + + def feed(self, data): + self.incoming.put(bytes(data)) + + def read(self, size): + if not self.pending: + try: + self.pending.extend(self.incoming.get(timeout=.02)) + except queue.Empty: + return b"" + data = bytes(self.pending[:size]) + del self.pending[:size] + return data + + def answer(self, text): + self.feed((" -> " + text + "\r\n").encode()) + + def start_job(self, result, error=False): + self.job += 1 + self.result, self.result_error = result, error + self.answer(f"OTApeer job={self.job:08X} state=pending") + + def write(self, raw): + data = bytes(raw) + self.writes.append(data) + if data.startswith(b"ms"): + checksum = 0 + for byte in data: + checksum ^= byte + assert checksum == 0 + self.binary.append(data) + if self.attaching: + if data[2] == 1: + assert data[3:5] == b"\0\1" + self.feed(request(2, b"\0")) + elif data[2] == 2: + assert len(data) == 43 + self.feed(request(3, struct.pack(" " + SOURCE) + elif command == "ota.peer status": + self.answer(f"OTApeer v=1 session={int(self.authenticated)} busy=0 peers={self.peer_count}") + elif command == "clock": + now = self.clock or datetime.now(timezone.utc) + self.answer(f"{now.hour:02}:{now.minute:02} - {now.day}/{now.month}/{now.year} UTC") + elif command.startswith("time "): + self.answer("OK - clock set") + elif command == "ota.peer discover": + self.start_job("discovery complete") + elif command == "ota.peer peers 0": + self.answer(f"OTApeer peer=0 key={self.peer_key} type={self.peer_type} rssi=-61 snr=3") + elif command.startswith("ota.peer login "): + self.authenticated = True + self.start_job("authenticated") + elif command.startswith("ota.peer cmd "): + if self.reject_login_once: + self.reject_login_once = False + self.authenticated = False + self.answer("ERR peer login required") + else: + text = command.split(" ", 3)[3] + result = self.command_results.get(text, "OK") + self.start_job(result, self.result_error) + elif command.startswith("ota.peer result"): + offset = int(command.split()[-1]) if len(command.split()) == 3 else 0 + if self.interleave: + self.feed(request(3, struct.pack(" 910.525,62.5,7,5") + elif command.startswith("set radio "): + self.answer("OK - applying in 2s") + else: + self.answer("Unknown command") + return len(data) + + +class RepeaterTransportTest(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.serial = FakeSerial() + self.link = RepeaterTransport("fake", serial_factory=lambda **kwargs: self.serial, + open_settle=0, command_timeout=.5, remote_timeout=.8, poll_seconds=.001) + await self.link.open() + + async def asyncTearDown(self): + await self.link.close() + + async def test_source_identity_and_zero_hop_discovery(self): + self.assertEqual(self.link.source_key, SOURCE) + self.serial.result_pending = 1 + peers = await self.link.discover() + self.assertEqual(peers, [{"public_key": TARGET, "type": 2, "rssi": -61, "snr": 3}]) + + async def test_password_echo_masking_and_full_multipart_result(self): + reply = await self.link.remote_command(TARGET, "secret with spaces", "stats-core") + self.assertEqual(reply, "x" * 150) + self.assertEqual(sum(text.startswith("ota.peer login") for text in self.serial.commands), 1) + await self.link.remote_command(TARGET, "secret with spaces", "ver") + self.assertEqual(sum(text.startswith("ota.peer login") for text in self.serial.commands), 1) + + async def test_room_discovery_preserves_role_and_unknown_key_cannot_login(self): + self.serial.peer_type = 3 + peers = await self.link.discover() + self.assertEqual(peers[0]["type"], 3) + await self.link.remote_command(TARGET, "password", "ver") + self.assertTrue(any(text.startswith("ota.peer login") for text in self.serial.commands)) + await self.link.reset_remote_session(TARGET) + before = len(self.serial.commands) + self.serial.peer_key = "c" * 64 + with self.assertRaisesRegex(RepeaterError, "fresh direct discovery"): + await self.link.remote_command(TARGET, "password", "ver") + self.assertFalse(any(text.startswith("ota.peer login") for text in self.serial.commands[before:])) + + async def test_one_port_answers_binary_requests_during_remote_commands(self): + stopped = asyncio.Event() + serving = asyncio.create_task(self.link.serve(package(), stopped)) + for _ in range(100): + if not self.serial.attaching and len(self.serial.binary) >= 3: + break + await asyncio.sleep(.01) + self.assertFalse(serving.done()) + self.serial.interleave = True + result = await self.link.remote_command(TARGET, "password", "stats-core") + self.assertEqual(result, "x" * 150) + self.assertTrue(any(reply[2] == 3 for reply in self.serial.binary)) + for reply in self.serial.binary: + if reply[2] == 3: + self.assertEqual(reply[3], 0) + stopped.set() + await asyncio.wait_for(serving, 1) + self.assertTrue(self.serial.folder_off) + self.assertIsNone(self.link._seeder) + + async def test_stale_job_and_unknown_results_do_not_resend_mutations(self): + await self.link.remote_command(TARGET, "password", "ver") + self.serial.stale_job = True + with self.assertRaisesRegex(RepeaterError, "changed"): + await self.link.remote_command(TARGET, "password", "ota install") + self.assertEqual(sum(text.endswith("ota install") for text in self.serial.commands), 1) + self.serial.stale_job = False + self.serial.result_error = True + self.serial.command_results["ota install"] = "request timed out; result is unknown" + with self.assertRaises(ota.TransmissionError): + await self.link.remote_command(TARGET, "password", "ota install") + self.assertEqual(sum(text.endswith("ota install") for text in self.serial.commands), 2) + + async def test_explicit_login_required_allows_one_new_login(self): + await self.link.remote_command(TARGET, "password", "ver") + self.serial.reject_login_once = True + self.assertEqual(await self.link.remote_command(TARGET, "password", "ver"), "v1.18.0.1") + self.assertEqual(sum(text.startswith("ota.peer login") for text in self.serial.commands), 2) + + async def test_target_reboot_reset_discards_authentication_without_detaching_source(self): + await self.link.remote_command(TARGET, "password", "ver") + self.assertEqual(self.link._authenticated_key, TARGET) + await self.link.reset_remote_session(TARGET) + self.assertIsNone(self.link._authenticated_key) + self.assertEqual(self.link.source_key, SOURCE) + self.assertFalse(self.serial.folder_off) + await self.link.remote_command(TARGET, "password", "ver") + self.assertEqual(sum(text.startswith("ota.peer login") for text in self.serial.commands), 2) + + async def test_reboot_discovery_absence_is_recoverable_before_target_returns(self): + await self.link.remote_command(TARGET, "password", "ver") + await self.link.reset_remote_session(TARGET) + self.serial.peer_count = 0 + with self.assertRaisesRegex(ota.OtaError, "fresh direct discovery"): + await self.link.remote_command(TARGET, "password", "ver") + self.assertIsNone(self.link._authenticated_key) + self.assertEqual(sum(text.startswith("ota.peer login") for text in self.serial.commands), 1) + self.serial.peer_count = 1 + self.assertEqual(await self.link.remote_command(TARGET, "password", "ver"), "v1.18.0.1") + self.assertEqual(sum(text.startswith("ota.peer login") for text in self.serial.commands), 2) + self.assertFalse(self.serial.folder_off) + + async def test_clock_is_advanced_when_old_and_future_clock_stops_before_login(self): + self.serial.clock = datetime(2001, 1, 1, tzinfo=timezone.utc) + await self.link.remote_command(TARGET, "password", "ver") + self.assertTrue(any(text.startswith("time ") for text in self.serial.commands)) + self.link._clock_checked = False + self.serial.clock = datetime(2099, 1, 1, tzinfo=timezone.utc) + count = len(self.serial.commands) + with self.assertRaisesRegex(RepeaterError, "ahead"): + await self.link.remote_command(TARGET, "password", "ver") + self.assertFalse(any(text.startswith("ota.peer login") for text in self.serial.commands[count:])) + + async def test_control_characters_and_oversize_are_rejected_before_writing(self): + count = len(self.serial.commands) + for text in ("ver\nreboot", "x" * 155): + with self.assertRaises(RepeaterError): + await self.link.remote_command(TARGET, "password", text) + with self.assertRaises(RepeaterError): + await self.link.remote_command(TARGET, "secret\rreboot", "ver") + self.assertEqual(len(self.serial.commands), count) + + async def test_cancellation_detaches_folder_and_close_releases_serial(self): + serving = asyncio.create_task(self.link.serve(package(), asyncio.Event())) + for _ in range(100): + if len(self.serial.binary) >= 3: + break + await asyncio.sleep(.01) + serving.cancel() + with self.assertRaises(asyncio.CancelledError): + await serving + self.assertTrue(self.serial.folder_off) + await self.link.close() + self.assertFalse(self.serial.is_open) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/lora_ota/test_ota_wizard_session.py b/tools/lora_ota/test_ota_wizard_session.py new file mode 100644 index 000000000..249e5fce2 --- /dev/null +++ b/tools/lora_ota/test_ota_wizard_session.py @@ -0,0 +1,322 @@ +"""Offline direct-update transactions with stateful radios and real containers.""" +from __future__ import annotations + +import asyncio +from dataclasses import replace +import hashlib +import json +from pathlib import Path +import tempfile +import unittest + +import lora_ota as ota +import ota_wizard_session as wizard +from test_lora_ota import firmware +from test_lora_ota_bootloader import boot_blob + + +KEY = "12" * 32 +SOURCE = "34" * 32 +TARGET = 0xE5331F02 +HW = "ikoka_stick_nrf_33dbm" +OLD = 0x01110109 +NEW = 0x01120001 +BOOT_OLD = 0x02040BFF +BOOT_NEW = 0x02040CFF +TEST_SIGNER = "8A88E3DD7409F195FD52DB2D3CBA5D72CA6709BF1D94121BF3748801B40F6F5C" + + +def application(version=NEW, body=b"new application" * 600): + return ota.parse_endf(firmware(body, version, TARGET, HW)) + + +def boot_status(staged="none mid=- hash=-"): + return ("BL board=28860044 target=28860044 name=XIAO_DFU " + "crc=6F9BE280 abi=3 caps=0E | staged:" + staged) + + +class FakeTransport: + """Apply mutations before optionally losing their replies, as real USB can.""" + source_key = SOURCE + + def __init__(self): + self.image = application(OLD, b"old application" * 600) + self.boot_version = BOOT_OLD + self.calls = [] + self.windows = {"source": False, "target": False} + self.rxps = {"source": "off,level=8,preamble=16,rx=10000,sleep=10000", + "target": "on,level=8,preamble=16,rx=10000,sleep=10000"} + self.policies = {"source": dict(hops="3", autofetch="off", autoinstall="off"), + "target": dict(hops="2", autofetch="on", autoinstall="on")} + self.seeder = True + self.unsupported_source = False + self.serving = None + self.serves = [] + self.mid = None + self.lost_pull = self.lost_install = False + self.bad_mid = self.bad_hash = self.bad_body = False + self.truncate_status = False + self.hold_download = False + self.download_started = asyncio.Event() + self.ignore_restore = False + self.restore_path_called = False + self.reset_calls = [] + self.login_stale = False + + async def local_command(self, text): + return await self.command("source", text) + + async def remote_command(self, key, password, text): + if key != KEY or password != "secret-for-test": + raise AssertionError("incorrect target or credentials") + if self.login_stale: + raise AssertionError("post-reboot probe needs a fresh authenticated session") + return await self.command("target", text) + + async def reset_remote_session(self, key): + self.reset_calls.append(key) + self.login_stale = False + + async def serve(self, blob, stop): + self.serving = ota.parse_mota(blob) + self.serves.append(self.serving) + await stop.wait() + + async def restore_contact_path(self): + self.restore_path_called = True + + async def command(self, label, text): + self.calls.append((label, text)) + if text == "get public.key": + return KEY if label == "target" else SOURCE + if text == "ver": + return ota.format_version(self.image.fw_version) + "-dev-test" + if text == "get bootloader.ver": + return f"OTAFIX{self.boot_version >> 24}.{self.boot_version >> 16 & 255}.{self.boot_version >> 8 & 255}" + if text == "ota status": + result = f"OTA | this fw 0000 hw={HW} | target:{TARGET:08X} | maxblk:2048 | bl:QSPI" + if self.mid: + mid = "DEADBEEF" if self.bad_mid else self.mid + state = "downloading" if self.hold_download else "ready to install" + result += f" | download: {state} 1/1 (100%)" + ("" if self.truncate_status else f" id={mid}") + else: + result += " | no download" + return result + if text == "ota stats": + mid = "DEADBEEF" if self.bad_mid else self.mid + state = "dl" if self.hold_download else "done" + return f"OTA | fw v1.17.1.9 id=DEADCAFE body=0000 | fetch {state} 1/1 100% id={mid} 0s" + if text == "ota self": + return f"self body=9000 image=9056 base_hash={self.image.body_hash.hex()} | bootloader: QSPI apply OK (abi=3 codecs=0x5)" + if text == "ota bootloader status": + if self.mid: + image_hash = "00" * 8 if self.bad_hash else self.serving.image_hash[:8].hex() + return boot_status(f"ready mid={self.mid} hash={image_hash}") + return boot_status() + if text == "ota config": + mode = "mode=seeder-only " if label == "source" and self.seeder else "" + return "ota config: " + mode + " ".join(f"{key}={value}" for key, value in self.policies[label].items()) + if text.startswith("ota config "): + _, _, field, value = text.split() + if label == "source" and self.seeder and field != "hops": + return "ERR seeder-only policy must stay off" + if not self.ignore_restore or value == "0": + self.policies[label][field] = value + return "OK" + if text == "get radio": + return "910.525024,62.5,7,5" + if text in ("get radio.rxps.config", "get radio.rxps"): + if label == "source" and self.unsupported_source: + return "ERR RX power saving is unsupported on this radio" + return self.rxps[label] + if text == "set radio.rxps off": + self.rxps[label] = self.rxps[label].replace("on,", "off,") + return "OK off" + if text == "set radio.rxps level 8 preamble 16": + self.rxps[label] = "on,level=8,preamble=16,rx=10000,sleep=10000" + return "OK on" + if text == "get tempradio": + if label == "source": + return "TempRadio active: 910.525024,62.5,7,5 14000s left,preamble=16" if self.windows[label] else "TempRadio inactive" + return "910.525024,62.5,7,5,240" if self.windows[label] else "off" + if text.startswith("tempradio "): + self.windows[label] = True + return "OK applying in 2s" + if text == "normalradio": + self.windows[label] = False + return "OK" + if text.startswith("ota key add "): + return "OK trusted key added" + if text == "ota ls": + return f"OTA | {self.serving.manifest_id}" + if text.startswith("ota pull "): + if not all(self.windows.values()): + raise AssertionError("pull without overlapping leases") + self.mid = text.split()[2] + self.download_started.set() + if self.lost_pull: + raise TimeoutError("lost pull acknowledgment") + return "OK pulling" + if text.startswith("ota bootloader install ") or text == "ota install": + if self.serving.is_bootloader: + self.boot_version = self.serving.fw_version + else: + self.image = ota.parse_endf(self.serving.payload) + if self.bad_body: + self.image = application(self.image.fw_version, b"different body" * 600) + self.windows["target"] = False # reboot clears the RAM lease + self.login_stale = True + self.mid = None + if self.lost_install: + raise TimeoutError("lost install acknowledgment") + return "OK rebooting" + raise AssertionError((label, text)) + + +class SessionTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.link = FakeTransport() + self.path = Path(self.temporary.name) / "session.json" + self.session = wizard.DirectSession(self.link, KEY, "secret-for-test", self.path, + progress=lambda _: None, poll_seconds=.001, + reboot_timeout=.03, transfer_timeout=1) + + async def test_inspect_proves_board_storage_and_boot_contract(self): + destination = await self.session.inspect() + self.assertEqual((destination.public_key, destination.target_id, destination.hw_id), (KEY, TARGET, HW)) + self.assertEqual((destination.platform, destination.external_storage, destination.codecs), ("nrf52", True, 5)) + self.assertEqual((destination.bootloader.storage_flags, destination.boot_version), (14, BOOT_OLD)) + + async def test_rejects_wrong_target_identity_before_mutations(self): + original = self.link.remote_command + async def wrong(key, password, text): + return "56" * 32 if text == "get public.key" else await original(key, password, text) + self.link.remote_command = wrong + with self.assertRaisesRegex(ota.OtaError, "public key"): + await self.session.inspect() + self.assertEqual(self.link.calls, []) + + async def test_lost_pull_and_install_replies_are_reconciled_once(self): + self.link.lost_pull = self.link.lost_install = self.link.truncate_status = True + before = await self.session.inspect() + package = wizard.full_package(application(), 1024) + after = await self.session.install(package, before) + self.assertEqual((after.version, after.base_hash), (NEW, application().body_hash)) + commands = [text for label, text in self.link.calls if label == "target"] + self.assertEqual(sum(text.startswith("ota pull ") for text in commands), 1) + self.assertEqual(commands.count("ota install"), 1) + self.assertIn("ota stats", commands) + self.assertEqual(await self.session.cleanup(), []) + self.assertEqual(self.link.rxps["source"].split(",")[0], "off") + self.assertEqual(self.link.rxps["target"].split(",")[0], "on") + self.assertEqual(self.link.policies["target"], dict(hops="2", autofetch="on", autoinstall="on")) + self.assertFalse(any(self.link.windows.values())) + self.assertTrue(self.link.restore_path_called) + journal = json.loads(self.path.read_text()) + self.assertEqual((journal["target_key"], journal["source_key"], journal["phase"]), (KEY, SOURCE, "finished")) + self.assertNotIn("secret-for-test", self.path.read_text()) + + async def test_bootloader_then_application_reopens_target_window(self): + before = await self.session.inspect() + boot = wizard.Package(boot_blob(storage=14, version=BOOT_NEW), signer=TEST_SIGNER) + after_boot = await self.session.install(boot, before) + self.assertEqual(after_boot.base_hash, before.base_hash) + self.assertEqual(after_boot.boot_version, BOOT_NEW) + after = await self.session.install(wizard.full_package(application(), 1024), after_boot) + self.assertEqual(after.version, NEW) + target = [text for label, text in self.link.calls if label == "target"] + self.assertEqual(sum(text.startswith("tempradio ") for text in target), 2) + self.assertEqual(len(self.link.serves), 2) + self.assertEqual(self.link.reset_calls, [KEY, KEY]) + self.assertEqual(await self.session.cleanup(), []) + + async def test_wrong_mid_fails_without_install(self): + before = await self.session.inspect() + self.link.bad_mid = True + try: + with self.assertRaisesRegex(ota.OtaError, "not the selected package"): + await self.session.install(wizard.full_package(application(), 1024), before) + finally: + self.assertEqual(await self.session.cleanup(), []) + self.assertNotIn(("target", "ota install"), self.link.calls) + + async def test_staged_bootloader_hash_must_match_exactly(self): + before = await self.session.inspect() + self.link.bad_hash = True + try: + with self.assertRaisesRegex(ota.OtaError, "MID/hash"): + await self.session.install(wizard.Package(boot_blob(storage=14, version=BOOT_NEW), signer=TEST_SIGNER), before) + finally: + await self.session.cleanup() + self.assertFalse(any(text.startswith("ota bootloader install ") for _, text in self.link.calls)) + + async def test_running_version_alone_is_not_install_success(self): + before = await self.session.inspect() + self.link.bad_body = True + try: + with self.assertRaisesRegex(ota.OtaError, "version and body hash"): + await self.session.install(wizard.full_package(application(), 1024), before) + finally: + await self.session.cleanup() + + async def test_destination_temporarily_missing_after_reboot_is_probed_again(self): + from ota_wizard_companion import CompanionError + before = await self.session.inspect() + original = self.link.remote_command + missing = 2 + async def reboot_discovery(key, password, text): + nonlocal missing + if self.link.image.fw_version == NEW and text == "get public.key" and missing: + missing -= 1 + raise CompanionError("No directly reachable destination replied yet") + return await original(key, password, text) + self.link.remote_command = reboot_discovery + after = await self.session.install(wizard.full_package(application(), 1024), before) + self.assertEqual(after.version, NEW) + self.assertEqual(missing, 0) + self.assertEqual(len(self.link.reset_calls), 3) + self.assertEqual([text for _, text in self.link.calls].count("ota install"), 1) + self.assertEqual(await self.session.cleanup(), []) + + async def test_cancelled_download_can_restore_every_setting(self): + before = await self.session.inspect() + self.link.hold_download = True + task = asyncio.create_task(self.session.install(wizard.full_package(application(), 1024), before)) + await asyncio.wait_for(self.link.download_started.wait(), 2) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + self.assertEqual(await self.session.cleanup(), []) + self.assertFalse(self.session.active) + self.assertFalse(any(self.link.windows.values())) + self.assertNotIn(("target", "ota install"), self.link.calls) + + async def test_cleanup_requires_policy_readback_and_can_be_retried(self): + await self.session.begin() + self.link.ignore_restore = True + self.assertTrue(await self.session.cleanup()) + self.assertTrue(self.session.active) + self.assertEqual(json.loads(self.path.read_text())["phase"], "needs cleanup") + self.link.ignore_restore = False + self.assertEqual(await self.session.cleanup(), []) + self.assertFalse(self.session.active) + + async def test_unsupported_source_rxps_is_explicitly_skipped(self): + self.link.unsupported_source = True + await self.session.begin() + self.assertEqual(self.session.saved["source"]["rxps"], "unsupported") + self.assertNotIn(("source", "set radio.rxps off"), self.link.calls) + self.assertEqual(await self.session.cleanup(), []) + + async def test_existing_window_prevents_any_mutation(self): + self.link.windows["source"] = True + with self.assertRaisesRegex(ota.OtaError, "already has a temporary"): + await self.session.begin() + self.assertFalse(self.session.active) + self.assertFalse(any(text.startswith("set ") or text.startswith("ota config ") for _, text in self.link.calls)) + + +if __name__ == "__main__": + unittest.main()