From dc4a8808c0a6d69e9df830c4b7dd383b266f6589 Mon Sep 17 00:00:00 2001 From: mikecarper Date: Wed, 19 Aug 2026 11:17:22 -0700 Subject: [PATCH] Harden and qualify compact RAK3401 OTA updates --- build.sh | 8 +- docs/companion_radio_full.md | 11 + docs/lora_ota_automation.md | 14 +- docs/ota_protocol.md | 18 +- docs/rak3401_mota_chain.md | 413 ++++++----- platformio.ini | 1 + src/Mesh.cpp | 7 +- src/Mesh.h | 4 +- src/helpers/ota/MotaSeederProto.h | 3 + src/helpers/ota/MotaSourceSerial.cpp | 20 +- src/helpers/ota/OtaManager.cpp | 129 +++- src/helpers/ota/OtaManager.h | 24 +- test/test_ota/test_ota_core.cpp | 131 ++++ test/test_trace_retry/test_trace_retry.cpp | 13 +- .../build_rak3401_accelerated_bridges.py | 13 +- .../lora_ota/build_rak3401_compact_bundle.py | 678 ++++++++++++++++++ tools/lora_ota/lora_ota.py | 203 +++++- tools/lora_ota/rak3401_compact_route.json | 125 ++++ .../rak3401_dynamic_workspace_backport.patch | 34 + tools/lora_ota/rak3401_mota_chain.py | 355 +++++++-- tools/lora_ota/test_lora_ota.py | 223 +++++- tools/mota/pio_endf.py | 6 +- 22 files changed, 2156 insertions(+), 277 deletions(-) create mode 100755 tools/lora_ota/build_rak3401_compact_bundle.py create mode 100644 tools/lora_ota/rak3401_compact_route.json create mode 100644 tools/lora_ota/rak3401_dynamic_workspace_backport.patch diff --git a/build.sh b/build.sh index 821b35a52..c1586ed6f 100755 --- a/build.sh +++ b/build.sh @@ -2388,10 +2388,10 @@ apply_companion_radio_full_profile() { return 0 fi - # ESP32 inherits the WiFi companion recipe and uses its dedicated TCP folder - # seeder instead of multiplexing mOTA data onto USB. - append_platformio_build_unflags "-DOTA_FOLDER_SERIAL" - export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -UOTA_FOLDER_SERIAL -DWIFI_OTA_SEEDER=1" + # ESP32 keeps both source transports. TCP remains the normal unattended + # path; a host may explicitly hold the native USB port in Companion terminal + # mode when WiFi is unavailable and serve the same folder protocol there. + export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DWIFI_OTA_SEEDER=1" # BLE + WiFi exhaust internal DRAM on these high-capacity ESP32 recipes. Use # measured-safe tables for FULL OTA without changing ordinary USB/BLE/WiFi diff --git a/docs/companion_radio_full.md b/docs/companion_radio_full.md index 06ccb51a1..c9927685b 100644 --- a/docs/companion_radio_full.md +++ b/docs/companion_radio_full.md @@ -289,6 +289,17 @@ Return to Binary mode with: Closing the USB data connection also resets the port to Binary mode. A different baud rate, including 57600, does not select ASCII mode. +On an ESP32 Full Companion built with `OTA_FOLDER_SERIAL`, `motatool` can keep +that terminal session open as an mOTA folder source when WiFi is unavailable: + +```sh +motatool serve --serial /dev/ttyACM0 --companion-terminal --dir ./motas -v +``` + +The explicit flag is required because an ESP32 Companion starts the same USB +port in binary Companion mode. TCP port 5001 remains the preferred unattended +source transport. + ## nRF52 USB mOTA mode The nRF52 full target has a third, exclusive USB mode for the host folder. diff --git a/docs/lora_ota_automation.md b/docs/lora_ota_automation.md index dbb472808..d54c718c2 100644 --- a/docs/lora_ota_automation.md +++ b/docs/lora_ota_automation.md @@ -1,9 +1,9 @@ # Scripted LoRa OTA from start to finish The dedicated [RAK3401 chain report](rak3401_mota_chain.md) records the -physical failure of the withdrawn 26-step migration and the corrected -v1.17.01 test candidate. Its runner blocks the withdrawn chain and requires an -explicit lab-only gate for the corrected chain until physical testing passes. +physical failures of withdrawn migrations and the physically qualified compact +nine-step replacement. Its runner blocks the withdrawn chains and pins every +accepted bridge image by SHA-256. [`tools/lora_ota/lora_ota.sh`](https://github.com/mikecarper/MeshCore/blob/keymindCascade/tools/lora_ota/lora_ota.sh) and [`tools/lora_ota/lora_ota.ps1`](https://github.com/mikecarper/MeshCore/blob/keymindCascade/tools/lora_ota/lora_ota.ps1) automate a @@ -51,7 +51,9 @@ The USB ASCII switch (`+++MESHCORE-TERM-START`) is the local control path, not the mOTA data framing. On an nRF52 full Companion, the script uses that mode briefly for `ota status` and TempRadio commands. It then closes the CLI and starts `motatool`, whose existing `ota folder on` preamble switches the same -USB port into exclusive mOTA mode. BLE remains available during that mode. +USB port into exclusive mOTA mode. On an ESP32 Full Companion with serial +folder support, `motatool --companion-terminal` keeps the ASCII session open +while the same link carries framed folder requests. BLE remains available. ## Destination requirements @@ -383,6 +385,10 @@ Useful controls: before every `ota install` transmission and refuses installation unless the destination reports `> off`. Use it for nRF52 chains whose bootloader cannot service an application watchdog inherited across reset. +- Managed `--relay` nodes have their original `rxdelay` and `txdelay` captured + in the work directory. During transfer the runner verifies `rxdelay 0` and + an airtime-scaled `txdelay 0.3` (override with `--relay-txdelay`), then + restores both values before the relay leaves TempRadio. - `--work-dir PATH` chooses a new, non-existent work directory. - `--meshcli PATH` and `--motatool PATH` select binaries not on `PATH`. diff --git a/docs/ota_protocol.md b/docs/ota_protocol.md index f97f6b035..e83a79984 100644 --- a/docs/ota_protocol.md +++ b/docs/ota_protocol.md @@ -552,7 +552,8 @@ This keeps each active transfer as one transmitter and one receiver while still repeaters between them and persistent archive nodes to improve future availability. TempRadio is treated as a private maintenance network. Active transfer packets use priority 0, bypass the -public-flood receive holdoff and relay jitter, and do not schedule generic flood retries. The bounded serving +public-flood receive holdoff, retain the relay role's airtime-scaled transmit collision window, and do not +schedule generic flood retries. The bounded serving queue admits at most two DATA/PROOF packets ahead of the radio while preserving at least four free packet-pool entries. CAD remains enabled to arbitrate the half-duplex channel, but its busy retry is scaled to one-quarter of a packet airtime and clamped to 5-50 ms instead of the ordinary 120-360 ms cadence. Discovery traffic keeps @@ -662,6 +663,21 @@ MotaDesc wire (38 B): mid[4] target_id(4) fw_version(4) codec(1) flags(1) status: 0 = OK, non-zero = error (out of range / past EOF). ``` +`SerialMotaSource` splits logical reads into replies of at most 192 payload +bytes. A manifest's leaf table can exceed 256 bytes and payload blocks are +normally 1 KiB; requesting either in one transaction can overrun common USB +CDC/UART receive rings even though the host successfully wrote the complete +reply. Chunking is internal to the transport and does not change `OP_READ` or +the `MotaSource` random-access contract. + +Manifest fragments are retained as bounded response jobs and admitted one at +a time. Their source-side gap follows the active maximum packet airtime and +dispatcher duty spacing, clamped to 100-1000 ms. The 100 ms floor protects +fast radios' TX-to-RX turnaround; the cap keeps the receiver's one-second +manifest progress/retry observation responsive. Relay collision delay is a +separate setting: active OTA floods honor the relay role's configured +`txdelay`, and the deployment runner temporarily uses `0.3` on managed relays. + **What to plug into `--serial`.** Use the USB serial console of an OTA-enabled MeshCore node built with `OTA_FOLDER_SERIAL`. The node must have a working LoRa radio plus an `ota folder on` command; that command confirms it can host and advertise the folder. A **KISS modem will not diff --git a/docs/rak3401_mota_chain.md b/docs/rak3401_mota_chain.md index 23eb63801..bb49f3c55 100644 --- a/docs/rak3401_mota_chain.md +++ b/docs/rak3401_mota_chain.md @@ -1,28 +1,24 @@ -# RAK3401 1W repeater LoRa update chain +# RAK3401 1W repeater compact LoRa update chain -> Status: accelerated release candidate. All 30 exact transitions passed -> independent offline reconstruction, container verification, and stable -> bootloader simulation. The byte-identical first bridge and the superseded -> 29-step baseline completed physical RAK3401 testing; the new complete 30-step -> byte sequence still needs its final full-chain hardware qualification. +> Status: physically qualified on 19-Aug-2026. All nine exact transitions +> completed on a deployed RAK3401 1W with its existing bootloader, using a +> Heltec V4 source at 909.950 MHz / 500 kHz / SF5 / CR5. Every post-boot EndF +> hash matched. The same files also pass independent reconstruction, container +> verification, and both deployed Preview 6 and current OTAFIX simulators. -The release is -[`rak3401-mota-v1.16.07-c1caa5ad-to-v1.17.01-c96bdd6e`](https://github.com/mikecarper/MeshCore/releases/tag/rak3401-mota-v1.16.07-c1caa5ad-to-v1.17.01-c96bdd6e). Use this asset: ```text -RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.01-8c5262c6.zip +RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a.zip ``` -- ZIP SHA-256: `b2781e02460b200a7c37bfae352bad81618716e550d1d042dca8aa29bfc73c29` -- inner `SHA256SUMS.txt` SHA-256: `1f7add658ae5771451cd66a0e5c58a5e461983c1b36029ef480093ae5d5f1020` -- endpoint image SHA-256: `a08b5791419410c760f31c26bb45c77d776eb7bbf68fde656e19bcd616a6227d` -- endpoint EndF body hash: `3DD88C3DD7B8A35B` +- ZIP SHA-256: `9f80eef191b88833bf4d2e4fea559cf5233ca53f9266ba310d447f37fa445f3a` +- inner `SHA256SUMS.txt` SHA-256: `73d96e23237896a3e342fe736be12d94087a813bf09ad609fb55330bbe586055` +- endpoint image SHA-256: `2784e4b645bc3dc198de0b8b18d3d7369cd02eca61cd71c46a51b61854da5345` +- endpoint EndF body hash: `4BB1526BF647547D` -The superseded 29-step v1.17.01 release, earlier v1.17.01 candidates, and the -v1.17.02 chain remain useful only for provenance or diagnosis. The runner -recognizes their hashes and refuses live installation before opening a device -or reading a password. +Until that asset is uploaded to the release tag pinned by the runner, pass its +local path with `--bundle`. ## Exact destination @@ -33,111 +29,88 @@ target: - hardware: `RAK_3401` - role: `RAK_3401_repeater_lora_ota_no_external_sensors` - start: `v1.16.7.0-c1caa5ad`, EndF `71F4026CBE4B8B74` -- endpoint: `v1.17.01-halo-keymind-cascade-dev-8c5262c6`, EndF `3DD88C3DD7B8A35B` +- endpoint source: MeshCore `e742333a0ee393b0d55d2414f25b28f2f67e1ea1` +- endpoint label: `v1.17.1.02-halo-keymind-cascade-dev-e742333a` +- endpoint packed version: `0x01110102` (EndF tools render this as `1.17.1.2`) - deployment target key: `63d8df6387eaffd2e25db7d2a8ad967a65202182a48d681d7e7a9260f917280d` Do not use the chain on another target ID, hardware family, starting image, or firmware body hash. The runner checks all four. -## Validation and speed design +## Why the old bootloader works -The direct physical baseline used: +The deployed bootloader is not changed and no package uses the newer expanded +`0xED000` ceiling. Each package remains bottom-aligned below the old +bootloader's `0xD4000` scan ceiling. Its detools workspace is page-aligned and +must satisfy: -- a Heltec V4.3 OLED 2 MB Full Companion on `/dev/ttyACM0`; -- the same Full Companion at `192.168.1.51`, with mOTA seeding on TCP `5001` - and its local OTA/TempRadio console on TCP `5002`; -- Full Companion key - `8ce031bc322f3cf77376eeacfe8800b30d8c5dfe927f3636d801f3455a2dd4cf`; -- one direct RF link to `RAK3401 1W Repeater`; -- temporary radio `909.950 MHz`, `500 kHz`, `SF5`, `CR5`; -- all 29 packages in the now-superseded `cd824765` chain. +```text +0x26000 + inplace_memory <= staged_package_start < 0xD4000 +``` -The baseline started at `2026-08-14T01:50:24Z` and reached its endpoint at -`2026-08-14T06:41:36Z`. This ZIP preserves that 29-row record as -`PHYSICAL-BASELINE-cd824765.jsonl`; it does not claim those rows describe the -new bridge bytes. +The old bootloader already reads and validates that encoded detools geometry +before its first destructive write. The previous application receiver was the +limiting part: it reserved a fixed `0x98000` workspace before accepting a +download. Step 1 therefore remains byte-for-byte identical to the physically +passed 89,844-byte package. Step 2 is a 46,363-byte fixed-workspace package +that installs the compatibility receiver. Later receivers stage above the +real EndF image extent and let the bootloader enforce each package's selected +workspace. -All 30 new containers were freshly generated. Every delta was reconstructed -with both zero-filled and `0xFF`-filled workspaces, verified independently by -`motatool`, and exercised by the stable OTAFIX apply simulator. The largest -package is the byte-identical, physically passed step 1 at 89,844 bytes, which -leaves 268 bytes of the fixed 90,112-byte staging region. Step 2 is the earliest -safe place for the terminal-consumption backport: adding it directly to step 1 -would exceed that staging limit. Step 26 remains an intentional compiler-layout -bridge. +An exhaustive search tested every page-aligned workspace relevant to a route +shorter than nine packages: 9,430 candidate geometries across 272 possible +layer-skipping edges. None fit. The 61 edges on the shortest-path graph were +then swept across another 2,296 workspace choices; 840 nine-package routes +were compared by total transfer size. The selected route is the byte-minimum +shortest route: -The speedup keeps 1 KiB logical blocks. Firmware currently accepts at most 1 -KiB, and larger logical blocks would still fragment into the same roughly -160-byte LoRa packets while increasing the amount at risk when a block must be -retried. Instead, one request can open an adaptive flight of multiple 1 KiB -blocks. It begins at one block, grows through two and three to four after clean -flights, and reduces the width after missing data or proofs. Retry timing is -derived from actual LoRa airtime and path length. Active transfer packets use -primary priority; periodic discovery remains background traffic. +| Step | From | To | Workspace | Package | Margin | +|---:|---|---|---:|---:|---:| +| 1 | 1.16.7.0 | 1.16.7.9 | `0x98000` | 89,844 | 0 | +| 2 | 1.16.7.9 | 1.16.7.10 | `0x98000` | 46,363 | 40,960 | +| 3 | 1.16.7.10 | 1.16.8.7 | `0x91000` | 106,030 | 12,288 | +| 4 | 1.16.8.7 | 1.16.9.105 | `0x89000` | 149,927 | 0 | +| 5 | 1.16.9.105 | 1.16.9.110 | `0x8E000` | 111,307 | 16,384 | +| 6 | 1.16.9.110 | 1.16.9.111 | `0x8D000` | 70,679 | 61,440 | +| 7 | 1.16.9.111 | 1.16.9.116 | `0x89000` | 143,441 | 4,096 | +| 8 | 1.16.9.116 | 1.16.9.118 | `0x81000` | 98,188 | 86,016 | +| 9 | 1.16.9.118 | 1.17.1.02 | `0x7D000` | 186,385 | 12,288 | -Step 1 installs the guarded rescue command, the four-block adaptive requester, -and primary transfer priority. The starting firmware still requests step 1 -serially. Step 2 installs terminal OTA bulk-packet consumption in the target, -so the echo/dispatch reduction takes effect for transfers 3 through 30. A -direct endpoint comparison confirmed the intended behavior: the older target -needed 147 host block reads for a 48-block package and reported ready after -267 seconds; the current target needed 82 reads for a larger 68-block package -and reported ready after 98 seconds. These differently sized transfers are a -protocol comparison, not a formal throughput benchmark. +Total mOTA transfer data is 1,002,164 bytes. `ROUTE.json`, `CHAIN.csv`, and +`validation-results.json` in the bundle pin the exact geometry and image hash +for every transition. ## Host requirements -Install: +Install Python 3.9 or newer, `meshcli` 1.6.0 or newer, the current `motatool`, +and an OTA-enabled Full Companion or repeater that can seed mOTA files. -- Python 3.9 or newer; -- `meshcli` 1.6.0 or newer; -- the current `motatool`; -- a Full Companion or OTA-enabled repeater that can seed mOTA files. - -The runner maintains one persistent controller connection. It logs into the -destination once, keeps that authenticated radio-node session across -application reboots, and does not treat a silent packet as lost -authentication. The source seeder reconnects for each isolated package but -does not need remote-admin login. - -## Verify the release offline +## Verify offline No password or device is needed: ```bash python3 tools/lora_ota/rak3401_mota_chain.py \ - --bundle /path/to/RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.01-8c5262c6.zip \ + --bundle /path/to/RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a.zip \ --work-dir ./rak3401-mota-chain-work \ --motatool /path/to/motatool \ --verify-only ``` -When `--bundle` is omitted, the runner downloads the pinned release asset. It -checks the outer ZIP hash, complete inner checksum coverage, all 30 manifests, -chain continuity, audited image anchors, final recovery image, and every mOTA -container with `motatool`. +The runner checks the outer ZIP, complete inner checksum coverage, all nine +manifests, continuity, image anchors, final recovery image, and every container +with `motatool`. -Offline verification does not need `--accept-test-candidate`. Live preflight or -installation does: the flag explicitly acknowledges that the exact 30-step -sequence has exhaustive offline validation but not yet a complete physical run. +## Direct recoverable bench run -## Direct recoverable bench run: BW500/SF5 - -Restore the test RAK locally with the ZIP's -`recovery/test-start/RAK3401-test-start-v1.16.7-c1caa5ad.uf2`. Before starting, -require start version `1.16.7.0`, body hash `71F4026CBE4B8B74`, target -`2FA509C1`, hardware `RAK_3401`, and OTAFIX mOTA ABI 2 with codec 2. The live -runner queries `get bootloader.ver`, reports the installed version, and then -uses `ota self` to verify those apply capabilities before changing any radio -or watchdog setting. - -For the tested direct topology: +Restore the test start locally with +`recovery/test-start/RAK3401-test-start-v1.16.7-c1caa5ad.uf2`, then run: ```bash export MESHCORE_ADMIN_PASSWORD='password' python3 tools/lora_ota/rak3401_mota_chain.py \ - --bundle /path/to/RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.01-8c5262c6.zip \ + --bundle /path/to/RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a.zip \ --work-dir ./rak3401-mota-chain-work \ --controller-serial /dev/ttyACM0 \ --source-tcp 192.168.1.51:5001 \ @@ -147,33 +120,179 @@ python3 tools/lora_ota/rak3401_mota_chain.py \ --temp-radio 909.950,500,5,5,120 \ --ota-hops 1 \ --motatool /path/to/motatool \ - --accept-test-candidate \ --yes ``` -BW500 is for the controlled direct bench setup. Confirm that the selected -frequency and bandwidth are legal at the test location. +Keep the work directory. Rerunning the same command resumes only when the live +EndF body hash matches an exact chain node. Never manually skip a package. -Keep the work directory. Each package gets an isolated attempt directory, and -each exact transition is appended to `progress.jsonl`. After a host or RF -interruption, rerun the identical command with the same work directory. The -runner resumes only when the live body hash matches the recorded chain; never -manually skip to a package. +## Manual operation -## Live deployment with two intermediate relays: BW250/SF5 +The pinned runner is the safer method. If it cannot be used, follow every +check below and keep a written copy of each original setting. Do not skip a +step even when a later package appears in `ota ls`. -For the intended private TempRadio deployment, use BW250/SF5 and two -intermediate relays. List relays from the destination side toward the -controller: farthest first, nearest last. With two relays there are three RF -links, so use `--ota-hops 3`. - -Run a non-mutating preflight first: +### 1. Verify and extract the asset ```bash -export MESHCORE_ADMIN_PASSWORD='your-destination-and-relay-password' +sha256sum RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a.zip +unzip RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a.zip +cd RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a +sha256sum -c SHA256SUMS.txt +for package in motas/*.mota; do motatool verify "$package" || exit 1; done +``` +The outer hash must be `9f80eef191b88833bf4d2e4fea559cf5233ca53f9266ba310d447f37fa445f3a`. +Do not continue after any checksum or verification error. + +### 2. Record and prepare the destination + +Before changing anything, save the complete replies from: + +```text +ver +ota self +get bootloader.ver +get system.watchdog +get radio.rxps +powersaving +get rxdelay +ota config +``` + +The starting `ota self` hash must be `71F4026CBE4B8B74`, the target must be +`2FA509C1`, and the hardware must be `RAK_3401`. Disable the nRF52 system +watchdog before the chain: + +```text +set system.watchdog off +``` + +The hardware watchdog cannot stop immediately. The RAK will reset once within +about 60 seconds. Reconnect, wait at least 90 seconds from the command, and +require `get system.watchdog` to report `> off` before continuing. + +For a direct bench link, apply these temporary transfer guardrails to the RAK +after every bridge reboot: + +```text +powersaving off +set radio.rxps off +set rxdelay 0 +ota config hops 1 +tempradio 909.950,500,5,5,120 +``` + +`powersaving off` is an isolation guardrail, not the fix for the observed +failure. The actual source failure was an oversized USB CDC reply; current +firmware splits those reads. Some historical bridge builds may report RXPS as +unsupported while already operating continuously. Verify the reported state +and do not substitute a normal reboot for the one-time watchdog reset. + +Put the source on the identical TempRadio tuple. If it is a binary-mode Full +Companion, the current `motatool` can switch modes for the serving session: + +```bash +motatool serve \ + --dir ./motas \ + --serial /dev/ttyACM1 \ + --baud 115200 \ + --companion-terminal \ + -v +``` + +Omit `--companion-terminal` for an ordinary text-console OTA source. Restart +`motatool` for every step so the source emits a fresh catalog advert. Leave it +running during the download and stop it with Ctrl-C only after the destination +reports `ready to install`. + +### 3. Install all nine packages in order + +| Step | Manifest ID | Before hash | Expected version | After hash | +|---:|---|---|---|---| +| 1 | `C147BCEF` | `71F4026CBE4B8B74` | `1.16.7.9` | `42BC53A64288E845` | +| 2 | `C510B628` | `42BC53A64288E845` | `1.16.7.10` | `6F6E51D63BF3E93F` | +| 3 | `8CAD7045` | `6F6E51D63BF3E93F` | `1.16.8.7` | `F04DA8DB515E5C6D` | +| 4 | `8B1EEFF6` | `F04DA8DB515E5C6D` | `1.16.9.105` | `3894A4D7B11ED282` | +| 5 | `687E3BEA` | `3894A4D7B11ED282` | `1.16.9.110` | `CDC5CA630881DA15` | +| 6 | `0D59A34B` | `CDC5CA630881DA15` | `1.16.9.111` | `D0CC4A9E135B9A4D` | +| 7 | `2C5C2082` | `D0CC4A9E135B9A4D` | `1.16.9.116` | `66272A0E7FEF2773` | +| 8 | `994AB743` | `66272A0E7FEF2773` | `1.16.9.118` | `8E00019AA08E00BF` | +| 9 | `FE986948` | `8E00019AA08E00BF` | `1.17.1.02` | `4BB1526BF647547D` | + +For each row, first prove that `ota self` exactly matches its **Before hash**. +Then run: + +```text +ota cancel +ota ls +ota pull MANIFEST_ID flash +ota status +``` + +Repeat `ota status` at a restrained interval until it says `ready to install`. +If the manifest is initially absent, wait for a fresh source advert, run +`ota ls` again, or restart `motatool`; do not pull a different ID. Stop the +seeder and install: + +```text +get system.watchdog +ota install +``` + +The watchdog reply must still be `> off`. Allow up to 90 seconds for the USB +port to disappear and return. Then run `ver`, `ota self`, and `ota help`. +Require the row's exact **After hash**, require `bootloader: apply OK`, and +require `rescue install ` in the help before exposing that bridge to +the next package. Clear only the now-proven retained download with `ota cancel`, +reapply the transfer guardrails and TempRadio tuple, restart `motatool`, and +continue with the next row. + +If the board boots but `ota self` says the EndF is invalid, stop. Only when +`ota status` still identifies that row's complete staged package may the +guarded `ota rescue install BEFORE_HASH` command be used. It is not a force +option and must use that row's exact Before hash. If the board does not boot, +recover the documented start/recovery UF2 locally over USB. + +### 4. Intermediate relays and restoration + +On every managed intermediate relay, save `get rxdelay` and `get txdelay`, then +use the airtime-scaled collision window during the maintenance session: + +```text +set rxdelay 0 +set txdelay 0.3 +``` + +Current OTA transfer packets honor that configured relay `txdelay`; setting +`0.3` gives competing relays a small randomized, airtime-scaled forwarding +window. The source's manifest-fragment gap independently follows active packet +airtime and duty spacing, clamped to 100-1000 ms. These two delays solve +different problems. + +After step 9 is proven, restore each saved destination and relay value exactly, +then restore normal radio operation. Re-enable the RAK system watchdog last: + +```text +normalradio +set system.watchdog on +get system.watchdog +``` + +Final success requires version `v1.17.1.02-halo-keymind-cascade-dev-e742333a`, +body hash `4BB1526BF647547D`, target `2FA509C1`, hardware `RAK_3401`, and +`get system.watchdog` reporting `> on`. If a relay cannot be restored before +its TempRadio lease ends, wait for it to return to the normal channel and +restore its saved `rxdelay` and `txdelay` there. + +## Two-relay deployment + +For two intermediate relays, list them farthest-to-nearest and use three OTA +hops. Run a non-mutating preflight first: + +```bash python3 tools/lora_ota/rak3401_mota_chain.py \ - --bundle /path/to/RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.01-8c5262c6.zip \ + --bundle /path/to/RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a.zip \ --work-dir ./rak3401-mota-chain-work \ --controller-serial /dev/ttyACM0 \ --source-tcp SOURCE_IP:5001 \ @@ -185,64 +304,38 @@ python3 tools/lora_ota/rak3401_mota_chain.py \ --temp-radio 909.950,250,5,5,120 \ --ota-hops 3 \ --motatool /path/to/motatool \ - --accept-test-candidate \ --preflight-only ``` -A bare relay name uses `MESHCORE_ADMIN_PASSWORD`. Append `=PASSWORD` only for -a relay with a different password. After preflight succeeds, rerun the same -command with `--yes` in place of `--preflight-only`. +After preflight succeeds, rerun with `--yes` in place of +`--preflight-only`. Confirm that the selected frequency and bandwidth are +legal at the deployment location. -Arm the relay chain farthest-to-nearest while all nodes are still reachable on -the ordinary mesh. The runner then moves the source and controller to the same -temporary tuple. Keep the private TempRadio network reserved for OTA during -the run; public-mesh flooding does not improve this transfer. A 120-minute -window is the minimum used here; raise it if the selected timeouts and relay -count require more time. +## Rescue and completion -## Guarded no-EndF rescue +The guarded `ota rescue install ` command is present after +step 1. It is not a force command: it refuses a valid normal EndF, a base +mismatch, wrong target/hardware, or invalid payload. A failure before step 1 +completes still requires local USB recovery because the deployed start image +predates the rescue command. -The rescue command is present from the first installed bridge onward. If a -bridge remains reachable but `ota self` reports no EndF, serve and fetch only -its exact successor package. Read that row's 16-hex `base_body_hash` from -`CHAIN.csv`, then issue: +Before the first mutation, the runner saves the destination's RXPS periods, +CPU power-saving state, and RX flood delay in the persistent work directory. +It verifies `radio.rxps off`, `powersaving off`, and `rxdelay 0` after every +bridge reboot, then restores the original values only after the exact endpoint +is proven. An interrupted run deliberately leaves those transfer guardrails +active; rerunning with the same work directory resumes and restores them. +CPU power saving was not the cause of the observed folder-source failure: that +was a USB CDC receive-ring overrun fixed by bounded serial reads. The runtime +settings remain conservative isolation and fast-link reliability guardrails. -```text -ota rescue install -``` +For every named `--relay`, the per-step runner also captures `rxdelay` and +`txdelay`, writes them to `relay-timing-settings.json`, applies `rxdelay 0` and +`txdelay 0.3` during transfer, verifies the readback, and restores both before +the relay leaves TempRadio. -This is not a force command. It refuses a valid normal EndF, a base mismatch, -wrong target or hardware, or invalid payload. OTAFIX hashes the physical -running application before its first write. A missing/corrupt EndF can -therefore continue through its exact successor instead of becoming stuck, -provided the installed bridge already contains the rescue command. The -c1caa5ad starting image predates rescue, so a failure before step 1 completes -still requires local USB recovery. - -## Safety and completion - -The runner performs the following guarded sequence: - -1. Persist `system.watchdog off`, allow the already-running watchdog to reset - once, and prove stable uptime with the watchdog off. -2. Recheck destination identity, OTA reach, source identity, and exact package - base before every step. -3. Require `system.watchdog` to report `off` immediately before every install. -4. After every install, probe `ota self` at 10 and 20 seconds rather than - sleeping for 90 seconds. Require the exact version and EndF body hash, then - prove the rescue command is present before exposing that bridge to another - step. Relayed runs keep the same 10-second probe cadence until the relays' - mandatory return-to-normal window has elapsed. -5. Re-enable the watchdog only after the exact step-30 endpoint boots. - -Success requires all of the following: - -- version `v1.17.01-halo-keymind-cascade-dev-8c5262c6`; -- EndF body hash `3DD88C3DD7B8A35B`; -- target `2FA509C1`, hardware `RAK_3401`; -- normal radio `910.525 MHz`, `62.5 kHz`, `SF7`, `CR5`; -- `system.watchdog` verified `on`. - -A completed download or an accepted install reply is not success without the -post-reboot identity checks. The bundle includes exact test-start and final -local recovery UF2/ZIP files if USB recovery is ever needed. +The runner also keeps the watchdog off across the chain, rechecks identity and +OTA reach before every package, requires the exact post-boot EndF hash, and +only re-enables the watchdog after step 9. Success requires endpoint body hash +`4BB1526BF647547D`, target `2FA509C1`, hardware `RAK_3401`, normal radio +`910.525 MHz / 62.5 kHz / SF7 / CR5`, and the watchdog verified on. diff --git a/platformio.ini b/platformio.ini index b73c0f28d..e82bd6747 100644 --- a/platformio.ini +++ b/platformio.ini @@ -274,6 +274,7 @@ build_src_filter = +<../src/helpers/ota/MerkleTree.cpp> +<../src/helpers/ota/MotaContainer.cpp> +<../src/helpers/ota/FirmwareInfo.cpp> + +<../src/helpers/ota/MotaSourceSerial.cpp> +<../src/helpers/ota/OtaProtocol.cpp> +<../src/helpers/ota/OtaManager.cpp> +<../src/helpers/ota/detools/detools.c> diff --git a/src/Mesh.cpp b/src/Mesh.cpp index dc2c28fee..c7ef9b010 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -520,7 +520,12 @@ void Mesh::decayOtaRelayBackoff() { uint32_t Mesh::getOtaRetransmitDelay(const mesh::Packet* packet) { if (packet == NULL) return 0; - if (isPrimaryOtaTraffic(packet->payload, packet->payload_len)) return 0; + // Active transfer packets use the role's configured flood collision window. + // Deployment tooling can temporarily select txdelay 0.3 on managed relays; + // the value is airtime-scaled by the role and therefore follows SF/BW. + if (isPrimaryOtaTraffic(packet->payload, packet->payload_len)) { + return getRetransmitDelay(packet); + } decayOtaRelayBackoff(); uint32_t airtime = _radio->getEstAirtimeFor(packet->getRawLength()); if (airtime == 0) return 0; diff --git a/src/Mesh.h b/src/Mesh.h index a1e2a6b7d..e74ebd9aa 100644 --- a/src/Mesh.h +++ b/src/Mesh.h @@ -227,8 +227,8 @@ protected: virtual uint32_t getRetransmitDelay(const Packet* packet); /** - * \returns TempRadio OTA relay delay. Primary transfer packets forward immediately on the private - * network; discovery retains collision jitter. + * \returns TempRadio OTA relay delay. Primary transfer packets use the role's configured flood delay; + * discovery uses its adaptive request-pressure window. */ virtual uint32_t getOtaRetransmitDelay(const Packet* packet); diff --git a/src/helpers/ota/MotaSeederProto.h b/src/helpers/ota/MotaSeederProto.h index 9525ee903..9291637d0 100644 --- a/src/helpers/ota/MotaSeederProto.h +++ b/src/helpers/ota/MotaSeederProto.h @@ -57,6 +57,9 @@ static const uint8_t MS_STATUS_OK = 0x00; static const uint8_t MS_STATUS_ERR = 0x01; static const uint16_t MOTA_DESC_WIRE = 38; // existing reserved bytes carry block geometry; wire size is unchanged +// Keep host-to-device replies below common 256-byte USB CDC/UART receive rings. Manifest leaves can exceed +// that ring, and payload reads are normally 1 KB; splitting them here prevents silent overrun/checksum loss. +static const uint16_t MOTA_SEEDER_READ_MAX = 192; static const uint16_t MOTA_SEEDER_WRITE_MAX = 512; // max data bytes per OP_WRITE/OP_SREAD (bounds frames) } // namespace ota diff --git a/src/helpers/ota/MotaSourceSerial.cpp b/src/helpers/ota/MotaSourceSerial.cpp index 98526316b..fb759de24 100644 --- a/src/helpers/ota/MotaSourceSerial.cpp +++ b/src/helpers/ota/MotaSourceSerial.cpp @@ -90,12 +90,20 @@ bool SerialMotaSource::describe(uint8_t idx, MotaDesc& out) { } bool SerialMotaSource::read(uint8_t idx, uint32_t off, uint8_t* buf, uint32_t len) { - if (len > 0xFFFF) return false; // single transaction caps at 64 KB (a block is <=1 KB) - uint8_t args[7]; - args[0] = idx; - wr_u32le(args + 1, off); - args[5] = (uint8_t)(len & 0xFF); args[6] = (uint8_t)(len >> 8); - return txn(MS_OP_READ, args, 7, buf, len); + if (!buf && len != 0) return false; + if (len > UINT32_MAX - off) return false; + uint32_t done = 0; + while (done < len) { + uint16_t chunk = (uint16_t)(len - done > MOTA_SEEDER_READ_MAX + ? MOTA_SEEDER_READ_MAX : len - done); + uint8_t args[7]; + args[0] = idx; + wr_u32le(args + 1, off + done); + args[5] = (uint8_t)(chunk & 0xFF); args[6] = (uint8_t)(chunk >> 8); + if (!txn(MS_OP_READ, args, 7, buf + done, chunk)) return false; + done += chunk; + } + return true; } } // namespace ota diff --git a/src/helpers/ota/OtaManager.cpp b/src/helpers/ota/OtaManager.cpp index b2e084bf2..98eb09b2b 100644 --- a/src/helpers/ota/OtaManager.cpp +++ b/src/helpers/ota/OtaManager.cpp @@ -362,28 +362,24 @@ void OtaManager::handleQuery(const uint8_t* m, uint16_t n) { } } -void OtaManager::handleGetManifest(const uint8_t* m, uint16_t n) { +bool OtaManager::handleGetManifest(const uint8_t* m, uint16_t n) { GetManifestMsg gm; - if (!decode_get_manifest(m, n, gm)) return; + if (!decode_get_manifest(m, n, gm)) return false; ServeView* v = resolve(gm.manifest_id); - if (!v) return; - // A signed v2 manifest (with hw_id[32]) exceeds one LoRa packet, so send it as fragments. Each carries - // up to OTA_MF_FRAG manifest bytes; the client reassembles by frag_idx. Only the fragments requested in - // want_mask are sent (0xFFFF = all), so a retry re-sends just the holes, not the whole manifest. - uint32_t mfl = v->mfl; - const uint8_t* src = v->m.manifest_start; - uint8_t ftotal = (uint8_t)((mfl + OTA_MF_FRAG - 1) / OTA_MF_FRAG); if (ftotal == 0) ftotal = 1; - for (uint8_t fi = 0; fi < ftotal; fi++) { - if (!(gm.want_mask & (1u << fi))) continue; // fetcher didn't ask for this manifest fragment - uint32_t off = (uint32_t)fi * OTA_MF_FRAG; - uint32_t fl = mfl - off; if (fl > OTA_MF_FRAG) fl = OTA_MF_FRAG; - ManifestMsg mm; - memcpy(mm.manifest_id, v->m.merkle_root, 4); - mm.frag_idx = fi; mm.frag_total = ftotal; - mm.bytes = src + off; mm.len = (uint16_t)fl; - uint8_t b[MAX_PACKET_PAYLOAD]; - emit(b, encode_manifest(b, sizeof(b), mm), false); + if (!v) return false; + OTA_DBG("OTA: GET_MANIFEST want=%04x pending=%u\n", + (unsigned)gm.want_mask, (unsigned)_n_manifest_jobs); + // Retain the request and let serviceEgress admit its fragments one at a time. In particular, a deployed + // receiver with the legacy 32-second SF5 flood delay must receive both fragments from the first request; + // serializing fragment 1 behind the receiver's next GET would exhaust its manifest retry window. + const bool queued = queueManifestJob(v->m.merkle_root, gm.want_mask); + if (!queued) { + OTA_DBG("OTA: GET_MANIFEST response queue full\n"); } + // A node that successfully queued the requested MID is the terminal source. Do not also re-flood the + // request; that echo competes with the manifest on the same temporary-radio channel. If the bounded queue + // is full, return false so another source or a later retry can service it. + return queued; } // Serve the target's merkle leaves[] in fragments (for a motatool folder-capture warm-start). Only the @@ -434,12 +430,101 @@ bool OtaManager::queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want return true; } +bool OtaManager::queueManifestJob(const uint8_t* mid, uint16_t want_mask) { + for (uint8_t i = 0; i < _n_manifest_jobs; i++) { + ManifestServeJob& job = _manifest_jobs[i]; + if (memcmp(job.mid, mid, 4) != 0) continue; + job.pending_mask |= (uint16_t)(want_mask & ~job.emitted_mask); + return true; + } + if (_n_manifest_jobs >= OTA_MANIFEST_SERVE_QUEUE) return false; + ManifestServeJob& job = _manifest_jobs[_n_manifest_jobs++]; + memcpy(job.mid, mid, 4); + job.pending_mask = want_mask; + job.emitted_mask = 0; + job.ready_at = _now_ms + manifestEgressGapMs(); + return true; +} + +uint32_t OtaManager::manifestEgressGapMs() const { + if (_radio_packet_airtime_ms == 0) return OTA_MANIFEST_EGRESS_MIN_GAP_MS; + uint64_t gap = (uint64_t)_radio_packet_airtime_ms * _tx_spacing_permille; + gap = (gap + 999u) / 1000u; + if (gap < OTA_MANIFEST_EGRESS_MIN_GAP_MS) gap = OTA_MANIFEST_EGRESS_MIN_GAP_MS; + if (gap > OTA_MANIFEST_EGRESS_MAX_GAP_MS) gap = OTA_MANIFEST_EGRESS_MAX_GAP_MS; + return (uint32_t)gap; +} + void OtaManager::clearPendingEgress() { + _n_manifest_jobs = 0; _n_serve_jobs = 0; _serve_block_len = 0; _serve_block_loaded = false; } +void OtaManager::popManifestJob() { + if (_n_manifest_jobs == 0) return; + for (uint8_t i = 1; i < _n_manifest_jobs; i++) _manifest_jobs[i - 1] = _manifest_jobs[i]; + _n_manifest_jobs--; +} + +bool OtaManager::serviceManifestEgress() { + if (_n_manifest_jobs == 0) return false; + ManifestServeJob& job = _manifest_jobs[0]; + if ((int32_t)(_now_ms - job.ready_at) < 0) { + return true; + } + ServeView* v = resolve(job.mid); + if (!v) { + popManifestJob(); + return true; + } + + const uint32_t mfl = v->mfl; + uint8_t ftotal = (uint8_t)((mfl + OTA_MF_FRAG - 1) / OTA_MF_FRAG); + if (ftotal == 0) ftotal = 1; + const uint16_t valid_mask = frag_full_mask(ftotal); + job.pending_mask &= valid_mask; + if (job.pending_mask == 0) { + popManifestJob(); + return true; + } + + uint8_t fragment = 0; + while (fragment < 16 && !(job.pending_mask & (1u << fragment))) fragment++; + if (fragment >= ftotal) { + popManifestJob(); + return true; + } + + const uint32_t off = (uint32_t)fragment * OTA_MF_FRAG; + uint32_t len = mfl - off; + if (len > OTA_MF_FRAG) len = OTA_MF_FRAG; + ManifestMsg message; + memcpy(message.manifest_id, v->m.merkle_root, 4); + message.frag_idx = fragment; + message.frag_total = ftotal; + message.bytes = v->m.manifest_start + off; + message.len = (uint16_t)len; + uint8_t wire[MAX_PACKET_PAYLOAD]; + const uint16_t wire_len = encode_manifest(wire, sizeof(wire), message); + if (emit(wire, wire_len, false)) { + OTA_DBG("OTA: MANIFEST tx frag=%u/%u len=%u\n", + (unsigned)fragment, (unsigned)ftotal, (unsigned)len); + const uint16_t bit = (uint16_t)(1u << fragment); + job.pending_mask &= (uint16_t)~bit; + job.emitted_mask |= bit; + if (job.pending_mask == 0) { + popManifestJob(); + } else { + // Follow the active packet airtime and dispatcher duty spacing while + // retaining a floor for fast radios' TX-to-RX turnaround. + job.ready_at = _now_ms + manifestEgressGapMs(); + } + } + return true; +} + void OtaManager::popServeJob() { if (_n_serve_jobs == 0) return; for (uint8_t i = 1; i < _n_serve_jobs; i++) _serve_jobs[i - 1] = _serve_jobs[i]; @@ -497,6 +582,10 @@ bool OtaManager::handleReqProof(const uint8_t* m, uint16_t n) { } void OtaManager::serviceEgress() { + // Metadata comes first so an older receiver gets every manifest fragment before its retry horizon. The + // send callback supplies radio-queue backpressure, and a rejected fragment remains in this descriptor. + if (serviceManifestEgress()) return; + // A proactive proof normally follows the final DATA fragment. If it was lost, or the source is older and // never sent one, issue the legacy proof request after a short grace. Stay RX-silent while another slot // still expects DATA, and restart the grace on every new packet, so this fallback cannot collide with a @@ -1586,7 +1675,7 @@ bool OtaManager::on_message(const uint8_t* msg, uint16_t len) { case OTA_ADV: handleAdv(msg, len); return false; case OTA_QUERY: handleQuery(msg, len); return false; case OTA_HAVE: handleHave(msg, len); return false; - case OTA_GET_MANIFEST: handleGetManifest(msg, len); return false; + case OTA_GET_MANIFEST: return handleGetManifest(msg, len); case OTA_MANIFEST: handleManifest(msg, len); return false; case OTA_GET_LEAVES: handleGetLeaves(msg, len); return false; case OTA_LEAVES: handleLeaves(msg, len); return false; diff --git a/src/helpers/ota/OtaManager.h b/src/helpers/ota/OtaManager.h index ff0f7ec7b..d81770c1d 100644 --- a/src/helpers/ota/OtaManager.h +++ b/src/helpers/ota/OtaManager.h @@ -59,6 +59,15 @@ typedef bool (*ServeReadFn)(void* ctx, uint32_t off, uint8_t* buf, uint32_t len) #ifndef OTA_MANIFEST_MAX_RETRY #define OTA_MANIFEST_MAX_RETRY 20 // give up (FAILED) after this many GET_MANIFEST retries - frees the slot #endif +#ifndef OTA_MANIFEST_SERVE_QUEUE +#define OTA_MANIFEST_SERVE_QUEUE 4 // bounded manifest response descriptors (fragments are emitted one at a time) +#endif +#ifndef OTA_MANIFEST_EGRESS_MIN_GAP_MS +#define OTA_MANIFEST_EGRESS_MIN_GAP_MS 100 // minimum fast-link TX-to-RX turnaround allowance +#endif +#ifndef OTA_MANIFEST_EGRESS_MAX_GAP_MS +#define OTA_MANIFEST_EGRESS_MAX_GAP_MS 1000 // do not outrun the receiver's one-second retry observation +#endif // Leaf-diff warm-start (motatool folder-capture only): bulk-fetch the target's leaves[], diff a seed build // locally, pull DATA only for mismatches. OTA_LEAVES is bitmap-fragmented like OTA_MANIFEST so a want_mask // retry never re-bursts (anti-deadlock). The mask is a fixed uint16, so leaves are capped at MAXFRAG frags. @@ -386,6 +395,7 @@ public: void serviceEgress(); void clearPendingEgress(); uint8_t pendingServeJobs() const { return _n_serve_jobs; } + uint8_t pendingManifestJobs() const { return _n_manifest_jobs; } // Drop the current fetch session back to IDLE so a fresh `ota pull` / advert starts a new one. void reset_session() { @@ -446,7 +456,7 @@ private: void handleAdv(const uint8_t* m, uint16_t n); // beacon -> sources table (+ query if interested) void handleQuery(const uint8_t* m, uint16_t n); // serve: reply OTA_HAVE catalog void handleHave(const uint8_t* m, uint16_t n); // peer: catalog rows (+ startFetch if a row matches) - void handleGetManifest(const uint8_t* m, uint16_t n); + bool handleGetManifest(const uint8_t* m, uint16_t n); void handleManifest(const uint8_t* m, uint16_t n); void handleGetLeaves(const uint8_t* m, uint16_t n); // serve: send the requested leaf fragments void handleLeaves(const uint8_t* m, uint16_t n); // fetcher (validate): reassemble the target leaves @@ -481,6 +491,10 @@ private: void registerSelfEntry(); // (re)build entry[0] from view0 static bool srcReadTramp(void* c, uint32_t off, uint8_t* buf, uint32_t len); // source payload reader bool queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want_mask); + bool queueManifestJob(const uint8_t* mid, uint16_t want_mask); + uint32_t manifestEgressGapMs() const; + bool serviceManifestEgress(); + void popManifestJob(); bool loadActiveServeBlock(); void popServeJob(); void sendQuery(const uint8_t* seeder, const uint8_t* digest, uint32_t filter_target, @@ -544,6 +558,14 @@ private: }; ServeJob _serve_jobs[OTA_SERVE_QUEUE]; uint8_t _n_serve_jobs = 0; + struct ManifestServeJob { + uint8_t mid[4]; + uint16_t pending_mask; + uint16_t emitted_mask; + uint32_t ready_at; + }; + ManifestServeJob _manifest_jobs[OTA_MANIFEST_SERVE_QUEUE]; + uint8_t _n_manifest_jobs = 0; uint8_t _serve_block[OTA_MAX_BLOCK]; uint16_t _serve_block_len = 0; bool _serve_block_loaded = false; diff --git a/test/test_ota/test_ota_core.cpp b/test/test_ota/test_ota_core.cpp index 511a12e7f..db3e0b72b 100644 --- a/test/test_ota/test_ota_core.cpp +++ b/test/test_ota/test_ota_core.cpp @@ -8,6 +8,8 @@ #include "helpers/ota/BlockBitmap.h" #include "helpers/ota/Multihash.h" #include "helpers/ota/FirmwareInfo.h" +#include "helpers/ota/MotaSeederProto.h" +#include "helpers/ota/MotaSourceSerial.h" #include "helpers/ota/SignerAllowlist.h" #include "helpers/ota/OtaStore.h" #include "helpers/ota/OtaProtocol.h" @@ -21,6 +23,80 @@ extern "C" { using namespace mesh::ota; +class FakeMotaSeederStream : public Stream { +public: + using Stream::write; + + size_t write(const uint8_t* request, size_t len) override { + request_valid = len == 11 && request[0] == MOTA_SEEDER_REQ_MAGIC0 + && request[1] == MOTA_SEEDER_REQ_MAGIC1 && request[2] == MS_OP_READ; + if (!request_valid) return len; + + uint8_t checksum = request[2]; + for (size_t i = 3; i + 1 < len; i++) checksum ^= request[i]; + request_valid = checksum == request[len - 1]; + const uint32_t offset = rd_u32le(request + 4); + const uint16_t read_len = rd_u16le(request + 8); + offsets.push_back(offset); + lengths.push_back(read_len); + + response.clear(); + response_pos = 0; + response.push_back(MOTA_SEEDER_RSP_MAGIC0); + response.push_back(MOTA_SEEDER_RSP_MAGIC1); + response.push_back(MS_OP_READ); + response.push_back(MS_STATUS_OK); + uint8_t response_checksum = MOTA_SEEDER_RSP_MAGIC0 ^ MOTA_SEEDER_RSP_MAGIC1 + ^ MS_OP_READ ^ MS_STATUS_OK; + for (uint16_t i = 0; i < read_len; i++) { + const uint8_t value = (uint8_t)(offset + i); + response.push_back(value); + response_checksum ^= value; + } + response.push_back(response_checksum); + return len; + } + + int read() override { + return response_pos < response.size() ? response[response_pos++] : -1; + } + + std::vector offsets; + std::vector lengths; + bool request_valid = true; + +private: + std::vector response; + size_t response_pos = 0; +}; + +TEST(MotaSourceSerial, SplitsOneKilobyteReadsBelowCdcReceiveRing) { + FakeMotaSeederStream stream; + SerialMotaSource source(stream); + std::array data{}; + + ASSERT_TRUE(source.read(0, 0x1000, data.data(), data.size())); + EXPECT_TRUE(stream.request_valid); + EXPECT_EQ(stream.offsets, + (std::vector{0x1000, 0x10C0, 0x1180, 0x1240, 0x1300, 0x13C0})); + EXPECT_EQ(stream.lengths, + (std::vector{192, 192, 192, 192, 192, 64})); + for (size_t i = 0; i < data.size(); i++) { + EXPECT_EQ(data[i], (uint8_t)(0x1000 + i)); + } +} + +TEST(MotaSourceSerial, AcceptsEmptyReadAndRejectsInvalidRange) { + FakeMotaSeederStream stream; + SerialMotaSource source(stream); + uint8_t byte = 0; + + EXPECT_TRUE(source.read(0, 123, nullptr, 0)); + EXPECT_FALSE(source.read(0, 123, nullptr, 1)); + EXPECT_FALSE(source.read(0, UINT32_MAX, &byte, 2)); + EXPECT_TRUE(stream.offsets.empty()); +} + // Build a flashed-image layout (body || fixed 56-byte EndF) the way the host packager / build hook do: // marker(4) body_len(4) body_hash8(8) fw_version(4) target_id(4) hw_id(32). Identity is always present // (zero/"" = unknown). @@ -840,6 +916,61 @@ TEST(OtaTransfer, ServerPacesOneKilobyteBlockAndProactiveProofWithBackpressure) EXPECT_EQ(server.pendingServeJobs(), 0u); } +TEST(OtaTransfer, ServerPacesManifestFragmentsAndConsumesResolvedRequest) { + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA, SIM_MOTA_LEN, manifest)); + + OtaManager server; + CapturedMessages sent; + server.begin(0, capture_send, &sent); + server.set_link_timing(80, 2000); // 160 ms at the active airtime/duty spacing + ASSERT_TRUE(server.serve(SIM_MOTA, SIM_MOTA_LEN)); + + GetManifestMsg request; + memcpy(request.manifest_id, manifest.merkle_root, 4); + request.want_mask = 0xFFFF; + uint8_t wire[MAX_PACKET_PAYLOAD]; + uint16_t wire_len = encode_get_manifest(wire, sizeof(wire), request); + ASSERT_GT(wire_len, 0); + + EXPECT_TRUE(server.on_message(wire, wire_len)); + EXPECT_TRUE(sent.items.empty()); + EXPECT_EQ(server.pendingManifestJobs(), 1u); + + const uint32_t manifest_gap = 160; + server.set_clock(manifest_gap - 1); + server.serviceEgress(); + EXPECT_TRUE(sent.items.empty()); + + server.set_clock(manifest_gap); + server.serviceEgress(); + ASSERT_EQ(sent.items.size(), 1u); + ManifestMsg fragment; + ASSERT_TRUE(decode_manifest(sent.items[0].data(), + (uint16_t)sent.items[0].size(), fragment)); + EXPECT_EQ(fragment.frag_idx, 0u); + EXPECT_EQ(fragment.frag_total, 2u); + + server.set_clock(2 * manifest_gap - 1); + server.serviceEgress(); + ASSERT_EQ(sent.items.size(), 1u); + + server.set_clock(2 * manifest_gap); + server.serviceEgress(); + ASSERT_EQ(sent.items.size(), 2u); + ASSERT_TRUE(decode_manifest(sent.items[1].data(), + (uint16_t)sent.items[1].size(), fragment)); + EXPECT_EQ(fragment.frag_idx, 1u); + EXPECT_EQ(server.pendingManifestJobs(), 0u); + + sent.items.clear(); + request.manifest_id[0] ^= 0xFF; + wire_len = encode_get_manifest(wire, sizeof(wire), request); + ASSERT_GT(wire_len, 0); + EXPECT_FALSE(server.on_message(wire, wire_len)); + EXPECT_TRUE(sent.items.empty()); +} + TEST(OtaTransfer, ServerQueuesEveryBlockInOneRequestFlight) { MotaManifest manifest; ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); diff --git a/test/test_trace_retry/test_trace_retry.cpp b/test/test_trace_retry/test_trace_retry.cpp index 5fdb68fb7..e5752f854 100644 --- a/test/test_trace_retry/test_trace_retry.cpp +++ b/test/test_trace_retry/test_trace_retry.cpp @@ -271,7 +271,7 @@ TEST(RepeaterTransport, OtaDiscoveryRelayKeepsCollisionJitter) { EXPECT_EQ(node.otaRelayDelay(&packet), 5u); // 0.5 * 10 ms } -TEST(RepeaterTransport, OtaTransferRelayIsImmediateEvenUnderRequestPressure) { +TEST(RepeaterTransport, OtaTransferRelayKeepsConfiguredCollisionJitter) { TraceTestClock clock; TraceTestRTC rtc; TraceTestRNG rng; @@ -291,18 +291,19 @@ TEST(RepeaterTransport, OtaTransferRelayIsImmediateEvenUnderRequestPressure) { mesh::Packet request = make_request(); node.receivePacket(&request); // first request establishes the key - EXPECT_EQ(node.otaRelayDelay(&request), 0u); + rng.value = 3; + EXPECT_EQ(node.otaRelayDelay(&request), 15u); for (uint32_t retry = 1; retry <= 3; retry++) { clock.now = retry * 3000; request = make_request(); // a fresh origin copy has the same zero-hop request key node.receivePacket(&request); // frequent repeats raise one level each } - rng.value = 20; - EXPECT_EQ(node.otaRelayDelay(&request), 0u); // private transfer never inherits flood backoff + rng.value = 3; + EXPECT_EQ(node.otaRelayDelay(&request), 15u); // request pressure does not widen primary traffic clock.now = 100000; - rng.value = 2; - EXPECT_EQ(node.otaRelayDelay(&request), 0u); + rng.value = 1; + EXPECT_EQ(node.otaRelayDelay(&request), 5u); } TEST(RepeaterTransport, TempRadioOtaBypassesReceiveHoldoffAndUsesFastCadRetry) { diff --git a/tools/lora_ota/build_rak3401_accelerated_bridges.py b/tools/lora_ota/build_rak3401_accelerated_bridges.py index 11281d634..d06aa1878 100755 --- a/tools/lora_ota/build_rak3401_accelerated_bridges.py +++ b/tools/lora_ota/build_rak3401_accelerated_bridges.py @@ -35,6 +35,7 @@ COMMON_PATCH = SCRIPT_DIR / "rak3401_terminal_ota_backport.patch" LEGACY_MESH_PATCH = SCRIPT_DIR / "rak3401_terminal_ota_mesh_legacy.patch" GUARDED_MESH_PATCH = SCRIPT_DIR / "rak3401_terminal_ota_mesh_guarded.patch" VERSION_PATCH = SCRIPT_DIR / "rak3401_four_component_endf_backport.patch" +WORKSPACE_PATCH = SCRIPT_DIR / "rak3401_dynamic_workspace_backport.patch" SELECTIVE_OS_HOOK = SCRIPT_DIR / "rak3401_selective_os.py" @@ -184,6 +185,7 @@ def apply_backport(source: Path, commit: str) -> None: required = ( COMMON_PATCH, LEGACY_MESH_PATCH if commit in LEGACY_MESH_COMMITS else GUARDED_MESH_PATCH, + WORKSPACE_PATCH, ) for patch in required: run(["git", "apply", "--check", str(patch)], f"check {patch.name}", source) @@ -213,7 +215,10 @@ def main() -> int: parser.add_argument("--keep-worktree", action="store_true") args = parser.parse_args() - for patch in (COMMON_PATCH, LEGACY_MESH_PATCH, GUARDED_MESH_PATCH, VERSION_PATCH): + for patch in ( + COMMON_PATCH, LEGACY_MESH_PATCH, GUARDED_MESH_PATCH, VERSION_PATCH, + WORKSPACE_PATCH, + ): if not patch.is_file(): raise BuildError(f"missing backport patch: {patch}") if not SELECTIVE_OS_HOOK.is_file(): @@ -243,7 +248,9 @@ def main() -> int: [ "git", "restore", "--source=HEAD", "--staged", "--worktree", "--", "src/Mesh.cpp", "src/helpers/ota/OtaManager.cpp", - "src/helpers/ota/OtaManager.h", "tools/mota/pio_endf.py", + "src/helpers/ota/OtaManager.h", + "src/helpers/ota/OtaFlashLayout_nrf52.h", + "tools/mota/pio_endf.py", ], "restore prior backport", source, @@ -353,6 +360,8 @@ def main() -> int: "guarded_mesh_patch_sha256": sha256_file(GUARDED_MESH_PATCH), "version_patch": VERSION_PATCH.name, "version_patch_sha256": sha256_file(VERSION_PATCH), + "workspace_patch": WORKSPACE_PATCH.name, + "workspace_patch_sha256": sha256_file(WORKSPACE_PATCH), "selective_os_hook": SELECTIVE_OS_HOOK.name, "selective_os_hook_sha256": sha256_file(SELECTIVE_OS_HOOK), "targets": records, diff --git a/tools/lora_ota/build_rak3401_compact_bundle.py b/tools/lora_ota/build_rak3401_compact_bundle.py new file mode 100755 index 000000000..ee74e74fc --- /dev/null +++ b/tools/lora_ota/build_rak3401_compact_bundle.py @@ -0,0 +1,678 @@ +#!/usr/bin/env python3 +"""Build the compact legacy-bootloader RAK3401 mOTA chain. + +The first package is retained byte-for-byte from the pinned physical baseline. +Later packages use page-aligned detools workspaces selected by the audited +route plan. Every package must fit below the deployed bootloader's original +0xD4000 scan ceiling; no expanded-ceiling bootloader feature is assumed. +""" + +from __future__ import annotations + +import argparse +import csv +from datetime import datetime, timezone +import json +from pathlib import Path +import re +import shutil +import subprocess +import sys +import zipfile + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parent.parent +sys.path.insert(0, str(REPO_ROOT / "tools" / "mota")) + +import motalib # noqa: E402 +import rebuild_rak3401_bundle as common # noqa: E402 + + +PREVIOUS_BUNDLE_SHA256 = ( + "b2781e02460b200a7c37bfae352bad81618716e550d1d042dca8aa29bfc73c29" +) +EXPECTED_TARGET_ID = 0x2FA509C1 +EXPECTED_HARDWARE = "RAK_3401" +EXPECTED_ENDPOINT_VERSION = "1.17.1.02" +EXPECTED_ENDPOINT_VERSION_PACKED = motalib.pack_version(EXPECTED_ENDPOINT_VERSION) +EXPECTED_ROUTE_NODES = (0, 1, 2, 7, 13, 16, 17, 22, 24, 30) +EXPECTED_STEPS = len(EXPECTED_ROUTE_NODES) - 1 +APP_BASE = 0x26000 +STAGE_CEILING = 0xD4000 +FIXED_WORKSPACE = 0x98000 +FLASH_PAGE = 4096 +SEGMENT_SIZE = 4096 +BLOCK_SIZE = 1024 + + +class CompactBuildError(RuntimeError): + pass + + +def run(command: list[str], label: str, cwd: Path | None = None, timeout: int = 900) -> str: + try: + result = subprocess.run( + command, + cwd=cwd, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout, + check=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise CompactBuildError(f"{label} failed: {exc}") from exc + if result.returncode != 0: + raise CompactBuildError( + f"{label} exited {result.returncode}:\n{result.stdout[-6000:]}" + ) + return result.stdout + + +def align_up(value: int, unit: int) -> int: + return (value + unit - 1) // unit * unit + + +def align_down(value: int, unit: int) -> int: + return value // unit * unit + + +def parse_int(value: object, label: str) -> int: + if isinstance(value, int): + result = value + elif isinstance(value, str) and re.fullmatch(r"(?:0[xX][0-9a-fA-F]+|[0-9]+)", value): + result = int(value, 0) + else: + raise CompactBuildError(f"{label} must be an integer or 0x-prefixed integer") + if result < 0: + raise CompactBuildError(f"{label} cannot be negative") + return result + + +def parse_simulators(values: list[str]) -> list[tuple[str, Path]]: + simulators: list[tuple[str, Path]] = [] + for value in values: + label, separator, raw_path = value.partition("=") + if not separator or not re.fullmatch(r"[A-Za-z0-9_.-]+", label): + raise CompactBuildError("--apply-sim must use LABEL=/path/to/apply_sim") + path = Path(raw_path) + if not path.is_file(): + raise CompactBuildError(f"apply simulator does not exist: {path}") + simulators.append((label, path)) + return simulators + + +def read_route(path: Path) -> tuple[list[dict[str, object]], dict[str, object]]: + try: + document = json.loads(path.read_text(encoding="ascii")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise CompactBuildError(f"cannot read route plan {path}: {exc}") from exc + if not isinstance(document, dict) or document.get("schema") != 1: + raise CompactBuildError("route plan must use schema 1") + if parse_int(document.get("app_base"), "route app_base") != APP_BASE: + raise CompactBuildError("route plan has the wrong application base") + if parse_int(document.get("stage_ceiling"), "route stage_ceiling") != STAGE_CEILING: + raise CompactBuildError("route plan does not target the deployed 0xD4000 ceiling") + if document.get("shortest_package_count") != EXPECTED_STEPS: + raise CompactBuildError( + f"route plan shortest_package_count must be {EXPECTED_STEPS}" + ) + raw_steps = document.get("steps") + if not isinstance(raw_steps, list) or len(raw_steps) != EXPECTED_STEPS: + raise CompactBuildError(f"route plan must contain {EXPECTED_STEPS} steps") + steps: list[dict[str, object]] = [] + nodes = [] + for number, raw in enumerate(raw_steps, 1): + if not isinstance(raw, dict): + raise CompactBuildError(f"route step {number} must be an object") + source = parse_int(raw.get("source_node"), f"route step {number} source_node") + target = parse_int(raw.get("target_node"), f"route step {number} target_node") + memory = parse_int(raw.get("inplace_memory"), f"route step {number} inplace_memory") + expected_size = parse_int( + raw.get("expected_container_size"), + f"route step {number} expected_container_size", + ) + expected_margin = parse_int( + raw.get("expected_staging_margin"), + f"route step {number} expected_staging_margin", + ) + expected_sha = raw.get("expected_target_sha256") + expected_version = raw.get("expected_target_version") + if not isinstance(expected_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", expected_sha): + raise CompactBuildError(f"route step {number} target SHA-256 is invalid") + if not isinstance(expected_version, str): + raise CompactBuildError(f"route step {number} target version is invalid") + try: + motalib.pack_version(expected_version) + except (TypeError, ValueError): + raise CompactBuildError(f"route step {number} target version is invalid") from None + reuse = raw.get("reuse_baseline_package", False) + if not isinstance(reuse, bool): + raise CompactBuildError(f"route step {number} reuse flag must be boolean") + if source >= target or memory % FLASH_PAGE: + raise CompactBuildError(f"route step {number} has invalid nodes or workspace alignment") + if memory < FLASH_PAGE or APP_BASE + memory >= STAGE_CEILING: + raise CompactBuildError(f"route step {number} workspace is outside legacy flash") + if number == 1: + nodes.append(source) + elif source != steps[-1]["target_node"]: + raise CompactBuildError(f"route step {number} is discontinuous") + nodes.append(target) + steps.append({ + "source_node": source, + "target_node": target, + "inplace_memory": memory, + "reuse_baseline_package": reuse, + "expected_container_size": expected_size, + "expected_staging_margin": expected_margin, + "expected_target_sha256": expected_sha, + "expected_target_version": expected_version, + }) + if tuple(nodes) != EXPECTED_ROUTE_NODES: + raise CompactBuildError( + f"route nodes are {tuple(nodes)}, expected {EXPECTED_ROUTE_NODES}" + ) + if not steps[0]["reuse_baseline_package"] or any( + step["reuse_baseline_package"] for step in steps[1:] + ): + raise CompactBuildError("only the first route step may reuse the baseline package") + if steps[0]["inplace_memory"] != FIXED_WORKSPACE or steps[1]["inplace_memory"] != FIXED_WORKSPACE: + raise CompactBuildError("the two fixed-receiver steps require a 0x98000 workspace") + return steps, document + + +def read_input_rows(root: Path) -> list[dict[str, str]]: + try: + with (root / "CHAIN.csv").open(newline="", encoding="ascii") as source: + rows = list(csv.DictReader(source)) + except OSError as exc: + raise CompactBuildError(f"cannot read baseline CHAIN.csv: {exc}") from exc + if len(rows) != 30: + raise CompactBuildError(f"baseline chain has {len(rows)} steps, expected 30") + return rows + + +def firmware_identity(image: bytes) -> tuple[int, str, str, int]: + if not motalib.has_endf(image): + raise CompactBuildError("firmware image has no valid EndF") + ident = motalib.parse_endf_ident(image) + assert ident is not None + if ident.target_id != EXPECTED_TARGET_ID or ident.hw_id != EXPECTED_HARDWARE: + raise CompactBuildError( + f"firmware identity is target={ident.target_id:08X} hw={ident.hw_id!r}" + ) + body, body_hash = motalib.parse_endf(image) + return ident.fw_version, body_hash.hex(), motalib.unpack_version(ident.fw_version), len(body) + + +def apply_in_place( + detools: Path, + base: bytes, + payload: bytes, + target_size: int, + memory_size: int, + work: Path, + label: str, + fill: int, +) -> bytes: + if len(base) > memory_size or target_size > memory_size: + raise CompactBuildError(f"{label} image exceeds the selected workspace") + memory_path = work / f"{label}-{fill:02x}.memory" + payload_path = work / f"{label}.patch" + memory = bytearray([fill]) * memory_size + memory[:len(base)] = base + memory_path.write_bytes(memory) + if not payload_path.exists(): + payload_path.write_bytes(payload) + run( + [str(detools), "apply_patch_in_place", str(memory_path), str(payload_path)], + f"apply {label} with fill 0x{fill:02x}", + ) + result = memory_path.read_bytes()[:target_size] + memory_path.unlink() + return result + + +def patch_geometry(detools: Path, payload_path: Path, label: str) -> tuple[int, int, int, int]: + output = run( + [str(detools), "patch_info", "--no-human", str(payload_path)], + f"inspect {label}", + ) + values: dict[str, int] = {} + for key, output_key in ( + ("Memory size", "memory"), + ("Segment size", "segment"), + ("From size", "from_size"), + ("To size", "to_size"), + ): + match = re.search(rf"^{re.escape(key)}:\s+([0-9]+) bytes$", output, re.MULTILINE) + if match is None: + raise CompactBuildError(f"{label} patch_info is missing {key}") + values[output_key] = int(match.group(1)) + return values["memory"], values["segment"], values["from_size"], values["to_size"] + + +def write_chain(path: Path, rows: list[dict[str, object]]) -> None: + fields = [ + "step", "from_version", "to_version", "mota_file", "mota_size", + "inplace_memory", "stage_start", "workspace_end", "staging_margin", + "target_image_size", "base_body_hash", "target_body_hash", + "target_sha256", "source_commit", + ] + with path.open("w", newline="", encoding="ascii") as output: + writer = csv.DictWriter(output, fieldnames=fields, lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + + +def write_docs( + root: Path, + source_commit: str, + endpoint: bytes, + previous_sha: str, + bridge_sha: str, + route_sha: str, + rows: list[dict[str, object]], +) -> None: + endpoint_sha = common.sha256_bytes(endpoint) + _packed, endpoint_body, _version, _body_size = firmware_identity(endpoint) + total_bytes = sum(int(row["mota_size"]) for row in rows) + largest = max(int(row["mota_size"]) for row in rows) + minimum_margin = min(int(row["staging_margin"]) for row in rows) + (root / "README.md").write_text( + f"""# RAK3401 compact mOTA update chain + +This archive contains {EXPECTED_STEPS} mandatory mOTA packages from the deployed +`1.16.7.0-c1caa5ad` image to +`v{EXPECTED_ENDPOINT_VERSION}-halo-keymind-cascade-dev-{source_commit[:8]}`. + +The deployed bootloader is unchanged. Every flash package is bottom-aligned +below its original `0x{STAGE_CEILING:X}` scan ceiling, and every encoded detools +workspace ends at or below that package. No byte from `0x{STAGE_CEILING:X}` +through `0xED000` is used by this chain. + +- Packages: {EXPECTED_STEPS} (the replaced chain used 30) +- Total transferred bytes: {total_bytes:,} +- Largest package: {largest:,} bytes +- Smallest workspace-to-stage margin: {minimum_margin:,} bytes +- Endpoint image SHA-256: `{endpoint_sha}` +- Endpoint EndF body hash: `{endpoint_body.upper()}` + +Step 1 is byte-for-byte identical to the physically passed baseline package. +Step 2 installs the dynamic-workspace receiver while still obeying the old +fixed `0x{FIXED_WORKSPACE:X}` receiver limit. Steps 3-{EXPECTED_STEPS} use the +page-aligned workspace selected in `ROUTE.json`. + +All {EXPECTED_STEPS} transitions passed container verification, zero-filled and +erased-workspace reconstruction, the supplied legacy bootloader simulator, and +the exact physical chain on a deployed RAK3401 1W on 19-Aug-2026. Use the pinned +chain runner and do not install packages out of order. +""", + encoding="ascii", + ) + (root / "PROVENANCE.md").write_text( + f"""# Build and validation provenance + +Endpoint source commit: `{source_commit}` +Endpoint requested version: `{EXPECTED_ENDPOINT_VERSION}` +Endpoint packed version: `0x{EXPECTED_ENDPOINT_VERSION_PACKED:08X}` +Endpoint profile name: `halo-keymind-cascade-dev` +Legacy stage ceiling: `0x{STAGE_CEILING:X}` +Application base: `0x{APP_BASE:X}` +Segment size: `{SEGMENT_SIZE}` +Logical mOTA block size: `{BLOCK_SIZE}` + +Pinned 30-step reconstruction input SHA-256: `{previous_sha}` +Dynamic bridge-image manifest SHA-256: `{bridge_sha}` +Compact route plan SHA-256: `{route_sha}` + +The route plan records the shortest path found by sweeping every page-aligned +workspace relevant to a path shorter than {EXPECTED_STEPS} packages. The +bundle generator independently checks the selected geometry against the old +bootloader ceiling before invoking either simulator. +""", + encoding="ascii", + ) + (root / "RUNBOOK.md").write_text( + """# Runbook + +Use `tools/lora_ota/rak3401_mota_chain.py` from the matching MeshCore commit. +Run `--verify-only`, then `--preflight-only`, before a live attempt. Live use +is enabled for this physically qualified exact chain. Keep the persistent work +directory so an interrupted transfer resumes from the target's exact EndF body +hash. + +Do not use `--inplace-memory 0x98000` as a manual override for these files. +Each package already carries its audited workspace in the detools payload. +""", + encoding="ascii", + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--previous-bundle", type=Path, required=True) + parser.add_argument("--endpoint-zip", type=Path, required=True) + parser.add_argument("--endpoint-uf2", type=Path, required=True) + parser.add_argument("--accelerated-images", type=Path, required=True) + parser.add_argument("--route-plan", type=Path, required=True) + parser.add_argument("--work-dir", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--motatool", type=Path, required=True) + parser.add_argument("--detools", type=Path, required=True) + parser.add_argument( + "--apply-sim", + action="append", + default=[], + metavar="LABEL=PATH", + help="apply simulator and stable provenance label; repeat to validate multiple bootloaders", + ) + parser.add_argument("--source-commit", required=True) + args = parser.parse_args() + + for path, label in ( + (args.previous_bundle, "previous bundle"), + (args.endpoint_zip, "endpoint ZIP"), + (args.endpoint_uf2, "endpoint UF2"), + (args.accelerated_images, "accelerated image manifest"), + (args.route_plan, "route plan"), + (args.motatool, "motatool"), + (args.detools, "detools"), + ): + if not path.is_file(): + raise CompactBuildError(f"{label} does not exist: {path}") + simulators = parse_simulators(args.apply_sim) + if not re.fullmatch(r"[0-9a-f]{40}", args.source_commit): + raise CompactBuildError("source commit must be a full lowercase 40-hex commit") + resolved = run( + ["git", "-C", str(REPO_ROOT), "rev-parse", "--verify", f"{args.source_commit}^{{commit}}"], + "resolve endpoint source commit", + ).strip() + if resolved != args.source_commit: + raise CompactBuildError( + f"endpoint source resolves to {resolved}, expected {args.source_commit}" + ) + if args.work_dir.exists(): + raise CompactBuildError(f"work directory already exists: {args.work_dir}") + args.work_dir.mkdir(parents=True) + + route, route_document = read_route(args.route_plan) + if ( + route_document.get("endpoint_source_commit") != args.source_commit + or route_document.get("endpoint_version") != EXPECTED_ENDPOINT_VERSION + or route_document.get("endpoint_name") != "halo-keymind-cascade-dev" + ): + raise CompactBuildError("route plan endpoint identity does not match this build") + previous_sha = common.sha256_file(args.previous_bundle) + if previous_sha != PREVIOUS_BUNDLE_SHA256: + raise CompactBuildError( + f"previous bundle SHA-256 is {previous_sha}, expected {PREVIOUS_BUNDLE_SHA256}" + ) + old_root = common.extract_zip(args.previous_bundle, args.work_dir / "old") + old_rows = read_input_rows(old_root) + start_zip = old_root / "recovery/test-start/RAK3401-test-start-v1.16.7-c1caa5ad.zip" + image0 = common.read_firmware_zip(start_zip) + firmware_identity(image0) + first_relative = common.safe_relative(old_rows[0]["mota_file"], "baseline step-1 mOTA") + first_package_path = old_root.joinpath(*first_relative.parts) + first_package = motalib.parse_container(first_package_path.read_bytes()) + reconstruction = args.work_dir / "reconstruction" + reconstruction.mkdir() + image1 = apply_in_place( + args.detools, image0, first_package.payload, + int(old_rows[0]["target_image_size"]), FIXED_WORKSPACE, + reconstruction, "baseline-step-01", 0xFF, + ) + image1_zero = apply_in_place( + args.detools, image0, first_package.payload, + int(old_rows[0]["target_image_size"]), FIXED_WORKSPACE, + reconstruction, "baseline-step-01", 0x00, + ) + if image1 != image1_zero or common.sha256_bytes(image1) != old_rows[0]["target_sha256"]: + raise CompactBuildError("baseline step 1 does not reconstruct its pinned target") + + accelerated, accelerated_sources = common.read_accelerated_images(args.accelerated_images) + endpoint = common.read_firmware_zip(args.endpoint_zip) + endpoint_packed, _body_hash, _version_text, _body_size = firmware_identity(endpoint) + if endpoint_packed != EXPECTED_ENDPOINT_VERSION_PACKED: + raise CompactBuildError( + f"endpoint packed version is 0x{endpoint_packed:08X}, " + f"expected 0x{EXPECTED_ENDPOINT_VERSION_PACKED:08X}" + ) + expected_label = f"v{EXPECTED_ENDPOINT_VERSION}-halo-keymind-cascade-dev-{args.source_commit[:8]}" + if expected_label.encode("ascii") not in endpoint: + raise CompactBuildError(f"endpoint firmware does not contain {expected_label}") + + images = {0: image0, 1: image1} + sources = {1: old_rows[0]["source_commit"]} + for node, (image, source_commit) in enumerate(zip(accelerated, accelerated_sources), 2): + images[node] = image + sources[node] = source_commit + images[30] = endpoint + sources[30] = args.source_commit + expected_nodes = set(range(31)) + if set(images) != expected_nodes: + # Do not silently accept a shortened or reordered bridge manifest. + raise CompactBuildError(f"assembled image nodes are {sorted(images)}") + + short_commit = args.source_commit[:8] + root_name = ( + "RAK3401-update-chain-v1.16.7-c1caa5ad-to-" + f"v{EXPECTED_ENDPOINT_VERSION}-{short_commit}" + ) + root = args.work_dir / root_name + motas = root / "motas" + recovery_start = root / "recovery/test-start" + recovery_final = root / "recovery/final" + motas.mkdir(parents=True) + recovery_start.mkdir(parents=True) + recovery_final.mkdir(parents=True) + validation_work = args.work_dir / "validation" + validation_work.mkdir() + + output_rows: list[dict[str, object]] = [] + validation_steps: list[dict[str, object]] = [] + for number, plan_step in enumerate(route, 1): + source_node = int(plan_step["source_node"]) + target_node = int(plan_step["target_node"]) + memory_size = int(plan_step["inplace_memory"]) + base_image = images[source_node] + target_image = images[target_node] + base_packed, base_body, base_version, _base_body_size = firmware_identity(base_image) + target_packed, target_body, target_version, _target_body_size = firmware_identity(target_image) + from_version = base_version + to_version = EXPECTED_ENDPOINT_VERSION if target_node == 30 else target_version + filename = f"step-{number:02d}__v{from_version}-to-v{to_version}.mota" + output_path = motas / filename + base_path = validation_work / f"base-{number:02d}.bin" + target_path = validation_work / f"target-{number:02d}.bin" + payload_path = validation_work / f"payload-{number:02d}.patch" + base_path.write_bytes(base_image) + target_path.write_bytes(target_image) + if bool(plan_step["reuse_baseline_package"]): + shutil.copy2(first_package_path, output_path) + build_kind = "byte-identical physical baseline" + else: + build_output = run( + [ + str(args.motatool), "build", "--fw", str(target_path), + "--base", str(base_path), "--patch-type", "in-place", + "--inplace-memory", hex(memory_size), + "--segment-size", str(SEGMENT_SIZE), + "--block-size", str(BLOCK_SIZE), "--out", str(output_path), + ], + f"build compact step {number}", + ) + if not build_output.strip(): + raise CompactBuildError(f"motatool produced no output for step {number}") + build_kind = "freshly generated" + run([str(args.motatool), "verify", str(output_path)], f"verify compact step {number}") + parsed = motalib.parse_container(output_path.read_bytes()) + payload_path.write_bytes(parsed.payload) + patch_memory, patch_segment, patch_from, patch_to = patch_geometry( + args.detools, payload_path, f"compact step {number}" + ) + if ( + patch_memory != memory_size + or patch_segment != SEGMENT_SIZE + or patch_from != len(base_image) + or patch_to != len(target_image) + ): + raise CompactBuildError( + f"step {number} detools geometry does not match its route plan/images" + ) + if parsed.manifest.fw_version != target_packed: + raise CompactBuildError(f"step {number} manifest version does not match target EndF") + if parsed.manifest.base_hash.hex() != base_body: + raise CompactBuildError(f"step {number} manifest base hash mismatch") + target_sha = common.sha256_bytes(target_image) + if parsed.manifest.image_hash.hex() != target_sha: + raise CompactBuildError(f"step {number} manifest target hash mismatch") + if target_sha != plan_step["expected_target_sha256"]: + raise CompactBuildError(f"step {number} target does not match its route pin") + if target_packed != motalib.pack_version(str(plan_step["expected_target_version"])): + raise CompactBuildError(f"step {number} target version does not match its route pin") + if parsed.manifest.target_id != EXPECTED_TARGET_ID: + raise CompactBuildError(f"step {number} manifest target ID mismatch") + if parsed.manifest.image_size != len(target_image): + raise CompactBuildError(f"step {number} manifest target size mismatch") + + package_size = output_path.stat().st_size + if package_size != plan_step["expected_container_size"]: + raise CompactBuildError( + f"step {number} is {package_size} bytes, expected " + f"{plan_step['expected_container_size']}" + ) + stage_start = align_down(STAGE_CEILING - package_size, FLASH_PAGE) + workspace_end = APP_BASE + memory_size + staging_margin = stage_start - workspace_end + if stage_start < APP_BASE + len(base_image): + raise CompactBuildError(f"step {number} staged package overlaps its running image") + if staging_margin < 0: + raise CompactBuildError( + f"step {number} needs {-staging_margin} bytes beyond the legacy ceiling" + ) + if staging_margin != plan_step["expected_staging_margin"]: + raise CompactBuildError( + f"step {number} staging margin is {staging_margin}, expected " + f"{plan_step['expected_staging_margin']}" + ) + if memory_size < align_up(max(len(base_image) + 2 * SEGMENT_SIZE, len(target_image)), SEGMENT_SIZE): + raise CompactBuildError(f"step {number} workspace is below detools' safe minimum") + + zero = apply_in_place( + args.detools, base_image, parsed.payload, len(target_image), memory_size, + validation_work, f"compact-step-{number:02d}", 0x00, + ) + erased = apply_in_place( + args.detools, base_image, parsed.payload, len(target_image), memory_size, + validation_work, f"compact-step-{number:02d}", 0xFF, + ) + if zero != target_image or erased != target_image: + raise CompactBuildError(f"step {number} reconstruction does not match target") + simulator_results = [] + for simulator_label, simulator in simulators: + run( + [str(simulator), str(base_path), str(output_path), str(target_path)], + f"{simulator_label} step {number}", + ) + simulator_results.append({ + "label": simulator_label, + "sha256": common.sha256_file(simulator), + }) + + output_rows.append({ + "step": number, + "from_version": from_version, + "to_version": to_version, + "mota_file": f"motas/{filename}", + "mota_size": package_size, + "inplace_memory": f"0x{memory_size:X}", + "stage_start": f"0x{stage_start:X}", + "workspace_end": f"0x{workspace_end:X}", + "staging_margin": staging_margin, + "target_image_size": len(target_image), + "base_body_hash": base_body, + "target_body_hash": target_body, + "target_sha256": target_sha, + "source_commit": sources[target_node], + }) + validation_steps.append({ + "step": number, + "source_node": source_node, + "target_node": target_node, + "build_kind": build_kind, + "inplace_memory": memory_size, + "stage_start": stage_start, + "workspace_end": workspace_end, + "staging_margin": staging_margin, + "mota_sha256": common.sha256_file(output_path), + "mota_size": package_size, + "target_sha256": target_sha, + "zero_fill_apply": "passed", + "erased_fill_apply": "passed", + "apply_simulators": simulator_results, + }) + print( + f"[compact] {number:02d}/{EXPECTED_STEPS} nodes={source_node}->{target_node} " + f"memory=0x{memory_size:X} size={package_size} margin={staging_margin}", + flush=True, + ) + + write_chain(root / "CHAIN.csv", output_rows) + shutil.copy2(start_zip, recovery_start) + start_uf2 = old_root / "recovery/test-start/RAK3401-test-start-v1.16.7-c1caa5ad.uf2" + shutil.copy2(start_uf2, recovery_start) + shutil.copy2(args.endpoint_zip, recovery_final) + shutil.copy2(args.endpoint_uf2, recovery_final) + shutil.copy2(args.accelerated_images, root / "BRIDGE-IMAGES.json") + shutil.copy2(args.route_plan, root / "ROUTE.json") + baseline = old_root / "PHYSICAL-BASELINE-cd824765.jsonl" + if baseline.is_file(): + shutil.copy2(baseline, root) + + route_sha = common.sha256_file(args.route_plan) + bridge_sha = common.sha256_file(args.accelerated_images) + write_docs( + root, args.source_commit, endpoint, previous_sha, bridge_sha, + route_sha, output_rows, + ) + validation = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "generator": "tools/lora_ota/build_rak3401_compact_bundle.py", + "source_commit": args.source_commit, + "endpoint_requested_version": EXPECTED_ENDPOINT_VERSION, + "endpoint_packed_version": EXPECTED_ENDPOINT_VERSION_PACKED, + "endpoint_name": "halo-keymind-cascade-dev", + "previous_bundle_sha256": previous_sha, + "bridge_images_sha256": bridge_sha, + "route_plan_sha256": route_sha, + "route_search": route_document.get("search"), + "app_base": APP_BASE, + "stage_ceiling": STAGE_CEILING, + "segment_size": SEGMENT_SIZE, + "block_size": BLOCK_SIZE, + "steps": validation_steps, + } + (root / "validation-results.json").write_text( + json.dumps(validation, indent=2, sort_keys=True) + "\n", + encoding="ascii", + ) + checksum_sha = common.write_checksums(root) + args.output_dir.mkdir(parents=True, exist_ok=True) + output_path = args.output_dir / f"{root_name}.zip" + common.make_zip(root, output_path) + print(f"[bundle] {output_path}") + print(f"[bundle] sha256={common.sha256_file(output_path)}") + print(f"[bundle] inner_sha256={checksum_sha}") + print(f"[bundle] endpoint_sha256={common.sha256_bytes(endpoint)}") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (CompactBuildError, common.RebuildError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + raise SystemExit(2) diff --git a/tools/lora_ota/lora_ota.py b/tools/lora_ota/lora_ota.py index e4934a572..e2c798a96 100755 --- a/tools/lora_ota/lora_ota.py +++ b/tools/lora_ota/lora_ota.py @@ -70,6 +70,9 @@ COMPANION_TERMINAL_STOP = "+++MESHCORE-TERM-STOP" # Firmware may hold the apply reboot for up to 15 seconds while its reply # drains. Do not interpret "still ready" as a failed install before that cap. INSTALL_RECONCILE_WAIT_SECONDS = 20 +DEFAULT_RELAY_TX_DELAY = 0.3 +RELAY_TIMING_COMMANDS_PER_RELAY = 12 +RELAY_TIMING_RECOVERY_FILE = "relay-timing-settings.json" T = TypeVar("T") @@ -174,6 +177,14 @@ class RadioSettings: ) +@dataclass(frozen=True) +class RelayTimingSettings: + name: str + password: str + rxdelay: float + txdelay: float + + def prompt_after_transmission_failure( label: str, error: TransmissionError, @@ -1838,6 +1849,8 @@ class SeederProcess: "--serial", args.source_serial, "--baud", str(args.source_baud), ]) + if getattr(args, "source_companion_terminal", False): + command.append("--companion-terminal") else: command.extend(["--tcp", args.source_tcp]) self.command = command @@ -1938,6 +1951,130 @@ def parse_relay(value: str, default_password: str) -> tuple[str, str]: return value, default_password +def parse_delay_reply(reply: str, label: str) -> float: + match = re.fullmatch(r"\s*>\s*([0-9]+(?:\.[0-9]+)?)\s*", reply) + if match is None: + raise OtaError(f"could not read {label}: {reply}") + return float(match.group(1)) + + +def read_relay_timing( + controller: Controller, + relay_name: str, + relay_password: str, +) -> RelayTimingSettings: + rxdelay = parse_delay_reply( + controller.remote_command( + relay_name, "get rxdelay", password=relay_password + ), + f"{relay_name} rxdelay", + ) + txdelay = parse_delay_reply( + controller.remote_command( + relay_name, "get txdelay", password=relay_password + ), + f"{relay_name} txdelay", + ) + return RelayTimingSettings( + name=relay_name, + password=relay_password, + rxdelay=rxdelay, + txdelay=txdelay, + ) + + +def write_relay_timing_recovery( + work_dir: Path, + settings: list[RelayTimingSettings], +) -> Path: + path = work_dir / RELAY_TIMING_RECOVERY_FILE + payload = [ + { + "name": item.name, + "rxdelay": item.rxdelay, + "txdelay": item.txdelay, + } + for item in settings + ] + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="ascii", + ) + temporary.chmod(0o600) + os.replace(temporary, path) + return path + + +def enforce_relay_timing( + controller: Controller, + saved: RelayTimingSettings, + txdelay: float, +) -> None: + if abs(saved.rxdelay) > 0.0001: + reply = controller.remote_command( + saved.name, "set rxdelay 0", password=saved.password + ) + if not reply.upper().startswith("OK"): + raise OtaError(f"{saved.name} did not disable rxdelay: {reply}") + if abs(saved.txdelay - txdelay) > 0.0001: + reply = controller.remote_command( + saved.name, + f"set txdelay {format_decimal(txdelay)}", + password=saved.password, + ) + if not reply.upper().startswith("OK"): + raise OtaError(f"{saved.name} did not accept OTA txdelay: {reply}") + + verified = read_relay_timing(controller, saved.name, saved.password) + if abs(verified.rxdelay) > 0.0001 or abs(verified.txdelay - txdelay) > 0.0001: + raise OtaError( + f"{saved.name} transfer timing did not read back as " + f"rxdelay=0, txdelay={format_decimal(txdelay)}" + ) + print( + f"[relays] {saved.name} verified at rxdelay 0, " + f"txdelay {format_decimal(txdelay)}" + ) + + +def restore_relay_timings( + controller: Controller, + settings: list[RelayTimingSettings], +) -> None: + errors: list[str] = [] + for saved in reversed(settings): + try: + current = read_relay_timing(controller, saved.name, saved.password) + if abs(current.rxdelay - saved.rxdelay) > 0.0001: + reply = controller.remote_command( + saved.name, + f"set rxdelay {format_decimal(saved.rxdelay)}", + password=saved.password, + ) + if not reply.upper().startswith("OK"): + raise OtaError(f"did not restore rxdelay: {reply}") + if abs(current.txdelay - saved.txdelay) > 0.0001: + reply = controller.remote_command( + saved.name, + f"set txdelay {format_decimal(saved.txdelay)}", + password=saved.password, + ) + if not reply.upper().startswith("OK"): + raise OtaError(f"did not restore txdelay: {reply}") + verified = read_relay_timing(controller, saved.name, saved.password) + if ( + abs(verified.rxdelay - saved.rxdelay) > 0.0001 + or abs(verified.txdelay - saved.txdelay) > 0.0001 + ): + raise OtaError("relay timing did not restore exactly") + print(f"[relays] {saved.name} original rxdelay/txdelay restored") + except (OtaError, OSError) as exc: + errors.append(f"{saved.name}: {exc}") + if errors: + raise OtaError("could not restore relay timings: " + "; ".join(errors)) + + def confirm_update( args: argparse.Namespace, target: TargetInfo, @@ -1957,6 +2094,11 @@ def confirm_update( print(f" update : {package.version} {package.kind} hw={package.hw_id or '?'}") print(f" mOTA id : {package.manifest_id}") print(f" TempRadio : {args.temp_radio}") + if args.relay: + print( + f" relay timing: rxdelay 0, " + f"txdelay {format_decimal(args.relay_txdelay)} (saved/restored)" + ) print(f" action : {'stage only' if args.no_install else 'install and reboot'}") current_version = ( parse_version(target.current_version) if target.current_version else None @@ -2738,6 +2880,15 @@ def build_parser() -> argparse.ArgumentParser: "--relay", action="append", default=[], metavar="NAME[=PASSWORD]", help="optional relay, ordered farthest-to-nearest; repeat as needed", ) + parser.add_argument( + "--relay-txdelay", + type=float, + default=DEFAULT_RELAY_TX_DELAY, + help=( + "temporary flood txdelay for managed relays; rxdelay is also " + "temporarily set to 0 and both original values are restored" + ), + ) parser.add_argument( "--temp-radio", default="909.950,250,5,5,120", help="frequency,bw,sf,cr,minutes", @@ -2849,6 +3000,8 @@ def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> args.temp_values = parse_temp_radio(args.temp_radio) except argparse.ArgumentTypeError as exc: parser.error(f"--temp-radio: {exc}") + if not math.isfinite(args.relay_txdelay) or not 0.0 <= args.relay_txdelay <= 2.0: + parser.error("--relay-txdelay must be between 0 and 2") clear_manifest = args.clear_completed_manifest clear_body_hash = args.clear_completed_on_body_hash if bool(clear_manifest) != bool(clear_body_hash): @@ -2944,6 +3097,11 @@ def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> 0 if args.source_already_temp or args.source_shares_controller else 30 ) final_reply_count = 1 if args.no_install else 4 + len(args.relay) + relay_timing_seconds = ( + len(args.relay) + * RELAY_TIMING_COMMANDS_PER_RELAY + * args.reply_timeout + ) required_temp_seconds = ( remote_setup_seconds + source_setup_seconds @@ -2953,6 +3111,7 @@ def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> + args.transfer_timeout_minutes * 60 + adaptive_poll_ceiling(args.poll_seconds) + args.reply_timeout * final_reply_count + + relay_timing_seconds ) temp_seconds = args.temp_values[4] * 60 if required_temp_seconds >= temp_seconds: @@ -3039,6 +3198,7 @@ def main( source_temp_owned = False target_temp_owned = False armed_relay_values: list[tuple[str, str]] = [] + relay_timing_settings: list[RelayTimingSettings] = [] password = args.password or os.environ.get("MESHCORE_ADMIN_PASSWORD", "") try: preflight_inputs(args) @@ -3128,6 +3288,19 @@ def main( source_temp_owned = True time.sleep(TEMP_RADIO_SWITCH_DELAY_SECONDS) + for relay_name, relay_password in args.relay_values: + saved_timing = read_relay_timing( + controller, relay_name, relay_password + ) + relay_timing_settings.append(saved_timing) + recovery_path = write_relay_timing_recovery( + work_dir, relay_timing_settings + ) + print(f"[relays] timing recovery settings: {recovery_path}") + enforce_relay_timing( + controller, saved_timing, args.relay_txdelay + ) + seeder = SeederProcess(args, package_path.parent, work_dir) seeder_attempted = True seeder.start() @@ -3136,6 +3309,8 @@ def main( if args.no_install: print(f"{args.target} is ready to install; leaving the verified update staged.") + restore_relay_timings(controller, relay_timing_settings) + relay_timing_settings.clear() if args.leave_controller_radio: print( "[cleanup] preserving TempRadio windows because " @@ -3168,6 +3343,8 @@ def main( # Once the target has accepted (or may have accepted) installation, # the relays are no longer needed on TempRadio. Give each a short, # bounded window, then verify through the restored normal route. + restore_relay_timings(controller, relay_timing_settings) + relay_timing_settings.clear() shorten_relay_temp_windows(controller, args) armed_relay_values.clear() @@ -3221,10 +3398,13 @@ def main( # this is harmless on TCP and explicitly cleans the serial case. if seeder_attempted and args.source_serial: source_cli_command(args, "ota folder off", check=False) - needs_remote_cleanup = target_temp_owned or bool(armed_relay_values) + needs_remote_cleanup = ( + target_temp_owned + or bool(armed_relay_values) + or bool(relay_timing_settings) + ) if ( needs_remote_cleanup - and not args.leave_controller_radio and controller is not None and original_radio is not None and temp_radio is not None @@ -3239,12 +3419,23 @@ def main( time.sleep(TEMP_RADIO_SWITCH_DELAY_SECONDS) except (OtaError, OSError) as exc: print( - "WARNING: could not reach TempRadio to shorten remote " - f"leases; their configured windows remain bounded: {exc}", + "WARNING: could not reach TempRadio to restore relay " + "timing or shorten remote leases; TempRadio windows " + f"remain bounded: {exc}", file=sys.stderr, ) if controller_changed: - if target_temp_owned: + if relay_timing_settings: + try: + restore_relay_timings(controller, relay_timing_settings) + relay_timing_settings.clear() + except (OtaError, OSError) as exc: + print( + "CRITICAL: could not restore every relay's " + f"rxdelay/txdelay; use {RELAY_TIMING_RECOVERY_FILE}: {exc}", + file=sys.stderr, + ) + if target_temp_owned and not args.leave_controller_radio: try: shorten_target_temp_window(controller, args) target_temp_owned = False @@ -3254,7 +3445,7 @@ def main( f"TempRadio lease: {exc}", file=sys.stderr, ) - if armed_relay_values: + if armed_relay_values and not args.leave_controller_radio: try: shorten_relay_temp_windows( controller, args, armed_relay_values diff --git a/tools/lora_ota/rak3401_compact_route.json b/tools/lora_ota/rak3401_compact_route.json new file mode 100644 index 000000000..bff209c7e --- /dev/null +++ b/tools/lora_ota/rak3401_compact_route.json @@ -0,0 +1,125 @@ +{ + "app_base": "0x26000", + "endpoint_name": "halo-keymind-cascade-dev", + "endpoint_source_commit": "e742333a0ee393b0d55d2414f25b28f2f67e1ea1", + "endpoint_version": "1.17.1.02", + "schema": 1, + "search": { + "dynamic_workspace_candidates_on_shortest_dag": 2296, + "fixed_source_0_reachable_nodes": [ + 1 + ], + "fixed_source_1_reachable_nodes": [ + 2, + 3, + 4, + 5, + 6 + ], + "objective": "minimum packages, then minimum total container bytes", + "page_size": 4096, + "selected_total_bytes": 1002164, + "shorter_path_edges_unlocked": 0, + "shorter_path_pairs": 272, + "shorter_path_workspace_candidates": 9430, + "shortest_dag_edges": 61, + "shortest_package_count": 9, + "shortest_route_count": 840, + "workspace_max": "0xAD000", + "workspace_minimum": "page_align(max(base_size + 8192, target_size))" + }, + "shortest_package_count": 9, + "stage_ceiling": "0xD4000", + "steps": [ + { + "expected_container_size": 89844, + "expected_staging_margin": 0, + "expected_target_sha256": "8364257a2b3a219905e870fad6fbb2040a96ca4b4bb7201b2867534cc2b45530", + "expected_target_version": "1.16.7.9", + "inplace_memory": "0x98000", + "reuse_baseline_package": true, + "source_node": 0, + "target_node": 1 + }, + { + "expected_container_size": 46363, + "expected_staging_margin": 40960, + "expected_target_sha256": "7031e4d7f883c5d6ba6637eab730d143bd7d0f3c755f09a6c52190f762d23e45", + "expected_target_version": "1.16.7.10", + "inplace_memory": "0x98000", + "reuse_baseline_package": false, + "source_node": 1, + "target_node": 2 + }, + { + "expected_container_size": 106030, + "expected_staging_margin": 12288, + "expected_target_sha256": "2a434bc74d4c5c282cf5709dff18c5cceb1d7d69321a2692944df0d8c14d43d1", + "expected_target_version": "1.16.8.7", + "inplace_memory": "0x91000", + "reuse_baseline_package": false, + "source_node": 2, + "target_node": 7 + }, + { + "expected_container_size": 149927, + "expected_staging_margin": 0, + "expected_target_sha256": "0a514424c121febb90c7951859f0f5ccf30ffc5392c3bc3185bc7141bde651ff", + "expected_target_version": "1.16.9.105", + "inplace_memory": "0x89000", + "reuse_baseline_package": false, + "source_node": 7, + "target_node": 13 + }, + { + "expected_container_size": 111307, + "expected_staging_margin": 16384, + "expected_target_sha256": "871039ef453faf98b694b14c666773c8c6d5ae159fc7a4573f57feb66bdb60be", + "expected_target_version": "1.16.9.110", + "inplace_memory": "0x8E000", + "reuse_baseline_package": false, + "source_node": 13, + "target_node": 16 + }, + { + "expected_container_size": 70679, + "expected_staging_margin": 61440, + "expected_target_sha256": "644a8c562080838acac32f47871774f04820511e5595c64d674210aebed6b419", + "expected_target_version": "1.16.9.111", + "inplace_memory": "0x8D000", + "reuse_baseline_package": false, + "source_node": 16, + "target_node": 17 + }, + { + "expected_container_size": 143441, + "expected_staging_margin": 4096, + "expected_target_sha256": "b81d219393897fb1453594d9ea6983b2695c1b2396b768836af7da58b8576e83", + "expected_target_version": "1.16.9.116", + "inplace_memory": "0x89000", + "reuse_baseline_package": false, + "source_node": 17, + "target_node": 22 + }, + { + "expected_container_size": 98188, + "expected_staging_margin": 86016, + "expected_target_sha256": "05ac521daf941b14426360e7ff81b0329f4788b84fe7db9d55f7da58ee336597", + "expected_target_version": "1.16.9.118", + "inplace_memory": "0x81000", + "reuse_baseline_package": false, + "source_node": 22, + "target_node": 24 + }, + { + "expected_container_size": 186385, + "expected_staging_margin": 12288, + "expected_target_sha256": "2784e4b645bc3dc198de0b8b18d3d7369cd02eca61cd71c46a51b61854da5345", + "expected_target_version": "1.17.1.02", + "inplace_memory": "0x7D000", + "reuse_baseline_package": false, + "source_node": 24, + "target_node": 30 + } + ] +} diff --git a/tools/lora_ota/rak3401_dynamic_workspace_backport.patch b/tools/lora_ota/rak3401_dynamic_workspace_backport.patch new file mode 100644 index 000000000..7351bda7f --- /dev/null +++ b/tools/lora_ota/rak3401_dynamic_workspace_backport.patch @@ -0,0 +1,34 @@ +diff --git a/src/helpers/ota/OtaFlashLayout_nrf52.h b/src/helpers/ota/OtaFlashLayout_nrf52.h +--- a/src/helpers/ota/OtaFlashLayout_nrf52.h ++++ b/src/helpers/ota/OtaFlashLayout_nrf52.h +@@ -63,4 +63,3 @@ + inline uint32_t mota_nrf52_stage_capacity(uint32_t app_base) { +- return mota_nrf52_layout_valid(app_base) +- ? MOTA_NRF52_FS_START - (app_base + MOTA_NRF52_INPLACE_MEMORY) : 0; ++ return mota_nrf52_layout_valid(app_base) ? MOTA_NRF52_FS_START - app_base : 0; + } +@@ -70,18 +69,17 @@ inline uint32_t mota_nrf52_stage_capacity(uint32_t app_base) { +-// ENTIRELY above both the running image and the in-place decoder workspace, and below the filesystem +-// region (ExtraFS/InternalFS - where user prefs live, assumed immutable). Reserving the full workspace +-// here prevents accepting a large container that the bootloader would later reject as overlapping it. ++// ENTIRELY above the running image and below ExtraFS/InternalFS, where user preferences live. ++// The exact package carries its own page-aligned detools memory size; the deployed OTAFIX bootloader ++// independently bounds that workspace below the package, so the receiver can use the real EndF extent. + // Returns false (and leaves out_start untouched) if it does not fit. This is the SINGLE place the FS + // ceiling + app-collision bounds are enforced; begin()/reopen() both go through it. Pure - no flash I/O - + // so it is unit-tested natively in test/test_ota/test_ota_flashplan.cpp. + inline bool mota_nrf52_stage_plan(uint32_t total_size, uint32_t app_base, uint32_t app_end, + uint32_t& out_start) { + if (!mota_nrf52_layout_valid(app_base) || app_end < app_base || +- app_end > app_base + MOTA_NRF52_INPLACE_MEMORY) return false; +- const uint32_t workspace_end = app_base + MOTA_NRF52_INPLACE_MEMORY; +- const uint32_t capacity = MOTA_NRF52_FS_START - workspace_end; ++ app_end > MOTA_NRF52_FS_START) return false; ++ const uint32_t capacity = MOTA_NRF52_FS_START - app_base; + if (total_size < 13 || total_size > capacity) return false; // 13 = header(8)+trailer(5); must fit below FS + uint32_t start = (MOTA_NRF52_FS_START - total_size) & ~(MOTA_NRF52_FLASH_PAGE - 1); // bottom-align down +- if (start < app_end || start < workspace_end) return false; // would overlap app / decoder workspace ++ if (start < app_end) return false; // would overlap the running application + out_start = start; + return true; + } diff --git a/tools/lora_ota/rak3401_mota_chain.py b/tools/lora_ota/rak3401_mota_chain.py index 794034b40..286f283b2 100755 --- a/tools/lora_ota/rak3401_mota_chain.py +++ b/tools/lora_ota/rak3401_mota_chain.py @@ -10,6 +10,7 @@ from datetime import datetime, timezone import getpass import hashlib import json +import math import os from pathlib import Path, PurePosixPath import re @@ -28,21 +29,34 @@ except ImportError: import lora_ota as ota -RELEASE_TAG = "rak3401-mota-v1.16.07-c1caa5ad-to-v1.17.01-c96bdd6e" +RELEASE_TAG = "rak3401-mota-v1.16.07-c1caa5ad-to-v1.17.1.02-e742333a" RELEASE_URL = f"https://github.com/mikecarper/MeshCore/releases/tag/{RELEASE_TAG}" -ASSET_NAME = "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.01-8c5262c6.zip" +ASSET_NAME = "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a.zip" ASSET_URL = ( f"https://github.com/mikecarper/MeshCore/releases/download/{RELEASE_TAG}/" f"{ASSET_NAME}" ) -ASSET_SHA256 = "b2781e02460b200a7c37bfae352bad81618716e550d1d042dca8aa29bfc73c29" -CHECKSUM_LIST_SHA256 = "1f7add658ae5771451cd66a0e5c58a5e461983c1b36029ef480093ae5d5f1020" -BUNDLE_ROOT_NAME = "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.01-8c5262c6" +ASSET_SHA256 = "9f80eef191b88833bf4d2e4fea559cf5233ca53f9266ba310d447f37fa445f3a" +CHECKSUM_LIST_SHA256 = "73d96e23237896a3e342fe736be12d94087a813bf09ad609fb55330bbe586055" +BUNDLE_ROOT_NAME = "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.02-e742333a" + +# The accelerated 30-step release is the pinned reconstruction input for the +# compact chain. Keep it available for offline provenance, but do not start a +# new live run with it now that the legacy-ceiling route is nine packages. +SUPERSEDED_30_ASSET_SHA256 = ( + "b2781e02460b200a7c37bfae352bad81618716e550d1d042dca8aa29bfc73c29" +) +SUPERSEDED_30_CHECKSUM_LIST_SHA256 = ( + "1f7add658ae5771451cd66a0e5c58a5e461983c1b36029ef480093ae5d5f1020" +) +SUPERSEDED_30_ROOT_NAME = ( + "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.01-8c5262c6" +) # This exact 29-step release completed its full direct physical-board test. It # is now superseded because its historical targets echo terminal OTA bulk # packets back into mesh dispatch. Keep it available for offline provenance, -# but require the rebuilt 30-step chain for any new live run. +# but require the compact 9-step chain for any new live run. SUPERSEDED_29_ASSET_SHA256 = ( "eac67a0be12690b7e22c4d1f6a15bfdeb5bd627c4850b246b1be4220e5607b34" ) @@ -107,12 +121,14 @@ DEFAULT_TARGET_KEY = ( ) EXPECTED_TARGET_ID = 0x2FA509C1 EXPECTED_HARDWARE = "RAK_3401" -EXPECTED_STEP_COUNT = 30 +EXPECTED_STEP_COUNT = 9 EXPECTED_START_VERSION = "1.16.7.0" -EXPECTED_FINAL_VERSION = "1.17.1.0" +EXPECTED_FINAL_VERSION = "1.17.1.02" +SUPERSEDED_FINAL_VERSION = "1.17.1.0" MIN_MESHCLI_VERSION = (1, 6, 0) WATCHDOG_RESET_WAIT_SECONDS = 90 WATCHDOG_STABILITY_WAIT_SECONDS = 90 +TRANSFER_SETTINGS_FILE = "target-transfer-settings.json" # Physical RAK3401 testing proved two retired packages unsafe. The original # v1.17.02 chain failed at step 6. Its software-SHA replacement then passed @@ -120,9 +136,9 @@ WATCHDOG_STABILITY_WAIT_SECONDS = 90 # that retained 386ae4a5 bridge still used unchecked CC310 SHA. Keep both # bundles useful for offline diagnosis, but fail closed before a live # connection. A prior exact 29-step replacement passed its complete direct -# physical-board run on 14-Aug-2026. The pinned 30-step release is rebuilt and -# exhaustively verified offline, but still requires an explicit candidate -# acknowledgement until that exact byte sequence completes a physical run. +# physical-board run on 14-Aug-2026. The compact 9-step release is exhaustively +# verified offline against the deployed bootloader and completed its exact +# nine-package physical RAK3401 run on 19-Aug-2026. KNOWN_UNSAFE_STEP = 6 KNOWN_UNSAFE_VERSION = "1.16.8.7" KNOWN_UNSAFE_IMAGE_SHA256 = ( @@ -171,6 +187,21 @@ SUPERSEDED_29_ANCHORS = ( (29, "5c8d94bb23e87c23b0374ffb5a46e0c1205d6eebcb9d8bae3be6fda2613f9f79"), ) +# Exact anchors for the compact nine-package release. The first target and +# package are byte-identical to the physically passed baseline; every later +# target is pinned here in addition to the outer and inner bundle checksums. +COMPACT_RELEASE_ANCHORS = ( + (1, "8364257a2b3a219905e870fad6fbb2040a96ca4b4bb7201b2867534cc2b45530"), + (2, "7031e4d7f883c5d6ba6637eab730d143bd7d0f3c755f09a6c52190f762d23e45"), + (3, "2a434bc74d4c5c282cf5709dff18c5cceb1d7d69321a2692944df0d8c14d43d1"), + (4, "0a514424c121febb90c7951859f0f5ccf30ffc5392c3bc3185bc7141bde651ff"), + (5, "871039ef453faf98b694b14c666773c8c6d5ae159fc7a4573f57feb66bdb60be"), + (6, "644a8c562080838acac32f47871774f04820511e5595c64d674210aebed6b419"), + (7, "b81d219393897fb1453594d9ea6983b2695c1b2396b768836af7da58b8576e83"), + (8, "05ac521daf941b14426360e7ff81b0329f4788b84fe7db9d55f7da58ee336597"), + (9, "2784e4b645bc3dc198de0b8b18d3d7369cd02eca61cd71c46a51b61854da5345"), +) + # Exact anchors for every image in the accelerated 30-step release. The outer # asset and inner checksum-list pins cover every byte as well; the complete # anchor table keeps structural validation fail-closed if those layers are @@ -234,22 +265,20 @@ SUPERSEDED_27_MESSAGE = ( "live installation of the superseded 27-step candidate is disabled: its " "physical test was stopped after step 2 so the adaptive primary requester " "could be moved into every historical bridge. Use --verify-only for that " - "bundle and use the pinned 30-step release." + "bundle and use the pinned compact release." ) SUPERSEDED_29_MESSAGE = ( "live installation of the superseded 29-step release is disabled: that " "exact chain passed its direct physical run, but its historical targets " "do not consume terminal OTA bulk packets before mesh dispatch. Use " - "--verify-only for provenance and deploy the accelerated 30-step release." + "--verify-only for provenance and deploy the compact release." ) -CANDIDATE_ACK_MESSAGE = ( - "live installation of the accelerated 30-step release requires " - "--accept-test-candidate: all 30 transitions passed independent offline " - "reconstruction and bootloader simulation, but this exact byte sequence " - "has not yet completed a full physical-chain run" +SUPERSEDED_30_MESSAGE = ( + "live installation of the superseded 30-step release is disabled: its " + "images remain the pinned reconstruction input for the compact chain, " + "which reaches the requested endpoint in nine packages without changing " + "the deployed bootloader. Use --verify-only for the older bundle." ) - - class KnownUnsafeReleaseError(ota.OtaError): """The pinned artifacts are intact but their live transition is unsafe.""" @@ -259,16 +288,25 @@ def require_live_release_safe( steps: list[ChainStep], ) -> None: if len(steps) == EXPECTED_STEP_COUNT: - for number, expected_sha256 in PINNED_RELEASE_ANCHORS: + for number, expected_sha256 in COMPACT_RELEASE_ANCHORS: if steps[number - 1].target_sha256 != expected_sha256: raise KnownUnsafeReleaseError( - "live installation is disabled: the 30-step release has " + "live installation is disabled: the compact release has " f"an unrecognized step-{number} image" ) - if not getattr(args, "accept_test_candidate", False): - raise KnownUnsafeReleaseError(CANDIDATE_ACK_MESSAGE) return + if len(steps) == 30: + if all( + steps[number - 1].target_sha256 == expected_sha256 + for number, expected_sha256 in PINNED_RELEASE_ANCHORS + ): + raise KnownUnsafeReleaseError(SUPERSEDED_30_MESSAGE) + raise KnownUnsafeReleaseError( + "live installation is disabled: this is an unrecognized variant " + "of the superseded 30-step release" + ) + if len(steps) == 29: if all( steps[number - 1].target_sha256 == expected_sha256 @@ -339,6 +377,15 @@ class ChainStep: package: ota.MotaInfo +@dataclass(frozen=True) +class TargetTransferSettings: + rxps_enabled: bool + rxps_rx_us: int + rxps_sleep_us: int + powersaving_enabled: bool + rxdelay: str + + def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as source: @@ -402,6 +449,7 @@ def download_release_asset(destination: Path) -> None: def extract_bundle(archive_path: Path, destination: Path) -> Path: root_names = ( BUNDLE_ROOT_NAME, + SUPERSEDED_30_ROOT_NAME, SUPERSEDED_29_ROOT_NAME, KNOWN_FAILED_V11701_ROOT_NAME, KNOWN_UNSAFE_ROOT_NAME, @@ -455,6 +503,7 @@ def locate_bundle(args: argparse.Namespace, work_dir: Path) -> Path: nested_roots = [ supplied / name for name in ( BUNDLE_ROOT_NAME, + SUPERSEDED_30_ROOT_NAME, SUPERSEDED_29_ROOT_NAME, KNOWN_FAILED_V11701_ROOT_NAME, KNOWN_UNSAFE_ROOT_NAME, @@ -471,6 +520,7 @@ def locate_bundle(args: argparse.Namespace, work_dir: Path) -> Path: actual = sha256_file(supplied) if actual not in { ASSET_SHA256, + SUPERSEDED_30_ASSET_SHA256, SUPERSEDED_29_ASSET_SHA256, SUPERSEDED_27_ASSET_SHA256, KNOWN_FAILED_V11701_ASSET_SHA256, @@ -490,6 +540,7 @@ def verify_checksum_list(bundle_root: Path) -> None: checksum_digest = sha256_file(checksum_path) expected_lists = { CHECKSUM_LIST_SHA256, + SUPERSEDED_30_CHECKSUM_LIST_SHA256, SUPERSEDED_29_CHECKSUM_LIST_SHA256, SUPERSEDED_27_CHECKSUM_LIST_SHA256, KNOWN_FAILED_V11701_CHECKSUM_LIST_SHA256, @@ -547,10 +598,10 @@ def parse_chain(bundle_root: Path) -> tuple[list[ChainStep], bytes]: rows = list(csv.DictReader(source)) except OSError as exc: raise ota.OtaError(f"cannot read {chain_path}: {exc}") from exc - if len(rows) not in (26, 27, 29, EXPECTED_STEP_COUNT): + if len(rows) not in (EXPECTED_STEP_COUNT, 26, 27, 29, 30): raise ota.OtaError( - f"chain contains {len(rows)} steps, expected 26, 27, 29, or " - f"{EXPECTED_STEP_COUNT}" + f"chain contains {len(rows)} steps, expected {EXPECTED_STEP_COUNT}, " + "26, 27, 29, or 30" ) steps: list[ChainStep] = [] @@ -601,19 +652,26 @@ def parse_chain(bundle_root: Path) -> tuple[list[ChainStep], bytes]: if steps[0].from_version != EXPECTED_START_VERSION: raise ota.OtaError("chain has an unexpected starting version") if len(steps) == EXPECTED_STEP_COUNT: + for number, expected_sha256 in COMPACT_RELEASE_ANCHORS: + if steps[number - 1].target_sha256 != expected_sha256: + raise ota.OtaError( + f"compact release step {number} does not match its audited image pin" + ) + expected_final_version = EXPECTED_FINAL_VERSION + elif len(steps) == 30: for number, expected_sha256 in PINNED_RELEASE_ANCHORS: if steps[number - 1].target_sha256 != expected_sha256: raise ota.OtaError( - f"pinned release step {number} does not match its audited image pin" + f"superseded 30-step release step {number} does not match its image pin" ) - expected_final_version = EXPECTED_FINAL_VERSION + expected_final_version = SUPERSEDED_FINAL_VERSION elif len(steps) == 29: for number, expected_sha256 in SUPERSEDED_29_ANCHORS: if steps[number - 1].target_sha256 != expected_sha256: raise ota.OtaError( f"superseded step {number} does not match its audited image pin" ) - expected_final_version = EXPECTED_FINAL_VERSION + expected_final_version = SUPERSEDED_FINAL_VERSION else: step6 = steps[KNOWN_UNSAFE_STEP - 1] recognized_step6_hashes = { @@ -650,7 +708,7 @@ def parse_chain(bundle_root: Path) -> tuple[list[ChainStep], bytes]: expected_final_version = ( "1.17.2.0" if step6.target_sha256 == KNOWN_UNSAFE_IMAGE_SHA256 - else EXPECTED_FINAL_VERSION + else SUPERSEDED_FINAL_VERSION ) if steps[-1].to_version != expected_final_version: raise ota.OtaError( @@ -837,6 +895,177 @@ def require_watchdog_state( return reply +def read_target_transfer_settings( + controller: ota.Controller, + target_name: str, +) -> TargetTransferSettings: + rxps_reply = controller.remote_command(target_name, "get radio.rxps") + rxps_match = re.fullmatch( + r"\s*>\s*(on|off),(\d+),(\d+)\s*", + rxps_reply, + re.IGNORECASE, + ) + if rxps_match is None: + raise ota.OtaError(f"could not read destination RXPS state: {rxps_reply}") + + powersaving_reply = controller.remote_command(target_name, "powersaving") + powersaving_match = re.fullmatch( + r"\s*>?\s*(on|off)\s*", + powersaving_reply, + re.IGNORECASE, + ) + if powersaving_match is None: + raise ota.OtaError( + f"could not read destination CPU power-saving state: {powersaving_reply}" + ) + + rxdelay_reply = controller.remote_command(target_name, "get rxdelay") + rxdelay_match = re.fullmatch( + r"\s*>\s*([0-9]+(?:\.[0-9]+)?)\s*", + rxdelay_reply, + ) + if rxdelay_match is None: + raise ota.OtaError(f"could not read destination RX delay: {rxdelay_reply}") + + return TargetTransferSettings( + rxps_enabled=rxps_match.group(1).lower() == "on", + rxps_rx_us=int(rxps_match.group(2)), + rxps_sleep_us=int(rxps_match.group(3)), + powersaving_enabled=powersaving_match.group(1).lower() == "on", + rxdelay=rxdelay_match.group(1), + ) + + +def load_or_capture_transfer_settings( + controller: ota.Controller, + target_name: str, + target_key: str, + work_dir: Path, +) -> TargetTransferSettings: + path = work_dir / TRANSFER_SETTINGS_FILE + if path.exists(): + try: + saved = json.loads(path.read_text(encoding="ascii")) + if saved["target_key"].lower() != target_key.lower(): + raise ota.OtaError( + f"{path} belongs to a different destination public key" + ) + if not isinstance(saved["rxps_enabled"], bool) or not isinstance( + saved["powersaving_enabled"], bool + ): + raise TypeError("saved power states must be JSON booleans") + settings = TargetTransferSettings( + rxps_enabled=saved["rxps_enabled"], + rxps_rx_us=int(saved["rxps_rx_us"]), + rxps_sleep_us=int(saved["rxps_sleep_us"]), + powersaving_enabled=saved["powersaving_enabled"], + rxdelay=str(saved["rxdelay"]), + ) + float(settings.rxdelay) + except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: + raise ota.OtaError(f"invalid saved transfer settings in {path}") from exc + print(f"[guardrail] loaded original destination settings from {path}") + return settings + + settings = read_target_transfer_settings(controller, target_name) + payload = { + "target_key": target_key.lower(), + "rxps_enabled": settings.rxps_enabled, + "rxps_rx_us": settings.rxps_rx_us, + "rxps_sleep_us": settings.rxps_sleep_us, + "powersaving_enabled": settings.powersaving_enabled, + "rxdelay": settings.rxdelay, + } + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="ascii") + temporary.chmod(0o600) + os.replace(temporary, path) + print(f"[guardrail] saved original destination settings to {path}") + return settings + + +def enforce_transfer_guardrails( + controller: ota.Controller, + target_name: str, +) -> None: + current = read_target_transfer_settings(controller, target_name) + if current.powersaving_enabled: + reply = controller.remote_command(target_name, "powersaving off") + if re.search(r"\boff\b", reply, re.IGNORECASE) is None: + raise ota.OtaError(f"target did not disable CPU power saving: {reply}") + if current.rxps_enabled: + reply = controller.remote_command(target_name, "set radio.rxps off") + if re.search(r"\boff\b", reply, re.IGNORECASE) is None: + raise ota.OtaError(f"target did not disable RXPS: {reply}") + if abs(float(current.rxdelay)) > 0.0001: + reply = controller.remote_command(target_name, "set rxdelay 0") + if not reply.upper().startswith("OK"): + raise ota.OtaError(f"target did not disable RX flood delay: {reply}") + + verified = read_target_transfer_settings(controller, target_name) + if ( + verified.powersaving_enabled + or verified.rxps_enabled + or abs(float(verified.rxdelay)) > 0.0001 + ): + raise ota.OtaError( + "destination transfer guardrails did not read back as " + "powersaving=off, RXPS=off, rxdelay=0" + ) + print("[guardrail] destination verified: RXPS off, rxdelay 0, CPU power saving off") + + +def restore_transfer_settings( + controller: ota.Controller, + target_name: str, + saved: TargetTransferSettings, +) -> None: + current = read_target_transfer_settings(controller, target_name) + if abs(float(current.rxdelay) - float(saved.rxdelay)) > 0.0001: + reply = controller.remote_command(target_name, f"set rxdelay {saved.rxdelay}") + if not reply.upper().startswith("OK"): + raise ota.OtaError(f"target did not restore RX flood delay: {reply}") + + if saved.rxps_enabled: + if ( + not current.rxps_enabled + or current.rxps_rx_us != saved.rxps_rx_us + or current.rxps_sleep_us != saved.rxps_sleep_us + ): + reply = controller.remote_command( + target_name, + f"set radio.rxps {saved.rxps_rx_us} {saved.rxps_sleep_us}", + ) + if re.search(r"\bon\b", reply, re.IGNORECASE) is None: + raise ota.OtaError(f"target did not restore RXPS: {reply}") + elif current.rxps_enabled: + reply = controller.remote_command(target_name, "set radio.rxps off") + if re.search(r"\boff\b", reply, re.IGNORECASE) is None: + raise ota.OtaError(f"target did not restore RXPS-off state: {reply}") + + verified = read_target_transfer_settings(controller, target_name) + if ( + verified.rxps_enabled != saved.rxps_enabled + or abs(float(verified.rxdelay) - float(saved.rxdelay)) > 0.0001 + or ( + saved.rxps_enabled + and ( + verified.rxps_rx_us != saved.rxps_rx_us + or verified.rxps_sleep_us != saved.rxps_sleep_us + ) + ) + ): + raise ota.OtaError("destination radio transfer settings did not restore exactly") + + if saved.powersaving_enabled: + reply = controller.remote_command(target_name, "powersaving on") + if re.search(r"\bon\b", reply, re.IGNORECASE) is None: + raise ota.OtaError(f"target did not restore CPU power saving: {reply}") + elif verified.powersaving_enabled: + raise ota.OtaError("destination CPU power saving unexpectedly remained enabled") + print("[guardrail] original destination transfer settings restored") + + def read_ota_hops(controller: ota.Controller, target_name: str) -> int: reply = controller.remote_command(target_name, "ota config") match = re.search(r"\bhops=(\d+)\b", reply) @@ -1021,6 +1250,7 @@ def run_step( *connection_arguments(args), "--controller-baud", str(args.controller_baud), "--source-baud", str(args.source_baud), + "--relay-txdelay", str(args.relay_txdelay), "--temp-radio", args.temp_radio, "--meshcli", args.meshcli, "--motatool", args.motatool, @@ -1055,8 +1285,8 @@ def run_step( def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description=( - "Download, verify, resume, and install the exact accelerated " - "30-step RAK3401 mOTA chain from c1caa5ad to 8c5262c6." + "Download, verify, resume, and install the exact compact 9-step " + "RAK3401 mOTA chain from c1caa5ad to e742333a." ) ) parser.add_argument( @@ -1074,11 +1304,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--accept-test-candidate", action="store_true", - help=( - "acknowledge that the exact accelerated 30-step byte sequence is " - "offline-validated but awaits its complete physical-chain run; this " - "never overrides a failed or superseded bundle" - ), + help=argparse.SUPPRESS, ) controller = parser.add_mutually_exclusive_group() @@ -1100,6 +1326,12 @@ def build_parser() -> argparse.ArgumentParser: metavar="NAME[=PASSWORD]", help="intermediate relay, farthest-to-nearest; repeat for each relay", ) + parser.add_argument( + "--relay-txdelay", + type=float, + default=ota.DEFAULT_RELAY_TX_DELAY, + help="temporary flood txdelay for managed intermediate relays", + ) parser.add_argument("--temp-radio", default="909.950,250,5,5,120") parser.add_argument( "--ota-hops", @@ -1179,6 +1411,8 @@ def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> ): if getattr(args, name) <= 0: parser.error(f"--{name.replace('_', '-')} must be positive") + if not math.isfinite(args.relay_txdelay) or not 0.0 <= args.relay_txdelay <= 2.0: + parser.error("--relay-txdelay must be between 0 and 2") if not 0 <= args.ota_hops <= 8: parser.error("--ota-hops must be from 0 through 8") try: @@ -1195,6 +1429,7 @@ def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> + args.transfer_timeout_minutes * 60 + ota.adaptive_poll_ceiling(args.poll_seconds) + args.reply_timeout * (4 + len(args.relay)) + + len(args.relay) * ota.RELAY_TIMING_COMMANDS_PER_RELAY * args.reply_timeout ) if not args.verify_only and temp_values[4] * 60 <= required_seconds: parser.error( @@ -1229,7 +1464,13 @@ def confirm_chain( ) print(f" TempRadio : {args.temp_radio}") print(f" relays : {len(args.relay)}") + if args.relay: + print( + f" relay timing: rxdelay 0, txdelay " + f"{ota.format_decimal(args.relay_txdelay)} (saved/restored)" + ) print(f" OTA reach : enforce {args.ota_hops} hops before every step") + print(" guardrails : save, disable RXPS/rxdelay/CPU sleep, restore at endpoint") print(" watchdog : disable, prove stable, gate every install, then re-enable") if args.preflight_only: print("Live preflight passed; no radio or watchdog settings were changed.") @@ -1257,7 +1498,16 @@ def main(argv: list[str] | None = None) -> int: verify_motatool(args, steps) if args.verify_only: print(f"Verified release bundle: {bundle_root}") - if len(steps) == 29: + if len(steps) == EXPECTED_STEP_COUNT: + print( + "Compact release verified: all 9 transitions passed its exact " + "physical RAK3401 run, zero-filled and erased-workspace " + "reconstruction, independent motatool verification, and " + "deployed/current bootloader simulation." + ) + elif len(steps) == 30: + print(f"WARNING: {SUPERSEDED_30_MESSAGE}", file=sys.stderr) + elif len(steps) == 29: print(f"WARNING: {SUPERSEDED_29_MESSAGE}", file=sys.stderr) elif steps[KNOWN_UNSAFE_STEP - 1].target_sha256 == KNOWN_UNSAFE_IMAGE_SHA256: print(f"WARNING: {KNOWN_UNSAFE_RELEASE_MESSAGE}", file=sys.stderr) @@ -1279,12 +1529,6 @@ def main(argv: list[str] | None = None) -> int: and steps[-1].target_sha256 == SUPERSEDED_27_FINAL_IMAGE_SHA256 ): print(f"WARNING: {SUPERSEDED_27_MESSAGE}", file=sys.stderr) - else: - print( - "Accelerated release verified offline: all 30 transitions " - "passed zero-filled and erased-workspace reconstruction, " - "independent motatool verification, and bootloader simulation." - ) return 0 # This must precede meshcli, password handling, source preflight, and @@ -1314,6 +1558,14 @@ def main(argv: list[str] | None = None) -> int: if first_index == len(steps): enforce_ota_hops(controller, target_name, args.ota_hops) + transfer_path = work_dir / TRANSFER_SETTINGS_FILE + if transfer_path.exists(): + transfer_settings = load_or_capture_transfer_settings( + controller, target_name, full_key, work_dir + ) + restore_transfer_settings( + controller, target_name, transfer_settings + ) if not args.keep_watchdog_off: enabled = controller.remote_command(target_name, "set system.watchdog on") if not enabled.lower().startswith("ok - system watchdog enabled"): @@ -1322,7 +1574,11 @@ def main(argv: list[str] | None = None) -> int: print("RAK3401 already matches the verified final endpoint.") return 0 + transfer_settings = load_or_capture_transfer_settings( + controller, target_name, full_key, work_dir + ) prepare_watchdog(controller, target_name) + enforce_transfer_guardrails(controller, target_name) enforce_ota_hops(controller, target_name, args.ota_hops) for index in range(first_index, len(steps)): step = steps[index] @@ -1339,6 +1595,7 @@ def main(argv: list[str] | None = None) -> int: f"live target is at chain index {current_index}, expected {index}" ) require_watchdog_state(controller, target_name, "off") + enforce_transfer_guardrails(controller, target_name) enforce_ota_hops(controller, target_name, args.ota_hops) print( f"\n[chain] step {step.number:02d}/{len(steps)}: " @@ -1385,6 +1642,7 @@ def main(argv: list[str] | None = None) -> int: final_target = query_live_target(controller, args, target_name) if find_resume_index(final_target, steps, final_body_hash) != len(steps): raise ota.OtaError("final target identity did not match the release endpoint") + restore_transfer_settings(controller, target_name, transfer_settings) if not args.keep_watchdog_off: enabled = controller.remote_command(target_name, "set system.watchdog on") if not enabled.lower().startswith("ok - system watchdog enabled"): @@ -1403,15 +1661,16 @@ def main(argv: list[str] | None = None) -> int: except KeyboardInterrupt: print( "\nInterrupted. Any partial mOTA remains resumable. The target watchdog " - "may intentionally remain off until the chain completes.", + "and transfer guardrails may intentionally remain off until the chain completes.", file=sys.stderr, ) return 130 except (ota.OtaError, OSError, zipfile.BadZipFile) as exc: print(f"ERROR: {exc}", file=sys.stderr) print( - "The target watchdog may intentionally remain off. Rerun the same command " - "to resume from the live version; do not skip a chain step.", + "The target watchdog and transfer guardrails may intentionally remain off. " + "Rerun the same command and work directory to resume and restore the saved " + "settings; do not skip a chain step.", file=sys.stderr, ) return 2 diff --git a/tools/lora_ota/test_lora_ota.py b/tools/lora_ota/test_lora_ota.py index 760028056..c26acc3e6 100644 --- a/tools/lora_ota/test_lora_ota.py +++ b/tools/lora_ota/test_lora_ota.py @@ -7,6 +7,7 @@ import argparse import contextlib import hashlib import io +import json import os from pathlib import Path import struct @@ -337,6 +338,24 @@ class SourceCliTests(unittest.TestCase): ) self.assertIn("OK - temp params", output) + def test_serial_companion_seeder_enters_terminal_mode(self) -> None: + args = argparse.Namespace( + motatool="motatool", + source_serial="/dev/source", + source_tcp=None, + source_baud=115200, + source_companion_terminal=True, + ) + seeder = ota.SeederProcess(args, Path("/served"), Path("/work")) + self.assertEqual( + seeder.command, + [ + "motatool", "serve", "--dir", "/served", "-v", + "--serial", "/dev/source", "--baud", "115200", + "--companion-terminal", + ], + ) + class CompatibilityTests(unittest.TestCase): def setUp(self) -> None: @@ -1434,6 +1453,79 @@ class ReliabilityTests(unittest.TestCase): ], ) + def test_managed_relay_timing_is_saved_guarded_and_restored(self) -> None: + class Controller: + def __init__(self) -> None: + self.rxdelay = 2.0 + self.txdelay = 0.5 + + def remote_command( + self, _target: str, command: str, **_kwargs: object + ) -> str: + if command == "get rxdelay": + return f"> {self.rxdelay}" + if command == "get txdelay": + return f"> {self.txdelay}" + if command.startswith("set rxdelay "): + self.rxdelay = float(command.split()[-1]) + return "OK" + if command.startswith("set txdelay "): + self.txdelay = float(command.split()[-1]) + return "OK" + raise AssertionError(command) + + controller = Controller() + saved = ota.read_relay_timing(controller, "relay", "secret") + ota.enforce_relay_timing(controller, saved, 0.3) + self.assertEqual(controller.rxdelay, 0.0) + self.assertEqual(controller.txdelay, 0.3) + + with tempfile.TemporaryDirectory() as directory: + path = ota.write_relay_timing_recovery(Path(directory), [saved]) + payload = json.loads(path.read_text(encoding="ascii")) + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + self.assertEqual( + payload, + [{"name": "relay", "rxdelay": 2.0, "txdelay": 0.5}], + ) + + ota.restore_relay_timings(controller, [saved]) + self.assertEqual(controller.rxdelay, 2.0) + self.assertEqual(controller.txdelay, 0.5) + + def test_unreachable_relay_does_not_block_other_relay_restoration(self) -> None: + class Controller: + def __init__(self) -> None: + self.rxdelay = 0.0 + self.txdelay = 0.3 + + def remote_command( + self, target: str, command: str, **_kwargs: object + ) -> str: + if target == "offline": + raise ota.OtaError("unreachable") + if command == "get rxdelay": + return f"> {self.rxdelay}" + if command == "get txdelay": + return f"> {self.txdelay}" + if command.startswith("set rxdelay "): + self.rxdelay = float(command.split()[-1]) + return "OK" + if command.startswith("set txdelay "): + self.txdelay = float(command.split()[-1]) + return "OK" + raise AssertionError(command) + + controller = Controller() + settings = [ + ota.RelayTimingSettings("online", "secret", 2.0, 0.5), + ota.RelayTimingSettings("offline", "secret", 1.0, 0.4), + ] + with self.assertRaisesRegex(ota.OtaError, "offline: unreachable"): + ota.restore_relay_timings(controller, settings) + self.assertEqual(controller.rxdelay, 2.0) + self.assertEqual(controller.txdelay, 0.5) + def test_source_cleanup_only_changes_a_script_owned_window(self) -> None: args = argparse.Namespace( source_already_temp=False, @@ -1472,6 +1564,104 @@ class ReliabilityTests(unittest.TestCase): sleep.assert_called_once_with(ota.TEMP_RADIO_SWITCH_DELAY_SECONDS) +class Rak3401TransferGuardrailTests(unittest.TestCase): + class Controller: + def __init__(self) -> None: + self.rxps_enabled = True + self.rxps_rx_us = 65625 + self.rxps_sleep_us = 60000 + self.powersaving_enabled = True + self.rxdelay = "2.0" + self.commands: list[str] = [] + + def remote_command(self, _target: str, command: str) -> str: + self.commands.append(command) + if command == "get radio.rxps": + state = "on" if self.rxps_enabled else "off" + return f"> {state},{self.rxps_rx_us},{self.rxps_sleep_us}" + if command == "set radio.rxps off": + self.rxps_enabled = False + return f"OK - off,{self.rxps_rx_us},{self.rxps_sleep_us}" + if command.startswith("set radio.rxps "): + _set, _key, rx_us, sleep_us = command.split() + self.rxps_enabled = True + self.rxps_rx_us = int(rx_us) + self.rxps_sleep_us = int(sleep_us) + return f"OK - on,{rx_us},{sleep_us}" + if command == "powersaving": + return "on" if self.powersaving_enabled else "off" + if command == "powersaving off": + self.powersaving_enabled = False + return "off" + if command == "powersaving on": + self.powersaving_enabled = True + return "on - Immediate effect" + if command == "get rxdelay": + return f"> {self.rxdelay}" + if command.startswith("set rxdelay "): + self.rxdelay = command.split(maxsplit=2)[2] + return "OK" + raise AssertionError(command) + + def test_guardrails_disable_and_restore_all_original_settings(self) -> None: + controller = self.Controller() + saved = rak_chain.read_target_transfer_settings(controller, "remote") + + rak_chain.enforce_transfer_guardrails(controller, "remote") + self.assertFalse(controller.rxps_enabled) + self.assertFalse(controller.powersaving_enabled) + self.assertEqual(controller.rxdelay, "0") + + rak_chain.restore_transfer_settings(controller, "remote", saved) + self.assertTrue(controller.rxps_enabled) + self.assertEqual(controller.rxps_rx_us, 65625) + self.assertEqual(controller.rxps_sleep_us, 60000) + self.assertTrue(controller.powersaving_enabled) + self.assertEqual(controller.rxdelay, "2.0") + self.assertEqual(controller.commands[-1], "powersaving on") + + def test_original_settings_are_persisted_for_resume(self) -> None: + controller = self.Controller() + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + saved = rak_chain.load_or_capture_transfer_settings( + controller, "remote", "AA" * 32, work_dir + ) + command_count = len(controller.commands) + controller.rxdelay = "0" + loaded = rak_chain.load_or_capture_transfer_settings( + controller, "remote", "AA" * 32, work_dir + ) + mode = (work_dir / rak_chain.TRANSFER_SETTINGS_FILE).stat().st_mode + + self.assertEqual(loaded, saved) + self.assertEqual(len(controller.commands), command_count) + self.assertEqual(mode & 0o777, 0o600) + + def test_saved_settings_reject_string_booleans(self) -> None: + controller = self.Controller() + with tempfile.TemporaryDirectory() as directory: + work_dir = Path(directory) + path = work_dir / rak_chain.TRANSFER_SETTINGS_FILE + path.write_text( + json.dumps( + { + "target_key": "aa" * 32, + "rxps_enabled": "false", + "rxps_rx_us": 65625, + "rxps_sleep_us": 60000, + "powersaving_enabled": False, + "rxdelay": "2.0", + } + ), + encoding="ascii", + ) + with self.assertRaisesRegex(ota.OtaError, "invalid saved transfer settings"): + rak_chain.load_or_capture_transfer_settings( + controller, "remote", "AA" * 32, work_dir + ) + + class Rak3401KnownUnsafeReleaseTests(unittest.TestCase): def test_chain_resume_uses_exact_body_hash_not_runtime_label(self) -> None: steps = [ @@ -1618,29 +1808,34 @@ class Rak3401KnownUnsafeReleaseTests(unittest.TestCase): argparse.Namespace(accept_test_candidate=True), steps ) - def test_accelerated_30_step_release_requires_candidate_ack(self) -> None: - steps = [mock.Mock(target_sha256="") for _ in range(30)] - for number, image_sha256 in rak_chain.PINNED_RELEASE_ANCHORS: + def test_physically_passed_compact_9_step_release_needs_no_lab_ack(self) -> None: + steps = [mock.Mock(target_sha256="") for _ in range(9)] + for number, image_sha256 in rak_chain.COMPACT_RELEASE_ANCHORS: steps[number - 1].target_sha256 = image_sha256 - with self.assertRaisesRegex( - rak_chain.KnownUnsafeReleaseError, - "requires --accept-test-candidate", - ): - rak_chain.require_live_release_safe( - argparse.Namespace(accept_test_candidate=False), steps - ) rak_chain.require_live_release_safe( - argparse.Namespace(accept_test_candidate=True), steps + argparse.Namespace(accept_test_candidate=False), steps ) - def test_accelerated_30_step_release_rejects_changed_anchor(self) -> None: + def test_compact_9_step_release_rejects_changed_anchor(self) -> None: + steps = [mock.Mock(target_sha256="") for _ in range(9)] + for number, image_sha256 in rak_chain.COMPACT_RELEASE_ANCHORS: + steps[number - 1].target_sha256 = image_sha256 + steps[5].target_sha256 = "00" * 32 + with self.assertRaisesRegex( + rak_chain.KnownUnsafeReleaseError, + "unrecognized step-6 image", + ): + rak_chain.require_live_release_safe( + argparse.Namespace(accept_test_candidate=True), steps + ) + + def test_superseded_30_step_release_is_blocked(self) -> None: steps = [mock.Mock(target_sha256="") for _ in range(30)] for number, image_sha256 in rak_chain.PINNED_RELEASE_ANCHORS: steps[number - 1].target_sha256 = image_sha256 - steps[10].target_sha256 = "00" * 32 with self.assertRaisesRegex( rak_chain.KnownUnsafeReleaseError, - "unrecognized step-11 image", + "superseded 30-step release", ): rak_chain.require_live_release_safe( argparse.Namespace(accept_test_candidate=True), steps diff --git a/tools/mota/pio_endf.py b/tools/mota/pio_endf.py index 507028ee9..1148d35fc 100644 --- a/tools/mota/pio_endf.py +++ b/tools/mota/pio_endf.py @@ -137,8 +137,10 @@ def _firmware_ident(): ver_s = (_cppdef("FIRMWARE_VERSION") or "").replace("\\", "").strip().strip('"').strip("'") if not ver_s: # not a -D -> read the header MeshCore ships ver_s = _version_from_headers() - m = re.search(r"(\d+)\.(\d+)(?:\.(\d+))?", ver_s) - fw_version = ml.pack_version(f"{m.group(1)}.{m.group(2)}.{m.group(3) or 0}") if m else 0 + m = re.search(r"(\d+)\.(\d+)(?:\.(\d+))?(?:\.(\d+))?", ver_s) + fw_version = ml.pack_version( + f"{m.group(1)}.{m.group(2)}.{m.group(3) or 0}.{m.group(4) or 0}" + ) if m else 0 return ml.FwIdent(fw_version=fw_version, target_id=target_id, hw_id=hw_id)