From e2da2701dab8a9c6b78fc27dcd520eb5cd5e526c Mon Sep 17 00:00:00 2001 From: mikecarper Date: Thu, 3 Sep 2026 01:07:06 -0700 Subject: [PATCH] Support 2 KiB DEFLATE OTA transport --- docs/lora_ota_automation.md | 9 +- docs/ota_protocol.md | 80 +- docs/ota_user_guide.md | 6 +- docs/rak3401_mota_chain.md | 123 ++- src/helpers/ota/MotaContainer.cpp | 4 +- src/helpers/ota/OtaCli.cpp | 30 +- src/helpers/ota/OtaContext.h | 2 +- src/helpers/ota/OtaFormat.h | 6 +- src/helpers/ota/OtaManager.cpp | 36 +- src/helpers/ota/OtaManager.h | 23 +- src/helpers/ota/OtaProtocol.h | 81 +- src/helpers/ota/OtaSelf.cpp | 7 +- src/helpers/ota/OtaVerify.cpp | 2 +- test/test_ota/mota_vectors.h | 11 + test/test_ota/test_ota_core.cpp | 553 +++++++++- tools/ble_mota/ble_mota_seeder.py | 6 +- tools/ble_mota/test_ble_mota_seeder.py | 24 +- .../lora_ota/build_rak3401_compact_bundle.py | 962 ++++++++++++++++-- .../lora_ota/build_rak3401_inflate_bridges.py | 105 +- tools/lora_ota/lora_ota.py | 46 +- .../rak3401_inflate_assets/OtaDeflate.cpp | 18 +- .../rak3401_inflate_assets/OtaDeflate.h | 9 +- tools/lora_ota/rak3401_mota_chain.py | 142 ++- tools/lora_ota/rak3401_route_search.py | 333 +++++- .../test_build_rak3401_compact_bundle.py | 562 +++++++++- .../test_build_rak3401_inflate_bridges.py | 360 ++++++- tools/lora_ota/test_lora_ota.py | 275 ++++- tools/lora_ota/test_rak3401_route_search.py | 336 +++++- tools/mota/README.md | 2 + tools/mota/gen_vectors.py | 38 + tools/mota/motalib.py | 17 +- tools/mota/test_mota.py | 26 + 32 files changed, 3877 insertions(+), 357 deletions(-) diff --git a/docs/lora_ota_automation.md b/docs/lora_ota_automation.md index cef2feec..e392a67e 100644 --- a/docs/lora_ota_automation.md +++ b/docs/lora_ota_automation.md @@ -200,6 +200,12 @@ reported bootloader ABI and codec mask against the selected package. If the version command is unavailable on older firmware, the script warns and falls back to the legacy `ota self` platform marker. If an nRF52 bootloader lacks the required capabilities, install the exact-board OTAFIX bootloader first. +Current firmware also reports `maxblk:2048` near the front of both OTA replies. +The runner treats a missing marker as the deployed 1 KiB receive limit, rejects +a ready 2 KiB package for such a target before transfer, and passes the live +limit to `motatool --block-size` when it builds from raw firmware. Thus updating +an older target from a release ZIP produces a compatible 1 KiB package while a +current target retains the 2 KiB application default. `ota stats` is only an optional EndF version probe. It uses one bounded retry cycle; unsupported firmware or a lost reply falls back directly to the required `ver` command instead of entering an operator continuation loop. @@ -581,7 +587,8 @@ the destination. capabilities, and save the controller's normal radio tuple. 3. Select or build one compatible **v2 application** mOTA and verify all block hashes, Merkle root, full-image hash where applicable, identity fields, - signature, codec, base, and the firmware's 1024-byte maximum block size. + signature, codec, base, and the live firmware's reported maximum block size + (1 KiB for legacy replies without `maxblk`, currently 2 KiB otherwise). Version-3 bootloader packages are refused before any target state changes. 4. Read every participant's version, save the destination's RXPS state, select the qualified destination policy, and show the confirmation prompt. diff --git a/docs/ota_protocol.md b/docs/ota_protocol.md index e6314931..936e5742 100644 --- a/docs/ota_protocol.md +++ b/docs/ota_protocol.md @@ -71,10 +71,18 @@ the authoritative reference for byte-level details. | `CODEC_FULL` / `_SEQUENTIAL` / `_INPLACE` | `0` / `1` / `2` | Section 5 | | `PAYLOAD_TYPE_OTA` | `0x0C` | MeshCore packet type (`src/Packet.h`) | | `MAX_PACKET_PAYLOAD` | `184` | usable bytes per packet (`src/MeshCore.h`) | -| Default block size | `1024` | `block_size_log2 = 0x0A` | +| Default application block size | `2048` | `block_size_log2 = 0x0B`; deployed 1 KiB application packages remain accepted | +| Bootloader-package block size | `1024` | fixed `block_size_log2 = 0x0A` for format 3 | | OTA discovery TX priority | `250` | background (`OTA_TX_PRIORITY`, `src/Mesh.h`) | | OTA active-transfer TX priority | `0` | primary (`OTA_TRANSFER_TX_PRIORITY`, `src/Mesh.h`) | +Current application firmware publishes its receive/reassembly ceiling as the compact `maxblk:2048` field +near the front of both `ota status` and `ota self`. Keeping the field ahead of variable diagnostics ensures +it survives the 160-byte remote-admin reply limit. Host updaters must treat an absent field as the deployed +legacy capability (`1024`), reject a ready package whose manifest block size exceeds that capability, and use +the reported size when turning a raw firmware image into a new application package. This capability does not +change the format-3 bootloader package's fixed 1 KiB geometry. + --- ## 2. Firmware image & the `EndF` trailer @@ -238,7 +246,7 @@ off size field notes 7 4 fw_version MAJOR<<24 | MINOR<<16 | PATCH<<8 | pre (comparable uint32) 11 4 image_size size of the reconstructed image (BODY||EndF) 15 4 payload_size PAYLOAD bytes in this container -19 1 block_size_log2 e.g. 0x0A = 1024 +19 1 block_size_log2 e.g. 0x0B = 2048 (new application default); 0x0A = deployed 1024 20 4 merkle_root sha2-256:4 over PAYLOAD blocks (Section 6) - also the manifest_id 24 32 image_hash sha2-256:32 of the reconstructed image - SECURITY anchor 56 1 codec_id 0=full/raw, 1=detools-sequential, 2=detools-in-place @@ -347,8 +355,9 @@ deltas use `--compression crle`. **Do not reimplement the codec** - use the vend Verifies each PAYLOAD block against the signed `merkle_root` **before** the whole payload exists, so corruption/forgery is localized to a block. Implemented in `MerkleTree.cpp`. -- **Blocks:** PAYLOAD splits into `BC = ceil(payload_size / B)` blocks, `B = 2^block_size_log2` (default - 1024). The last block is its real length (**no zero padding**). +- **Blocks:** PAYLOAD splits into `BC = ceil(payload_size / B)` blocks, `B = 2^block_size_log2` (new + application default 2048; deployed 1024-byte application packages remain valid). The last block is its + real length (**no zero padding**). Format-3 bootloader packages remain fixed at 1024 bytes. - **Leaf:** `leaves[i] = sha2-256:4( block_i_bytes )`. - **Internal node:** `node = sha2-256:4( left || right )` (4+4 input bytes). - **Odd level:** an odd count promotes the **last node unchanged** to the next level (no duplication). @@ -545,12 +554,17 @@ OTA_GET_MANIFEST: manifest_id[4] want_mask(uint16) # bit k = send manifest f OTA_MANIFEST: manifest_id[4] frag_idx(1) frag_total(1) bytes[] # up to OTA_MF_FRAG=176 B/frag OTA_REQ: manifest_id[4] { block_idx(uint16) want_mask(uint16) }[1..4] # legacy: bit k requests its 160-byte fragment - # v2: bit15=marker, bit14=allow transport DEFLATE, bits0..12=fragment bitmap + # v2: bit15=marker, bit14=allow transport DEFLATE, + # bit13=2 KiB descriptor, bits0..12=fragment bitmap; + # bit12 must be zero when bit13 is set (0xFFFF stays legacy) OTA_DATA legacy: manifest_id[4] block_idx(uint16) frag_off(uint16) data[<=160] OTA_DATA v2: manifest_id[4] block_idx(uint16) descriptor(uint16) stream_id[4] data[<=171] - # descriptor: bit15=marker, bit14=DEFLATE, bits13..10=fragment, - # bits9..0=complete encoded length minus one + # deployed 1 KiB descriptor: bit15=marker, bit14=DEFLATE, + # bits13..10=fragment, bits9..0=complete encoded length minus one + # negotiated 2 KiB descriptor: bit15=marker, bits14..11=fragment, + # bits10..0=complete encoded length minus one; DEFLATE iff that + # length is strictly less than the manifest-derived raw block length OTA_REQ_PROOF: manifest_id[4] block_idx(uint16) OTA_PROOF: manifest_id[4] block_idx(uint16) n_proof(1) proof[] # n_proof x 4 bytes OTA_GET_LEAVES: manifest_id[4] want_mask(uint16) # bit k = send leaves fragment k; 0xFFFF = all @@ -571,21 +585,24 @@ OTA_LEAVES: manifest_id[4] frag_idx(1) frag_total(1) bytes[] # up is answering `OTA_GET_LEAVES` with leaves the node already holds, so any node's firmware can be captured. - **Negotiated 171-byte fragments:** the deployed profile remains unchanged: `frag_off` is a byte offset and - `data[]` carries at most 160 bytes. A new fetcher marks an `OTA_REQ` row as v2 and includes all seven legacy - bits on its first request. Old sources ignore/mask the high flags and can therefore return a complete legacy - block immediately; receipt of an untagged `OTA_DATA` switches that fetch session to the old geometry. If no - v2 data appears by the first adaptive deadline, the fetcher retries with an ordinary legacy mask. New - sources answer v2 with a packed descriptor, a repeated 4-byte representation ID, and exactly 171 data bytes - except the final fragment. A raw 1 KiB block therefore falls from seven data packets to six. Message type - IDs do not change, so multi-hop relays continue to forward request/data/proof packets opaquely with the same - priority. + `data[]` carries at most 160 bytes. A new fetcher marks an `OTA_REQ` row as v2. For a deployed 1 KiB block, + its first request also includes every fragment bit needed by the legacy representation, so an old source can + answer it completely. The extended 2 KiB profile reserves bit 12 as zero; this both fits its twelve v2 + fragments and ensures the deployed `0xFFFF` legacy all-fragments request is never misclassified. Receipt of + untagged `OTA_DATA` switches the fetch session to the old geometry. If no v2 data appears by the first + adaptive deadline, the fetcher retries with an ordinary legacy mask. New sources answer v2 with a packed descriptor, a repeated 4-byte representation + ID, and exactly 171 data bytes except the final fragment. A raw 1 KiB block falls from seven data packets to + six; a raw 2 KiB block falls from thirteen to twelve. Message type IDs do not change, so multi-hop relays + continue to forward request/data/proof packets opaquely with the same priority. - **Transport-only DEFLATE:** a fetcher sets the v2 DEFLATE-permission bit only when its application includes and configures the decoder. A source may encode each logical block as an independent raw RFC 1951 stream with full stored, fixed-Huffman, and dynamic-Huffman support (`BTYPE=0/1/2`). Compression is used only when strictly smaller, otherwise the source returns raw v2 data. The receiver inflates to the manifest-derived block length, then performs the unchanged Merkle proof and writes - the original bytes to staging. Thus existing `.mota` containers, signatures, resume markers, and legacy - bootloader apply remain unchanged. The 4-byte `stream_id` is SHA-256:4 of the complete raw or compressed + the original bytes to staging. Thus staging and bootloader apply remain uncompressed. Deployed 1 KiB + `.mota` containers remain compatible; new 2 KiB containers require a receiver advertising the extended + descriptor. Format-3 bootloader packages retain their strict 1 KiB geometry. The 4-byte `stream_id` is + SHA-256:4 of the complete raw or compressed representation and appears in every fragment. A receiver locks `{encoding, encoded length, stream_id}` for a block, preventing fragments from independently encoded seeders from being mixed. After one sparse retry, a second stalled interval clears the full in-flight window and switches the session to legacy geometry, so @@ -594,12 +611,11 @@ OTA_LEAVES: manifest_id[4] frag_idx(1) frag_total(1) bytes[] # up bounds output to the exact logical block length and rejects truncation, malformed streams, and trailing whole bytes. Builds without OTA do not link the decoder. This application capability is independent of the bootloader because transport data is inflated before the unchanged staged container is written. -- **Adaptive flight size is not signed block size.** The container continues to use 1 KiB Merkle leaves and - each slot is one existing 1 KiB block. A clean link changes how many of those blocks one `OTA_REQ` names: - 1, then 2, then 3, then the compiled cap. The manifest stores `block_size_log2`, so 3 KiB is not a valid - logical geometry; enabling 2 KiB would require larger device reassembly buffers and new-package/bootloader - validation while saving only one request/proof pair per 2 KiB. It does not address premature retries, which - were the dominant packet multiplier. +- **Adaptive flight size is not signed block size.** Each slot holds one manifest-defined block: 1 KiB for + deployed application containers or 2 KiB for new ones. A clean link changes how many complete blocks one + `OTA_REQ` names: 1, then 2, then 3, then the compiled cap. It never changes block geometry. Two KiB halves + the leaf, block-request, and proof count for a payload while giving each independent DEFLATE stream a larger + history window. - **Append-only request-window compatibility:** the first four-byte request row is exactly the original `block_idx + want_mask` body. Old sources decode that row and ignore appended bytes. New sources queue all rows. If a new fetcher meets an old source, the unserved tail rows eventually time out and are recovered as @@ -614,8 +630,8 @@ OTA_LEAVES: manifest_id[4] frag_idx(1) frag_total(1) bytes[] # up half-duplex radios: re-requesting a whole multi-fragment burst let the periodic retry (a transmit) collide with the tail of the in-flight burst and drop the same fragment forever - a hang. Requesting only the hole removes the burst, so there is nothing to collide with. The block/manifest mask matches the 16-bit - reassembly bitmap (<=16 fragments/block; legacy 1 KB blocks = 7, raw v2 = 6). `OTA_PROOF` is a single packet - and needs no mask. + reassembly bitmap (<=16 fragments/block; 1 KiB is 7 legacy or 6 raw-v2 fragments; 2 KiB is 13 legacy or 12 + raw-v2 fragments). `OTA_PROOF` is a single packet and needs no mask. - **Data and proof remain separate packets, without a normal extra round trip.** A server retains requested blocks in a bounded descriptor queue, admits at most one response per main-loop pass, and sends one `OTA_PROOF` after each block's requested fragments. Before admitting that proactive proof, the source leaves @@ -631,14 +647,14 @@ OTA_LEAVES: manifest_id[4] frag_idx(1) frag_total(1) bytes[] # up | message | fixed overhead | payload/packet | |---|---|---| -| `OTA_DATA` legacy | 9 B (type+mid4+idx2+off2) | `OTA_FRAG_DATA = 160` -> 7 frags per 1 KB block | -| `OTA_DATA` v2 | 13 B (legacy header + stream ID) | `OTA_FRAG_DATA_V2 = 171` -> 6 raw frags per 1 KB block; fewer when compressed | +| `OTA_DATA` legacy | 9 B (type+mid4+idx2+off2) | `OTA_FRAG_DATA = 160` -> 7 raw frags/1 KiB or 13/2 KiB | +| `OTA_DATA` v2 | 13 B (legacy header + stream ID) | `OTA_FRAG_DATA_V2 = 171` -> 6 raw frags/1 KiB or 12/2 KiB; fewer when compressed | | `OTA_MANIFEST` | 7 B | `OTA_MF_FRAG = 176` -> signed manifest ~ 2 frags | | `OTA_HAVE` | 12 B | 10 rows x 16 B per fragment | | `OTA_PROOF` | 8 B | up to ~44 sibling digests (>> any real tree) | -A served mota supports up to `OTA_MAX_BLOCK/4` leaves in the default 4 KB proof scratch (<=1024 blocks ~ 1 MB -payload); larger self-images pass a bigger scratch buffer. +A served mota supports up to 1024 leaves in the default 4 KiB proof scratch (about 2 MiB of payload at the +new 2 KiB default, or 1 MiB for a deployed 1 KiB container); larger self-images pass a bigger scratch buffer. ### 8.6 Temporary-radio and transfer boundary @@ -754,7 +770,7 @@ class MotaSource { }; ``` -To serve an external mota the node reads its manifest-minus-leaves + `leaves[]` into RAM (<=4 KB for <=1024 +To serve an external mota the node reads its manifest-minus-leaves + `leaves[]` into RAM (<=4 KiB for <=1024 blocks) and streams payload blocks from the source on demand; proofs are generated from the read leaves. ### 10.2 The `mota-seeder` transport (`MotaSeederProto.h`) @@ -784,8 +800,8 @@ status: 0 = OK, non-zero = error (out of range / past EOF). ``` `SerialMotaSource` splits logical reads into replies of at most 192 payload -bytes. A manifest's leaf table can exceed 256 bytes and payload blocks are -normally 1 KiB; requesting either in one transaction can overrun common USB +bytes. A manifest's leaf table can exceed 256 bytes and new payload blocks are +normally 2 KiB; requesting either in one transaction can overrun common USB CDC/UART receive rings even though the host successfully wrote the complete reply. Chunking is internal to the transport and does not change `OP_READ` or the `MotaSource` random-access contract. diff --git a/docs/ota_user_guide.md b/docs/ota_user_guide.md index 0ff07136..b406185e 100644 --- a/docs/ota_user_guide.md +++ b/docs/ota_user_guide.md @@ -235,12 +235,12 @@ trusted - every kept block is checked against the target's own fingerprints, so just means those blocks are fetched over the radio (correct result, only slower). The node fetches from one source as **primary traffic**, with bounded adaptive request flights. Every session -probes with one 1 KiB block, then clean flights grow `1 -> 2 -> 3 -> 4` blocks on RAK3401 OTA builds. All blocks +probes with one manifest block, then clean flights grow `1 -> 2 -> 3 -> 4` blocks on RAK3401 OTA builds. New +application packages use 2 KiB blocks; deployed 1 KiB packages remain supported. All blocks in a flight share one request packet, and the receiver stays silent until the source/relays finish returning them. A recovery halves the next flight. Retry timing follows the active SF/BW airtime, duty budget, and path length, so faster settings recover sooner without a fixed one-second request colliding with a valid multi-hop -response. The signed block size itself remains 1 KiB. Mesh repeaters carry the packets only while their -temporary-radio windows are active. +response. Mesh repeaters carry the packets only while their temporary-radio windows are active. Check progress with `ota status`. If a `folder` pull loses its link mid-transfer, `ota status` shows **paused** - the host keeps the diff --git a/docs/rak3401_mota_chain.md b/docs/rak3401_mota_chain.md index 37c2d121..d7ee4482 100644 --- a/docs/rak3401_mota_chain.md +++ b/docs/rak3401_mota_chain.md @@ -3,25 +3,25 @@ > Status: **published prerelease for controlled, recoverable lab use.** On > 29-Aug-2026 steps 1-9 completed on the target RAK3401 and are byte-for-byte > identical to the corresponding transitions in the physically passed -> `fd98bc90` chain. The replacement step 10 installs the current `b40d2e6c` +> `fd98bc90` chain. The replacement step 10 installs the current `3f6eddd5` > endpoint. That replacement package and the hardened current runner have -> passed offline reconstruction plus both required bootloader simulators, but +> passed offline reconstruction plus the exact deployed bootloader simulator, but > have not had a new physical end-to-end run. Use the lab gate and keep local > DFU/SWD recovery available. Multi-hop and alternate-bandwidth estimates > remain planning data, not physical qualification. The prerelease is -[`rak3401-mota-v1.16.07-c1caa5ad-to-v1.17.1.5-b40d2e6c`](https://github.com/mikecarper/MeshCore/releases/tag/rak3401-mota-v1.16.07-c1caa5ad-to-v1.17.1.5-b40d2e6c). +[`rak3401-mota-v1.16.07-c1caa5ad-to-v1.17.1.6-3f6eddd5`](https://github.com/mikecarper/MeshCore/releases/tag/rak3401-mota-v1.16.07-c1caa5ad-to-v1.17.1.6-3f6eddd5). Use this asset: ```text -RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-b40d2e6c.zip +RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.6-3f6eddd5.zip ``` -- prerelease ZIP SHA-256: `8e7f0c2565f37d4372fa51dc3ba72b591e7f38b74be09303e48641623f99bce4` -- inner `SHA256SUMS.txt` SHA-256: `29a02a6c4c83eafa80168a80562b0ded5d8432e6cc02cfaca289cb2ed426681a` -- endpoint image SHA-256: `a7761da8af1f2447d7f01d72e2b1456b99c0e874a165d6483f48ab1f58e8d905` -- endpoint EndF body hash: `1FA6AAA3C28D8BD8` +- prerelease ZIP SHA-256: `135783dd8777490db2422a7543f7d80e3cf03f621d5ac362da930abbc1850dc4` +- inner `SHA256SUMS.txt` SHA-256: `93835ed497c579be0a2322292e7f15ab095e7c8ab7892833fe7df3130dbdaba4` +- endpoint image SHA-256: `cd1e9b819f9e918f9f2ceaf37b8a74bf242e2be78f319af2cf02a42d7729d0f4` +- endpoint EndF body hash: `9BD7CF682EE065AE` `--verify-only` remains the safe default. Live preflight and installation of this prerelease require `--accept-test-candidate`. That switch bypasses only @@ -37,11 +37,20 @@ target: - hardware: `RAK_3401` - role: `RAK_3401_repeater_lora_ota_no_external_sensors` - start: `v1.16.7.0-c1caa5ad`, EndF `71F4026CBE4B8B74` -- endpoint source: MeshCore `b40d2e6cb2a185db740175e8b161ea71736d19d9` -- endpoint label: `v1.17.1.5-halo-keymind-cascade-dev-b40d2e6c` -- endpoint packed version: `0x01110105` +- endpoint source: MeshCore `3f6eddd5c20f1fa3df6ba1cc208efbe3e2fffb43` +- endpoint label: `v1.17.1.6-halo-keymind-cascade-dev-3f6eddd5` +- endpoint packed version: `0x01110106` - deployment target key: `63d8df6387eaffd2e25db7d2a8ad967a65202182a48d681d7e7a9260f917280d` +The exact starting application is the +[`lora-ota-v1.16.07-halo-keymind-cascade-dev-3a6af3bf`](https://github.com/mikecarper/MeshCore/releases/tag/lora-ota-v1.16.07-halo-keymind-cascade-dev-3a6af3bf) +RAK3401 asset. Its outer ZIP SHA-256 is +`58ead2ea18efafda186bc9d983fe08cdf65f69b0e5a3a1c72c18fe35948a254c`; +its 566,912-byte `firmware.bin` SHA-256 is +`4301fc63ebd661c70f9bc40e7eca44d3cb8a358cb3c0075e700d9676a57478cb`. +The endpoint build ZIP SHA-256 is +`a9fca9be145a96feb650b57c43b5ed173d74d2f3194da12e31b1c54352635b56`. + ### Retained and omitted hardware support Despite the legacy `no_external_sensors` target name, the endpoint retains the @@ -62,7 +71,7 @@ identity without helping this route. Do not use the chain on another target ID, hardware family, starting image, or firmware body hash. The runner checks all four. -## Why Preview 5 and the old staging ceiling work +## Why the deployed OTAFIX2.4 and old staging ceiling work The deployed bootloader is not changed and no package uses the newer expanded `0xED000` ceiling. Each package remains bottom-aligned below the old @@ -85,19 +94,30 @@ workspace. Every package in this candidate is an ordinary application container using format ABI 2 and in-place codec 2. The historical bridge applications scan the bootloader capability marker byte-by-byte, so they recognize the released -Preview 5 RAK4631 marker at absolute address `0xFCDEA` even though it is two -bytes off a word boundary. The `b40d2e6c` endpoint retains that exact Preview -5 application-update fallback. Its separately compiled internal -bootloader-update feature remains unavailable on Preview 5 because privileged -self-update requires a current ABI 3 marker with the exact storage profile. +`0.9.2-OTAFIX2.4` marker at absolute address `0xFCD16`. That immutable marker +advertises ABI 2, codec mask `0x0004`, and internal storage profile 0. The +`3f6eddd5` endpoint retains the matching legacy application-update fallback. +Its separately compiled internal bootloader-update feature remains unavailable +because privileged self-update requires a current ABI 3 marker with the exact +storage profile. Installing this chain therefore does not replace the bootloader, and the final application can still accept a future valid ABI-2/codec-2 application update. -The schema-2 exhaustive search checked 17,371 candidate geometries and found -121 feasible forward edges in the declared 33-image inventory. Fourteen -ten-package routes tied for the minimum package count; the selected route has -the smallest total container size among them. That minimum applies to the -declared inventory, not every conceivable firmware image. The selected route +The exact installed bootloader is the +[`0.9.2-OTAFIX2.4` RAK4631 asset](https://github.com/mikecarper/Adafruit_nRF52_Bootloader_OTAFIX/releases/download/0.9.2-OTAFIX2.4/wiscore_rak4631_board_bootloader-0.9.2-OTAFIX2.4_s140_6.1.1.zip). +Its outer ZIP SHA-256 is +`5e29e7a8982cc2ed5f8868435556eb09387250fe5b9a7e8443bac325247dfed1`; +its 191,040-byte `sd_bl.bin` SHA-256 is +`a97f834388db0f1da6c29f6d8f458ff6d3e6a26034709044566384838919bb23`. +The exact simulator was compiled from release commit +`d73de8372e89b8ef352747c8bc7a1aaeab80fbfe`; its executable SHA-256 is +`9dacf24b1023fe2f4c620419417649ccc9538dd4a611b47bc19b7fedb97ceba6`. + +The schema-2 exhaustive search checked 1,799 candidate geometries and found 12 +feasible forward edges in the declared exact 12-node inventory. It found one +shortest ten-package route and selected the smallest total container bytes as +the secondary objective. This minimum applies only to that declared inventory +and geometry table, not every conceivable new bridge image. The selected route is: | Step | From | To | Workspace | Package | Margin | @@ -111,20 +131,28 @@ is: | 7 | 1.16.9.113 | 1.16.9.117 | `0x8B000` | 139,637 | 0 | | 8 | 1.16.9.117 | 1.16.10.0 | `0x7E000` | 174,202 | 20,480 | | 9 | 1.16.10.0 | 1.17.1.3 | `0x6B000` | 271,569 | 0 | -| 10 | 1.17.1.3 | 1.17.1.5 | `0x74000` | 174,744 | 61,440 | +| 10 | 1.17.1.3 | 1.17.1.6 | `0x76000` | 197,825 | 28,672 | -Total mOTA transfer data is 1,384,252 bytes. `ROUTE.json`, `CHAIN.csv`, and +Total mOTA transfer data is 1,407,333 bytes. `ROUTE.json`, `CHAIN.csv`, and `validation-results.json` in the bundle pin the exact geometry and image hash for every transition. Steps 1-9 are byte-for-byte identical to the transitions that passed on the physical RAK3401. The replacement step 10, from the physically reached -`FE65A6135A1E7B3F` v1.17.1.3 body to `1FA6AAA3C28D8BD8`, has offline +`FE65A6135A1E7B3F` v1.17.1.3 body to `9BD7CF682EE065AE`, has offline qualification only. All ten packages passed independent container checks, -zero-filled and erased-workspace reconstruction, the OTAFIX 2.4.1 Preview 5 -simulator, and the current OTAFIX simulator. Those results prove the package -bytes and apply geometry; they are not a physical claim for the replacement -step 10 or the hardened runner. +zero-filled and erased-workspace reconstruction, and the exact deployed +`0.9.2-OTAFIX2.4` simulator. Those results prove the package bytes and apply +geometry; they are not a physical claim for the replacement step 10 or the +hardened runner. + +The 171-byte transport DEFLATE support lands only in the final `3f6eddd5` +application. The starting receiver and every receiver that accepts steps 1-10 +lack that inflate path, so all ten legacy-chain transfers remain raw and the +1,407,333-byte total above is not automatically reduced over LoRa. A later +update can negotiate transport compression after this endpoint is running; +the application inflates before staging, and the immutable bootloader still +applies an ordinary ABI-2/codec-2 container. ## Host requirements @@ -137,7 +165,7 @@ No password or device is needed: ```bash python3 tools/lora_ota/rak3401_mota_chain.py \ - --bundle /path/to/RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-b40d2e6c.zip \ + --bundle /path/to/RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.6-3f6eddd5.zip \ --work-dir ./rak3401-mota-chain-work \ --motatool /path/to/motatool \ --verify-only @@ -166,7 +194,7 @@ Restore the test start locally with export MESHCORE_ADMIN_PASSWORD='password' python3 tools/lora_ota/rak3401_mota_chain.py \ - --bundle /path/to/RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-b40d2e6c.zip \ + --bundle /path/to/RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.6-3f6eddd5.zip \ --work-dir ./rak3401-mota-chain-work \ --controller-serial /dev/ttyACM0 \ --source-tcp 192.168.1.51:5001 \ @@ -188,6 +216,23 @@ zero and restores it at the endpoint. Use that option only where the selected frequency and local duty-cycle rules permit a full transmit budget; omit it otherwise. +## Superseded `b40d2e6c` prerelease + +The immediately preceding prerelease remains pinned for offline provenance: + +- release: [`rak3401-mota-v1.16.07-c1caa5ad-to-v1.17.1.5-b40d2e6c`](https://github.com/mikecarper/MeshCore/releases/tag/rak3401-mota-v1.16.07-c1caa5ad-to-v1.17.1.5-b40d2e6c) +- asset: `RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-b40d2e6c.zip` +- outer ZIP SHA-256: `8e7f0c2565f37d4372fa51dc3ba72b591e7f38b74be09303e48641623f99bce4` +- inner `SHA256SUMS.txt` SHA-256: `29a02a6c4c83eafa80168a80562b0ded5d8432e6cc02cfaca289cb2ed426681a` +- endpoint image SHA-256: `a7761da8af1f2447d7f01d72e2b1456b99c0e874a165d6483f48ab1f58e8d905` +- endpoint EndF body hash: `1FA6AAA3C28D8BD8` + +Its first nine packages are byte-for-byte identical to the current release and +the physical `fd98bc90` evidence. Its different step 10 had offline +qualification only. The runner recognizes this exact archive separately and +keeps live use gated; it is not interchangeable with the current `3f6eddd5` +asset. + ## Legacy `fd98bc90` physical and SWD qualification The earlier local candidate remains the exact physical evidence source for @@ -207,9 +252,9 @@ deployed `0.9.2-OTAFIX2.4` bootloader and a Heltec V4 source at 909.950 MHz / 500 kHz / SF5 / CR5. Every intermediate EndF hash matched. Independent SWD readback then matched the endpoint application, original bootloader, SoftDevice/MBR, and UICR byte-for-byte. Steps 1-9 are also the exact first nine -packages in the `b40d2e6c` prerelease. The two candidates have different step -10 packages and endpoints, so the physical result and step-10 timing must not -be attributed to `b40d2e6c`. +packages in the `b40d2e6c` and `3f6eddd5` prereleases. All three candidates +have different step 10 packages or endpoints, so the physical result and +step-10 timing must not be attributed to either later prerelease. ### Measured direct-link timing for `fd98bc90` @@ -291,17 +336,17 @@ never skip a step even when a later package appears in `ota ls`. ### 1. Verify and extract the asset ```bash -sha256sum RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-b40d2e6c.zip -unzip RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-b40d2e6c.zip -cd RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-b40d2e6c +sha256sum RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.6-3f6eddd5.zip +unzip RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.6-3f6eddd5.zip +cd RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.6-3f6eddd5 sha256sum -c SHA256SUMS.txt for package in motas/*.mota; do motatool verify "$package" || exit 1; done ``` The outer hash must be -`8e7f0c2565f37d4372fa51dc3ba72b591e7f38b74be09303e48641623f99bce4`, +`135783dd8777490db2422a7543f7d80e3cf03f621d5ac362da930abbc1850dc4`, and the SHA-256 of the extracted `SHA256SUMS.txt` must be -`29a02a6c4c83eafa80168a80562b0ded5d8432e6cc02cfaca289cb2ed426681a`. +`93835ed497c579be0a2322292e7f15ab095e7c8ab7892833fe7df3130dbdaba4`. Do not continue after any checksum or verification error. ### 2. Record and prepare the destination diff --git a/src/helpers/ota/MotaContainer.cpp b/src/helpers/ota/MotaContainer.cpp index 671c06e9..6596a069 100644 --- a/src/helpers/ota/MotaContainer.cpp +++ b/src/helpers/ota/MotaContainer.cpp @@ -44,7 +44,9 @@ static bool parse_manifest_fields(ByteReader& r, MotaManifest& out) { out.approval = r.take(4); if (!r.ok) return false; if (out.block_size_log2 == 0 || out.block_size_log2 > 24 || out.payload_size == 0) return false; - out.block_count = (out.payload_size + out.block_size() - 1) / out.block_size(); + const uint32_t block_size = out.block_size(); + out.block_count = out.payload_size / block_size + + (out.payload_size % block_size != 0 ? 1u : 0u); if (out.is_bootloader()) { static const uint8_t zero_base[8] = {0}; if (out.fw_version == 0 || out.codec_id != CODEC_FULL || out.image_size != 0xA000u || diff --git a/src/helpers/ota/OtaCli.cpp b/src/helpers/ota/OtaCli.cpp index bae1b490..e2166eb4 100644 --- a/src/helpers/ota/OtaCli.cpp +++ b/src/helpers/ota/OtaCli.cpp @@ -212,7 +212,9 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board uint8_t dig[4]; c.manager.servedDigest(dig); char dighx[9]; mesh::Utils::toHex(dighx, dig, 4); snprintf(reply, 160, - "OTA seeder | install:disabled | folder:%s | serving:%u dg=%s | target:00000000 (source only)", + "OTA seeder | install:disabled | target:00000000 | maxblk:%u | folder:%s | " + "serving:%u dg=%s (source only)", + (unsigned)ota_max_block_capability(), c.folder_active ? c.folder_dest_info : "not connected", (unsigned)c.manager.servedCount(), dighx); #else @@ -252,22 +254,27 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board #endif const uint8_t last_rc = ota_bootloader_last_rc(); const char* rc_name = ota_nrf52_boot_update_result(last_rc) ? "blup" : "blrc"; - // Keep the fixed-width parser/diagnostic fields ahead of the variable download text and optional - // human target name. hw is bounded to 32 bytes, so target + the complete blup/blrc token always fit - // in the 160-byte reply; snprintf may truncate only the descriptive tail. + // Keep the fixed-width parser/capability fields ahead of the variable download text and optional + // human target name. hw is bounded to 32 bytes, so target, maxblk, and the complete blup/blrc token + // always fit in the 160-byte reply; snprintf may truncate only the descriptive tail. snprintf(reply, 160, - "OTA | this fw %s (%uK) hw=%s | target:%08X | bl:%s %s:%02X | %s | " + "OTA | this fw %s (%uK) hw=%s | target:%08X | maxblk:%u | bl:%s %s:%02X | %s | " "serving:%s (%u) | keys:%u | env:%s", selfhx, (unsigned)((s ? fi.image_len : 0) / 1024), hw, - (unsigned)c.manager.target(), bl_state, rc_name, last_rc, dl, + (unsigned)c.manager.target(), (unsigned)ota_max_block_capability(), + bl_state, rc_name, last_rc, dl, c.serving ? "on" : "off", (unsigned)c.manager.servedCount(), (unsigned)c.allow.count(), tenv ? tenv : "?"); #else + // target and maxblk precede the variable download text so remote-admin truncation cannot hide the + // package-geometry preflight fields. snprintf(reply, 160, - "OTA | this fw %s (%uK) hw=%s | %s | serving:%s (%u) | keys:%u | target:%08X (%s)", - selfhx, (unsigned)((s ? fi.image_len : 0) / 1024), hw, dl, + "OTA | this fw %s (%uK) hw=%s | target:%08X | maxblk:%u | %s | " + "serving:%s (%u) | keys:%u | env:%s", + selfhx, (unsigned)((s ? fi.image_len : 0) / 1024), hw, + (unsigned)c.manager.target(), (unsigned)ota_max_block_capability(), dl, c.serving ? "on" : "off", (unsigned)c.manager.servedCount(), - (unsigned)c.allow.count(), (unsigned)c.manager.target(), tenv ? tenv : "?"); + (unsigned)c.allow.count(), tenv ? tenv : "?"); #endif #endif @@ -720,7 +727,10 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board SelfFwInfo fi; if (!ota_self_firmware(fi) || !fi.valid) { strcpy(reply, "ERR no EndF (firmware lacks the trailer?)"); return true; } char hx[17]; mesh::Utils::toHex(hx, fi.body_hash, 8); - int n = snprintf(reply, 160, "self body=%u image=%u base_hash=%s", (unsigned)fi.body_len, (unsigned)fi.image_len, hx); + // maxblk is before the optional bootloader diagnostics so it survives the 160-byte remote reply cap. + int n = snprintf(reply, 160, "self body=%u image=%u base_hash=%s | maxblk:%u", + (unsigned)fi.body_len, (unsigned)fi.image_len, hx, + (unsigned)ota_max_block_capability()); #if defined(NRF52_PLATFORM) // nRF52 applies via the bootloader, so surface whether THIS device's bootloader can install this store. const OtaBlCaps& bl = c.bootloaderAppCaps(); // cached (flash scanned once) diff --git a/src/helpers/ota/OtaContext.h b/src/helpers/ota/OtaContext.h index 017667a3..bcf5163b 100644 --- a/src/helpers/ota/OtaContext.h +++ b/src/helpers/ota/OtaContext.h @@ -128,7 +128,7 @@ struct OtaContext { // flash-backed self-serve: cached merkle leaves (heap, freed on re-serve) + assembled manifest of our // own running firmware. The payload is read from flash per block; only the metadata is held in RAM. // serve_self_proof is the proof-gen working buffer (>= block_count*4) - sized to OUR image's block - // count (the manager's fixed 4 KB scratch only covers <=1024 blocks; a >1 MB image needs more). + // count (the manager's fixed 4 KiB scratch covers <=1024 blocks, about 2 MiB at the new default). uint8_t* serve_self_leaves = nullptr; uint8_t* serve_self_proof = nullptr; uint8_t serve_self_manifest[MOTA_MFL]; // fixed-layout full+unsigned manifest-minus-leaves (197 B) diff --git a/src/helpers/ota/OtaFormat.h b/src/helpers/ota/OtaFormat.h index 0ea22039..1069dd1c 100644 --- a/src/helpers/ota/OtaFormat.h +++ b/src/helpers/ota/OtaFormat.h @@ -119,7 +119,11 @@ inline bool ota_is_transfer_message(uint8_t type) { } } -static const uint16_t OTA_DEFAULT_BLOCK_SIZE = 1024; +// New application containers use 2 KiB Merkle/transport blocks. Parsers still +// accept the deployed 1 KiB geometry, while bootloader-update containers keep +// their separate, fixed 1 KiB contract. +static const uint16_t OTA_DEFAULT_BLOCK_SIZE = 2048; +static const uint16_t OTA_MAX_APPLICATION_BLOCK_SIZE = 2048; static const uint8_t OTA_DEFAULT_HOP_LIMIT = 3; } // namespace ota diff --git a/src/helpers/ota/OtaManager.cpp b/src/helpers/ota/OtaManager.cpp index 2150864e..14467491 100644 --- a/src/helpers/ota/OtaManager.cpp +++ b/src/helpers/ota/OtaManager.cpp @@ -67,10 +67,11 @@ bool OtaManager::serve(const uint8_t* mota, uint32_t len) { uint8_t* scratch = ensureScratch(); if (!scratch) return false; if (!mota_parse(mota, len, _view0.m)) return false; + if (_view0.m.block_size() == 0 || _view0.m.block_size() > OTA_MAX_BLOCK) return false; _view0.mfl = (uint16_t)(_view0.m.leaves - _view0.m.manifest_start); // contiguous container _view0.read = nullptr; _view0.read_ctx = nullptr; // payload is contiguous _view0.m.payload _view0.read_deflated = nullptr; _view0.deflate_ctx = nullptr; - _view0.scratch = scratch; _view0.scratch_sz = OTA_PROOFGEN_SCRATCH; // <=1024 blocks (RAM .mota is small) + _view0.scratch = scratch; _view0.scratch_sz = OTA_PROOFGEN_SCRATCH; // <=1024 leaves/blocks _view0.valid = true; clearPendingEgress(); if (_n_src_obj) refresh_sources(); else registerSelfEntry(); @@ -82,6 +83,7 @@ bool OtaManager::serve_self(const uint8_t* manifest, uint16_t mfl, const uint8_t ServeReadFn read, void* ctx) { if (proof_scratch_sz < (uint64_t)block_count * 4) return false; // proof-gen needs count*4 working bytes if (!mota_parse_manifest(manifest, mfl, _view0.m)) return false; // fixed fields: root, image_hash, sizes + if (_view0.m.block_size() == 0 || _view0.m.block_size() > OTA_MAX_BLOCK) return false; _view0.m.manifest_start = manifest; _view0.m.leaves = leaves; // pre-computed, caller-owned (heap) _view0.m.payload = nullptr; // read on demand via `read` @@ -417,7 +419,8 @@ void OtaManager::handleGetLeaves(const uint8_t* m, uint16_t n) { } // Smallest mask covering `nf` fragments: bit k set for k in [0, nf). Caps at 16 (matches each pipeline -// slot mask and the manifest reassembly), which bounds a block at 16 fragments - our 1 KB blocks are 7. +// slot mask and the manifest reassembly), which bounds a block at 16 fragments. A 2 KiB block is 13 legacy +// fragments or 12 v2 fragments. static inline uint16_t frag_full_mask(uint32_t nf) { return (nf >= 16) ? 0xFFFFu : (uint16_t)((1u << nf) - 1); } @@ -426,14 +429,15 @@ static inline uint16_t frag_full_mask(uint32_t nf) { // while a block is already queued merge only fragments not yet admitted to the radio queue; a later retry can // enqueue the block again if one of those admitted packets was actually lost over the air. bool OtaManager::queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want_mask, - bool wire_v2, bool allow_deflate) { + bool wire_v2, bool allow_deflate, bool extended_length) { for (uint8_t i = 0; i < _n_serve_jobs; i++) { ServeJob& job = _serve_jobs[i]; if (job.block != block || memcmp(job.mid, mid, 4) != 0) continue; // A proof-only request can complete whichever representation is already queued. DATA requests with // different geometries remain distinct because their bitmap positions do not describe the same bytes. if (want_mask != 0 && (job.wire_v2 != wire_v2 || - (wire_v2 && job.allow_deflate != allow_deflate))) continue; + (wire_v2 && (job.allow_deflate != allow_deflate || + job.extended_length != extended_length)))) continue; job.pending_mask |= (uint16_t)(want_mask & ~job.emitted_mask); if (want_mask == 0) job.proof_requested = true; return true; @@ -448,6 +452,7 @@ bool OtaManager::queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want job.proof_requested = want_mask == 0; job.wire_v2 = wire_v2; job.allow_deflate = allow_deflate; + job.extended_length = extended_length; return true; } @@ -633,6 +638,10 @@ bool OtaManager::handleReq(const uint8_t* m, uint16_t n) { const bool wire_v2 = ota_req_is_v2(rq.items[i].want_mask); const bool allow_deflate = wire_v2 && (rq.items[i].want_mask & OTA_REQ_V2_ALLOW_DEFLATE) != 0; + const bool extended_length = wire_v2 && + ota_req_v2_extended_length(rq.items[i].want_mask); + const bool block_requires_extended = v->m.block_size() > OTA_DATA_V2_LEGACY_MAX_ENCODED; + if (wire_v2 && extended_length != block_requires_extended) continue; const uint16_t valid_mask = frag_full_mask((blen + (wire_v2 ? OTA_FRAG_DATA_V2 : OTA_FRAG_DATA) - 1) / (wire_v2 ? OTA_FRAG_DATA_V2 : OTA_FRAG_DATA)); @@ -640,7 +649,7 @@ bool OtaManager::handleReq(const uint8_t* m, uint16_t n) { ? ota_req_v2_fragments(rq.items[i].want_mask) : rq.items[i].want_mask; const uint16_t want = (uint16_t)(requested & valid_mask); if (want != 0) accepted |= queueServeJob(v->m.merkle_root, (uint16_t)idx, want, - wire_v2, allow_deflate); + wire_v2, allow_deflate, extended_length); } return accepted; } @@ -710,7 +719,10 @@ void OtaManager::serviceEgress() { dm.block_idx = job.block; uint8_t v2_data[OTA_DATA_V2_STREAM_ID_BYTES + OTA_FRAG_DATA_V2]; if (job.wire_v2) { - if (!ota_data_v2_pack(fragment, _serve_wire_len, _serve_wire_deflated, dm.frag_off)) { + const bool packed = job.extended_length + ? ota_data_v2_pack_extended(fragment, _serve_wire_len, dm.frag_off) + : ota_data_v2_pack(fragment, _serve_wire_len, _serve_wire_deflated, dm.frag_off); + if (!packed) { popServeJob(); return; } @@ -1342,7 +1354,7 @@ void OtaManager::handleManifest(const uint8_t* m, uint16_t n) { uint32_t bs = 1u << bsl; // a block must fit our reassembly buffer (and be non-empty) - reject an oversized block_size up front if (bs == 0 || bs > OTA_MAX_BLOCK || payload_size == 0) { failFetch(FETCH_ERROR_GEOMETRY); return; } - uint32_t bc = (payload_size + bs - 1) / bs; + uint32_t bc = payload_size / bs + (payload_size % bs != 0 ? 1u : 0u); if (bc > 0xFFFFu) { failFetch(FETCH_ERROR_TOO_LARGE); return; } // uint16 block index on the wire if (_archive_fetch && (uint64_t)bc * 4 > OTA_PROOFGEN_SCRATCH) { failFetch(FETCH_ERROR_TOO_LARGE); return; // retaining an image we cannot seed is useless @@ -1659,7 +1671,12 @@ bool OtaManager::handleData(const uint8_t* m, uint16_t n) { uint8_t fragment = 0; uint16_t encoded_len = 0; bool deflated = false; - if (!ota_data_v2_unpack(dm.frag_off, fragment, encoded_len, deflated)) return false; + const bool extended_length = _fbs > OTA_DATA_V2_LEGACY_MAX_ENCODED; + const bool unpacked = extended_length + ? ota_data_v2_unpack_extended(dm.frag_off, (uint16_t)blen, + fragment, encoded_len, deflated) + : ota_data_v2_unpack(dm.frag_off, fragment, encoded_len, deflated); + if (!unpacked) return false; if (encoded_len > blen || (deflated ? encoded_len >= blen : encoded_len != blen)) return false; const uint16_t fragment_len = (uint16_t)(dm.data_len - OTA_DATA_V2_STREAM_ID_BYTES); const uint32_t fragment_off = (uint32_t)fragment * OTA_FRAG_DATA_V2; @@ -1795,7 +1812,8 @@ uint16_t OtaManager::wireRequestMask(uint32_t block, uint16_t need, uint16_t rec want = frag_full_mask((blockLen(block) + OTA_FRAG_DATA - 1) / OTA_FRAG_DATA); } if (want == 0) want = need; - return ota_req_make_v2(want, _wire_allow_deflate); + return ota_req_make_v2( + want, _wire_allow_deflate, _fbs > OTA_DATA_V2_LEGACY_MAX_ENCODED); } if (want == 0) want = need; return want; diff --git a/src/helpers/ota/OtaManager.h b/src/helpers/ota/OtaManager.h index d913bb2b..577f1181 100644 --- a/src/helpers/ota/OtaManager.h +++ b/src/helpers/ota/OtaManager.h @@ -15,8 +15,9 @@ // the signed merkle root via proofs). It is portable (no Arduino / radio / Ed25519) so it can be // driven by a host simulation; a thin Mesh adapter wires it to PAYLOAD_TYPE_OTA on device. // -// Transfer keeps 1 KB logical blocks. A server paces each block's self-describing DATA fragments through a -// bounded response queue and follows them with the Merkle PROOF. A short radio-aware turnaround gap lets a +// Transfer accepts deployed 1 KiB and new 2 KiB logical blocks. A server paces each block's self-describing +// DATA fragments through a bounded response queue and follows them with the Merkle PROOF. A short +// radio-aware turnaround gap lets a // legacy fetcher send its immediate REQ_PROOF without colliding with that proactive proof; newer fetchers wait // briefly for the proactive proof and retain REQ_PROOF as a loss/legacy-server fallback. @@ -49,15 +50,21 @@ typedef bool (*ServeDeflateReadFn)(void* ctx, uint16_t block, uint8_t* dst, #ifndef OTA_PROOFGEN_SCRATCH #if defined(OTA_SD_STORE) - #define OTA_PROOFGEN_SCRATCH 8192 // SD archive can seed <=2048 blocks (about 2 MB at 1 KB/block) + #define OTA_PROOFGEN_SCRATCH 8192 // SD archive can seed <=2048 blocks (about 4 MiB at 2 KiB/block) #else #define OTA_PROOFGEN_SCRATCH 4096 // server proof-gen working buffer (supports up to 1024 blocks) #endif #endif #ifndef OTA_MAX_BLOCK -#define OTA_MAX_BLOCK 1024 // largest logical block (merkle leaf unit) = reassembly buffer size +#define OTA_MAX_BLOCK 2048 // largest logical block (Merkle leaf / independent DEFLATE unit) #endif +static_assert(OTA_MAX_BLOCK <= OTA_DATA_V2_MAX_ENCODED, + "OTA_MAX_BLOCK exceeds the negotiated v2 DATA descriptor"); + +// Live application capability advertised by `ota status` / `ota self`. Keep this tied to the actual +// receive/reassembly buffers rather than a release label so a host can safely choose package geometry. +static constexpr uint16_t ota_max_block_capability() { return (uint16_t)OTA_MAX_BLOCK; } #ifndef OTA_CHECKPOINT_BLOCKS #define OTA_CHECKPOINT_BLOCKS 4 // persist progress (store.checkpoint) every N committed blocks (resume) #endif @@ -562,7 +569,8 @@ private: static bool srcDeflateReadTramp(void* c, uint16_t block, uint8_t* buf, uint16_t cap, uint16_t* len); bool queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want_mask, - bool wire_v2 = false, bool allow_deflate = false); + bool wire_v2 = false, bool allow_deflate = false, + bool extended_length = false); bool queueManifestJob(const uint8_t* mid, uint16_t want_mask); uint32_t manifestEgressGapMs() const; uint32_t proofEgressGapMs() const; @@ -625,7 +633,7 @@ private: uint8_t _scratch[OTA_PROOFGEN_SCRATCH]; // proof-gen / fetch root-check working buffer #endif - // Server-side response descriptors are tiny. The active 1 KB block has its own buffer so proof generation + // Server-side response descriptors are tiny. The active logical block has its own buffer so proof generation // or a simultaneous fetch cannot overwrite DATA retained behind radio-queue backpressure. struct ServeJob { uint8_t mid[4] = {0}; @@ -636,6 +644,7 @@ private: bool proof_requested = false; bool wire_v2 = false; bool allow_deflate = false; + bool extended_length = false; }; ServeJob _serve_jobs[OTA_SERVE_QUEUE]; uint8_t _n_serve_jobs = 0; @@ -698,7 +707,7 @@ private: // Bounded adaptive multi-block client flight. Each slot independently reassembles DATA and awaits its // Merkle proof. One append-only OTA_REQ packet opens the whole flight; the receiver then stays quiet until // every slot completes or an airtime-aware deadline expires. Clean flights grow 1->2->3->4; any recovery - // makes the next flight smaller. Logical/signed block geometry remains 1 KB. + // makes the next flight smaller. New application containers use 2 KiB logical/signed blocks. struct ReassemblySlot { uint32_t block = NO_BLOCK; uint16_t mask = 0; // received FRAG_DATA-slice bitmap diff --git a/src/helpers/ota/OtaProtocol.h b/src/helpers/ota/OtaProtocol.h index b4ffc211..5af28b71 100644 --- a/src/helpers/ota/OtaProtocol.h +++ b/src/helpers/ota/OtaProtocol.h @@ -71,9 +71,11 @@ struct ManifestMsg { // // Legacy want_mask bit k = "send fragment k" (the slice at byte offset k*OTA_FRAG_DATA). A recovery request // uses only the missing bits, so one lost fragment costs one fragment to re-send rather than the whole block. -// New receivers reserve bits 15..14 as an in-band request profile: bit 15 asks for the v2 171-byte geometry -// and bit 14 permits per-block raw DEFLATE. An old source masks those bits away and returns the ordinary -// 160-byte DATA train; the new receiver detects that response and downgrades without stranding old fleets. +// New receivers reserve bits 15..13 as an in-band request profile: bit 15 asks for the v2 171-byte geometry, +// bit 14 permits per-block raw DEFLATE, and bit 13 selects the 2 KiB descriptor layout. Bit 12 must be clear +// in that extended profile, preserving the deployed 0xFFFF legacy "all fragments" request. An old source +// masks the profile bits away and returns ordinary 160-byte DATA; the new receiver detects that response and +// downgrades without stranding old fleets. struct ReqMsg { uint8_t manifest_id[4]; uint16_t block_idx; uint16_t want_mask; }; struct ReqItem { uint16_t block_idx; uint16_t want_mask; }; #ifndef OTA_REQ_MAX_ITEMS @@ -101,43 +103,65 @@ struct DataMsg { const uint8_t* data; uint16_t data_len; }; -// OTA_REQ want_mask extension. The v2 marker and DEFLATE permission are deliberately outside every valid -// fragment bit for the fixed <=1 KiB blocks used by this protocol. A v2 request includes all seven legacy -// fragment bits on its first attempt, so a deployed source which ignores these flags can answer completely. +// OTA_REQ want_mask extension. The profile bits are deliberately outside every valid fragment bit: a 2 KiB +// block needs at most 13 legacy 160-byte fragments or 12 v2 171-byte fragments. A first request includes every +// legacy fragment bit, so a deployed source which ignores the profile flags can still answer a 1 KiB block. static const uint16_t OTA_REQ_V2_MARK = 0x8000u; static const uint16_t OTA_REQ_V2_ALLOW_DEFLATE = 0x4000u; -static const uint16_t OTA_REQ_V2_RESERVED = 0x2000u; +static const uint16_t OTA_REQ_V2_EXTENDED_LENGTH = 0x2000u; static const uint16_t OTA_REQ_V2_FRAGMENT_MASK = 0x1FFFu; +static const uint16_t OTA_REQ_V2_EXT_FRAGMENT_MASK = 0x0FFFu; +static const uint16_t OTA_REQ_V2_EXT_RESERVED = 0x1000u; -// OTA_DATA v2 frag_off packing: +// OTA_DATA v2 frag_off packing has two negotiated layouts. The deployed-compatible 1 KiB layout is retained: // bit 15 v2 marker -// bit 14 data[] is one raw-RFC1951-DEFLATE stream fragment (clear = raw block bytes) -// bits 13..10 fragment index (0..15; byte offset = index * OTA_FRAG_DATA_V2) -// bits 9..0 total encoded block length minus one (1..1024 bytes) +// bit 14 raw-RFC1951-DEFLATE stream (clear = raw block bytes) +// bits 13..10 fragment index; bits 9..0 encoded length minus one (1..1024) +// A request carrying OTA_REQ_V2_EXTENDED_LENGTH selects the 2 KiB layout: +// bit 15 v2 marker +// bits 14..11 fragment index; bits 10..0 encoded length minus one (1..2048) +// In the extended layout DEFLATE is unambiguous without a flag: encoders may select it only when the encoded +// stream is strictly shorter than the manifest-derived raw block. Equal length therefore means raw bytes. static const uint16_t OTA_DATA_V2_MARK = 0x8000u; static const uint16_t OTA_DATA_V2_DEFLATED = 0x4000u; static const uint16_t OTA_DATA_V2_FRAGMENT_BITS = 0x3C00u; static const uint16_t OTA_DATA_V2_LENGTH_BITS = 0x03FFu; static const uint8_t OTA_DATA_V2_FRAGMENT_SHIFT = 10; -static const uint16_t OTA_DATA_V2_MAX_ENCODED = 1024; +static const uint16_t OTA_DATA_V2_LEGACY_MAX_ENCODED = 1024; +static const uint16_t OTA_DATA_V2_EXT_FRAGMENT_BITS = 0x7800u; +static const uint16_t OTA_DATA_V2_EXT_LENGTH_BITS = 0x07FFu; +static const uint8_t OTA_DATA_V2_EXT_FRAGMENT_SHIFT = 11; +static const uint16_t OTA_DATA_V2_MAX_ENCODED = 2048; static const uint8_t OTA_DATA_V2_STREAM_ID_BYTES = 4; inline bool ota_req_is_v2(uint16_t want_mask) { - return (want_mask & OTA_REQ_V2_MARK) != 0 && (want_mask & OTA_REQ_V2_RESERVED) == 0; + if ((want_mask & OTA_REQ_V2_MARK) == 0) return false; + return (want_mask & OTA_REQ_V2_EXTENDED_LENGTH) == 0 || + (want_mask & OTA_REQ_V2_EXT_RESERVED) == 0; } inline uint16_t ota_req_v2_fragments(uint16_t want_mask) { - return (uint16_t)(want_mask & OTA_REQ_V2_FRAGMENT_MASK); + const uint16_t mask = (want_mask & OTA_REQ_V2_EXTENDED_LENGTH) + ? OTA_REQ_V2_EXT_FRAGMENT_MASK : OTA_REQ_V2_FRAGMENT_MASK; + return (uint16_t)(want_mask & mask); } -inline uint16_t ota_req_make_v2(uint16_t fragments, bool allow_deflate) { - return (uint16_t)((fragments & OTA_REQ_V2_FRAGMENT_MASK) | OTA_REQ_V2_MARK | - (allow_deflate ? OTA_REQ_V2_ALLOW_DEFLATE : 0)); +inline bool ota_req_v2_extended_length(uint16_t want_mask) { + return (want_mask & OTA_REQ_V2_EXTENDED_LENGTH) != 0; +} + +inline uint16_t ota_req_make_v2(uint16_t fragments, bool allow_deflate, + bool extended_length = false) { + const uint16_t fragment_mask = extended_length + ? OTA_REQ_V2_EXT_FRAGMENT_MASK : OTA_REQ_V2_FRAGMENT_MASK; + return (uint16_t)((fragments & fragment_mask) | OTA_REQ_V2_MARK | + (allow_deflate ? OTA_REQ_V2_ALLOW_DEFLATE : 0) | + (extended_length ? OTA_REQ_V2_EXTENDED_LENGTH : 0)); } inline bool ota_data_v2_pack(uint8_t fragment, uint16_t encoded_len, bool deflated, uint16_t& packed) { - if (fragment >= 16 || encoded_len == 0 || encoded_len > OTA_DATA_V2_MAX_ENCODED) return false; + if (fragment >= 16 || encoded_len == 0 || encoded_len > OTA_DATA_V2_LEGACY_MAX_ENCODED) return false; packed = (uint16_t)(OTA_DATA_V2_MARK | (deflated ? OTA_DATA_V2_DEFLATED : 0) | ((uint16_t)fragment << OTA_DATA_V2_FRAGMENT_SHIFT) | @@ -154,6 +178,27 @@ inline bool ota_data_v2_unpack(uint16_t packed, uint8_t& fragment, uint16_t& enc return true; } +inline bool ota_data_v2_pack_extended(uint8_t fragment, uint16_t encoded_len, + uint16_t& packed) { + if (fragment >= 16 || encoded_len == 0 || encoded_len > OTA_DATA_V2_MAX_ENCODED) return false; + packed = (uint16_t)(OTA_DATA_V2_MARK | + ((uint16_t)fragment << OTA_DATA_V2_EXT_FRAGMENT_SHIFT) | + (encoded_len - 1u)); + return true; +} + +inline bool ota_data_v2_unpack_extended(uint16_t packed, uint16_t raw_len, + uint8_t& fragment, uint16_t& encoded_len, + bool& deflated) { + if ((packed & OTA_DATA_V2_MARK) == 0 || raw_len == 0 || raw_len > OTA_DATA_V2_MAX_ENCODED) return false; + fragment = (uint8_t)((packed & OTA_DATA_V2_EXT_FRAGMENT_BITS) >> + OTA_DATA_V2_EXT_FRAGMENT_SHIFT); + encoded_len = (uint16_t)((packed & OTA_DATA_V2_EXT_LENGTH_BITS) + 1u); + if (encoded_len > raw_len) return false; + deflated = encoded_len < raw_len; + return true; +} + // ---- OTA_GET_LEAVES: request the target's merkle leaves[] in bulk (direct; motatool warm-start only) ---- // body: manifest_id(4) want_mask(2). want_mask bit k = "send leaves fragment k"; 0xFFFF = "send all" on the // first ask (before frag_total is known); a retry requests only the still-missing fragments (anti-burst, diff --git a/src/helpers/ota/OtaSelf.cpp b/src/helpers/ota/OtaSelf.cpp index 3d1fc7ff..5efed279 100644 --- a/src/helpers/ota/OtaSelf.cpp +++ b/src/helpers/ota/OtaSelf.cpp @@ -144,8 +144,9 @@ bool ota_serve_self(OtaContext& c, uint32_t fw_version) { #endif SelfFwInfo fi; if (!ota_self_firmware(fi) || !fi.valid) return false; - // 1 KB logical blocks (delivered as multiple LoRa fragments): 8x fewer merkle leaves than 128 B, so a - // ~530 KB image is ~518 blocks (proof-gen scratch ~2 KB) instead of ~4150 (which overflowed the scratch). + // 2 KiB logical blocks (delivered as multiple LoRa fragments) reduce Merkle/proof overhead and give each + // independent transport-DEFLATE stream more useful history than the deployed 1 KiB geometry. A ~530 KiB + // image is about 265 blocks (roughly 1 KiB of leaves) instead of ~4,240 128-byte leaves. const uint32_t image_size = fi.image_len, BS = OTA_DEFAULT_BLOCK_SIZE; const uint32_t bc = (image_size + BS - 1) / BS; if ((uint64_t)bc * 4 > OTA_SELF_LEAVES_MAX) return false; @@ -186,7 +187,7 @@ bool ota_serve_self(OtaContext& c, uint32_t fw_version) { m[0] = MOTA_FORMAT_VER; m[1] = MFLAG_FULL; m[2] = HASH_ALGO_SHA256; wr_u32le(m + 3, out_target); wr_u32le(m + 7, out_ver); wr_u32le(m + 11, image_size); wr_u32le(m + 15, image_size); // full: payload == image - m[19] = 10; // block_size_log2 = 10 (1024 B logical block) + m[19] = 11; // block_size_log2 = 11 (2048 B logical block) memcpy(m + 20, root, 4); memcpy(m + 24, image_hash, 32); m[56] = CODEC_FULL; diff --git a/src/helpers/ota/OtaVerify.cpp b/src/helpers/ota/OtaVerify.cpp index 6cda29ab..02714a30 100644 --- a/src/helpers/ota/OtaVerify.cpp +++ b/src/helpers/ota/OtaVerify.cpp @@ -47,7 +47,7 @@ static VerifyResult ota_verify_store(const OtaStore& store, if (!mota_parse_manifest(manifest, sizeof(manifest), m)) return r; const uint32_t leaves_off = 8 + MOTA_MFL; const uint64_t payload_off64 = (uint64_t)leaves_off + (uint64_t)m.block_count * 4; - if (m.block_size() > OTA_DEFAULT_BLOCK_SIZE || + if (m.block_size() > OTA_MAX_APPLICATION_BLOCK_SIZE || payload_off64 > UINT32_MAX || payload_off64 + m.payload_size + 5 != total) return r; const uint32_t payload_off = (uint32_t)payload_off64; diff --git a/test/test_ota/mota_vectors.h b/test/test_ota/mota_vectors.h index b9885946..8d0d862d 100644 --- a/test/test_ota/mota_vectors.h +++ b/test/test_ota/mota_vectors.h @@ -99,6 +99,17 @@ static const uint8_t SIM_MOTA_1K[2278] = {109,79,84,65,230,8,0,0,2,3,18,190,186, static const uint32_t SIM_MOTA_1K_LEN = 2278; static const uint32_t SIM_MOTA_1K_BLOCKS = 3; +// 2 KiB-block signed .mota: one full block plus a short tail +static const uint8_t SIM_MOTA_2K[2459] = {109,79,84,65,155,9,0,0,2,3,18,190,186,254,202,0,0,1,3,193,8,0,0,193,8,0,0,11,215,231,3,172,108,182,235,118,117,144,154,116,108,17,3,95,148,199,59,101,154,141,10,217,94,113,26,75,186,162,4,12,251,124,40,158,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,161,7,191,243,206,16,190,29,112,221,24,231,75,192,153,103,228,214,48,155,165,13,95,29,220,134,100,18,85,49,184,112,233,110,114,111,38,182,17,125,35,32,237,155,232,221,22,219,162,105,92,72,126,98,190,122,199,153,55,34,3,96,147,196,73,213,151,142,144,239,68,90,135,125,89,141,89,54,238,22,82,134,79,80,161,194,141,199,109,134,124,188,202,221,7,255,255,255,255,126,165,133,214,82,63,130,220,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,105,66,32,116,114,97,110,115,112,111,114,116,32,68,69,70,76,65,84,69,32,118,101,99,116,111,114,46,32,77,101,115,104,67,111,114,101,32,50,32,75,69,110,100,70,137,8,0,0,248,179,41,235,185,112,93,235,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,118,107,52,57,54}; + +static const uint32_t SIM_MOTA_2K_LEN = 2459; +static const uint32_t SIM_MOTA_2K_BLOCKS = 2; +static const uint8_t SIM_MOTA_2K_DEFLATED[127] = {243,77,45,206,112,206,47,74,85,48,82,240,206,116,82,40,41,74,204,43,46,200,47,42,81,112,113,117,243,113,12,113,85,40,75,77,46,201,47,210,83,240,29,85,56,170,112,84,225,168,194,81,133,163,10,71,21,14,23,133,0,211,83,240,77,45,206,112,206,47,74,85,48,82,240,206,116,82,40,41,74,204,43,46,200,47,42,81,112,113,117,243,113,12,113,85,40,75,77,46,201,47,210,163,177,66,215,188,20,183,78,14,6,134,31,155,53,95,239,44,136,125,205,64,36,0,0}; + +static const uint16_t SIM_MOTA_2K_DEFLATED_OFFSETS[2] = {0,61}; + +static const uint16_t SIM_MOTA_2K_DEFLATED_LENGTHS[2] = {61,66}; + // detools sequential+crle delta: apply DT_PATCH to DT_BASE -> DT_TARGET static const uint8_t DT_BASE[3056] = {120,46,186,148,77,51,227,185,104,193,183,194,67,136,62,162,208,188,127,90,106,134,186,157,246,55,79,139,180,84,132,19,187,198,255,221,52,176,192,186,119,236,181,212,223,167,37,136,54,222,105,250,14,197,89,160,106,119,31,185,190,35,195,83,99,84,88,203,51,83,109,106,81,145,54,231,222,104,58,52,10,191,57,195,4,248,221,66,216,129,81,197,245,145,205,180,107,157,28,84,217,167,155,199,59,60,254,118,93,34,51,94,126,152,214,160,36,67,99,159,86,85,240,181,255,182,119,220,43,175,178,196,220,33,84,236,52,148,175,16,25,240,215,44,1,230,38,112,180,60,89,58,20,50,205,72,61,177,118,154,67,123,134,225,111,169,248,106,51,215,18,77,77,71,34,144,169,187,64,134,25,126,55,228,50,200,99,45,131,217,57,81,90,192,179,228,204,11,148,230,238,173,139,96,239,189,184,243,162,18,30,58,14,132,32,241,212,53,232,162,157,236,22,242,129,44,60,124,149,204,187,42,41,22,32,158,26,207,241,152,143,207,254,154,161,7,152,27,143,46,139,178,80,0,31,71,7,46,15,26,162,219,159,172,158,187,53,148,53,165,48,118,47,121,80,69,187,116,162,112,213,183,206,210,55,102,150,221,114,221,107,152,179,34,225,53,41,79,101,50,192,45,91,116,175,3,30,85,172,0,197,57,192,129,107,168,249,9,54,155,118,141,127,140,206,12,110,85,2,130,87,141,249,230,240,224,65,170,187,40,57,154,248,27,211,203,216,110,17,245,225,34,44,6,220,86,78,242,234,64,126,41,92,213,119,237,111,241,125,157,83,40,9,197,31,30,93,108,162,244,46,129,180,24,13,168,111,228,6,207,233,227,240,69,59,30,81,139,222,145,35,54,145,149,27,110,26,241,153,180,178,68,111,143,40,195,59,240,0,131,31,50,96,137,146,104,114,146,201,44,212,165,236,63,141,235,195,74,230,208,70,151,91,115,163,28,103,101,194,72,81,128,133,58,60,210,199,206,91,58,107,201,119,146,79,73,245,172,175,236,49,119,165,138,13,64,97,211,166,53,67,105,132,34,167,80,72,176,137,206,241,34,195,23,129,56,118,155,71,75,63,165,132,99,189,72,244,47,246,228,233,247,122,206,93,247,7,152,165,96,177,16,193,185,231,34,25,108,156,82,48,255,196,244,244,19,194,148,65,8,206,163,198,66,171,217,133,48,241,218,204,111,42,49,182,120,211,68,17,118,31,25,151,68,33,191,98,200,250,150,212,165,25,57,193,149,60,42,75,166,229,35,209,177,238,206,98,175,27,249,33,86,105,84,161,179,85,140,210,192,60,5,154,71,97,35,145,68,43,129,204,230,65,55,225,230,140,33,179,109,189,138,48,32,208,33,172,177,59,64,6,163,157,173,28,251,108,135,107,8,121,39,70,182,92,118,88,74,123,227,173,148,114,232,141,165,8,174,233,72,47,98,166,229,126,163,92,128,124,93,240,40,16,129,188,248,249,141,68,50,46,112,119,83,159,1,36,188,29,108,12,72,193,168,191,20,181,224,21,171,122,118,241,53,134,128,172,191,216,59,171,163,169,80,167,131,162,77,124,75,148,14,173,54,172,186,138,121,182,62,79,166,255,80,64,84,10,35,159,137,202,138,66,88,88,38,13,89,3,158,80,192,80,50,47,239,72,206,116,135,12,251,40,25,201,123,163,67,100,13,105,51,174,34,34,93,3,115,92,192,106,231,63,31,87,218,156,115,83,1,188,172,152,212,206,133,105,183,160,198,113,51,32,213,29,130,254,173,239,245,104,126,255,101,127,115,186,56,236,11,217,178,248,114,215,168,163,134,209,19,244,68,229,206,194,29,94,160,221,137,217,92,17,83,227,202,123,155,151,121,220,191,199,202,241,205,234,70,164,104,185,211,96,128,163,65,230,28,4,63,219,134,230,237,183,181,229,109,41,192,38,48,3,205,3,69,15,27,85,41,249,58,60,229,132,70,199,19,39,247,213,215,10,64,234,230,209,250,91,21,134,94,143,43,27,92,64,200,3,206,207,147,130,194,137,110,114,158,192,62,200,9,173,197,36,195,222,90,68,139,0,117,232,231,62,93,112,14,37,53,86,17,150,199,31,117,20,200,185,21,58,0,193,20,229,210,41,161,119,53,225,179,50,59,120,212,104,243,159,148,20,197,167,106,194,135,192,98,133,68,210,210,227,227,161,183,180,36,62,92,189,50,39,122,5,177,246,255,185,32,70,105,173,233,213,61,174,111,102,212,174,40,98,96,113,237,146,163,30,186,69,139,67,157,37,83,30,161,170,92,20,88,126,31,148,49,32,133,35,206,117,26,155,154,102,119,184,196,144,146,104,188,112,247,229,124,82,237,146,81,215,157,134,60,215,44,249,56,157,202,243,236,105,6,130,117,190,131,93,11,78,190,251,41,176,233,229,147,70,59,56,145,165,174,147,111,242,160,241,201,203,118,88,132,232,178,22,117,168,133,225,76,86,240,4,246,158,51,191,121,245,52,164,72,112,109,131,246,21,37,123,166,143,107,76,69,128,184,87,200,23,39,48,21,138,248,161,140,8,151,174,127,172,55,217,205,164,191,41,40,21,65,102,38,231,245,193,137,82,79,200,219,185,83,168,181,87,125,247,47,112,61,195,99,115,20,131,186,193,11,9,106,72,43,175,52,216,11,212,247,27,151,35,162,53,237,20,208,201,47,218,30,26,86,4,185,26,142,255,250,59,219,212,222,152,155,131,117,49,122,51,2,78,187,225,215,90,99,196,255,71,7,19,218,205,190,132,213,178,193,114,94,81,104,244,138,51,254,116,38,23,73,190,214,169,31,220,41,172,172,205,57,159,93,237,199,135,219,63,101,42,222,90,192,55,33,98,171,182,18,47,63,214,206,6,76,251,28,178,150,174,154,63,127,87,170,172,107,230,55,12,253,165,15,70,55,191,80,114,158,203,222,55,160,212,2,147,59,21,136,61,126,64,193,123,132,221,255,44,162,133,94,13,143,204,9,115,165,105,81,111,100,16,20,44,80,204,209,171,181,144,67,195,225,253,33,209,171,193,155,183,25,142,150,232,170,231,94,127,241,58,4,97,162,46,174,96,95,35,15,98,225,46,228,222,216,179,230,92,153,238,220,253,247,20,57,60,168,242,236,186,70,34,78,184,1,123,95,111,107,91,156,13,3,71,10,125,232,250,215,249,13,223,48,132,48,156,61,129,121,46,148,222,129,228,3,189,217,19,25,6,29,125,180,13,205,152,222,79,198,73,205,97,155,156,229,15,195,235,128,254,9,137,30,63,146,187,223,104,18,14,16,87,252,21,24,8,109,65,199,160,20,169,27,205,213,146,211,166,109,129,203,224,196,235,198,154,251,215,11,228,122,121,254,118,28,97,98,132,36,48,194,97,45,127,162,252,193,144,38,51,235,54,12,75,128,236,76,100,184,5,223,113,92,189,7,229,144,154,126,214,210,14,217,255,3,240,73,227,232,82,220,33,93,55,112,7,32,76,178,247,134,245,50,157,101,68,55,237,191,114,53,220,235,56,124,190,163,8,192,216,1,26,101,153,181,14,174,1,213,255,111,61,46,65,109,153,124,203,111,65,13,206,176,115,10,245,43,152,144,152,119,211,90,150,27,127,253,111,51,198,47,247,9,224,245,233,188,100,81,18,83,249,180,142,84,249,109,215,178,75,102,203,163,250,154,185,204,231,27,239,194,79,230,120,16,249,205,89,189,108,124,23,74,196,171,118,88,103,14,22,9,250,115,195,156,95,122,201,54,2,16,21,23,8,89,182,241,168,220,85,132,206,169,166,61,26,8,208,0,235,19,187,18,27,197,95,7,147,13,48,151,71,243,218,70,3,189,109,122,228,228,24,168,225,138,111,120,81,184,200,93,61,118,227,105,174,242,24,113,22,201,161,12,120,202,28,112,5,43,46,172,168,46,226,229,44,48,20,91,10,59,228,230,236,190,97,0,217,185,27,113,192,7,100,14,89,93,107,66,95,179,55,124,165,105,168,237,38,3,204,188,169,189,35,154,233,228,249,76,132,68,255,95,118,181,182,132,234,105,157,178,176,132,195,183,64,62,170,40,20,135,99,6,238,75,241,135,101,245,19,28,239,79,117,251,11,161,103,148,230,156,97,82,245,248,36,29,77,232,24,61,115,206,10,85,185,234,141,140,183,158,86,215,114,111,36,207,92,252,210,111,241,157,99,97,115,199,170,90,232,213,30,185,191,109,54,216,177,170,225,232,90,48,170,58,215,86,94,202,54,39,191,150,47,189,120,234,161,59,91,219,95,38,160,190,181,2,118,54,254,2,89,50,63,108,80,84,9,115,125,140,243,178,117,100,5,38,237,219,200,62,254,182,2,127,93,146,47,79,13,150,88,221,178,75,232,71,43,46,81,92,163,53,180,207,202,243,40,229,120,166,221,150,6,20,155,26,22,24,92,163,85,117,214,210,191,112,30,227,92,77,38,111,245,111,152,252,10,35,129,158,153,153,20,150,170,209,86,177,142,81,169,6,55,32,170,218,25,23,47,109,30,135,107,196,169,200,79,178,229,12,27,72,239,20,61,227,166,234,174,86,177,184,197,72,161,19,162,9,20,149,29,178,171,11,75,218,23,33,14,131,126,156,122,48,129,25,221,92,199,2,61,248,38,79,17,27,76,70,170,222,166,233,192,96,154,178,148,64,21,136,4,86,222,208,52,174,19,163,74,215,130,27,118,18,129,120,192,188,202,75,171,253,246,36,163,136,10,247,100,10,119,218,67,158,103,253,48,145,245,89,110,244,154,197,9,238,221,133,47,78,160,159,204,225,4,169,209,223,136,127,179,234,176,178,32,244,109,158,123,117,143,142,122,200,37,130,29,87,226,67,209,10,52,142,29,49,199,189,107,161,95,142,101,182,230,0,3,181,31,212,100,211,194,68,88,239,122,38,199,231,62,143,195,76,198,217,49,152,198,98,169,6,111,161,208,229,37,157,26,175,58,235,194,12,130,208,47,58,96,40,177,162,78,24,167,121,80,70,17,255,86,54,223,60,120,243,107,155,52,51,131,168,140,230,94,207,167,67,162,146,199,15,19,23,147,144,161,222,118,19,234,109,214,155,142,65,191,43,26,239,28,102,167,172,222,237,26,148,36,49,105,211,242,73,60,84,80,26,221,208,83,138,48,87,250,120,37,33,218,171,246,97,171,6,247,86,59,64,65,8,160,226,204,127,68,59,77,12,12,224,140,114,52,166,11,25,44,233,41,5,3,182,252,41,105,251,215,25,153,44,189,144,150,73,1,142,202,175,232,240,68,218,90,200,93,130,163,19,144,168,137,22,244,71,172,30,111,151,230,127,28,105,156,154,140,249,220,201,203,136,93,28,117,47,165,141,138,122,248,160,217,119,113,167,209,245,182,1,252,150,25,54,36,62,194,197,136,4,190,12,126,29,230,234,246,252,254,36,99,6,237,175,42,95,94,164,0,220,157,137,123,47,38,152,42,140,245,31,54,38,211,14,178,164,2,211,234,250,121,117,69,98,190,78,20,238,181,194,96,144,209,138,137,232,226,205,225,115,24,57,129,221,89,191,35,96,149,25,151,73,139,213,145,7,162,175,53,142,12,248,115,235,28,27,222,249,253,184,233,157,176,24,138,48,219,6,132,62,41,213,69,73,110,37,72,178,91,57,202,151,2,180,18,216,169,25,181,105,47,144,143,2,187,101,193,163,107,54,42,18,254,14,108,234,166,29,33,219,251,212,119,76,247,204,242,159,49,224,204,189,75,30,135,183,182,147,252,14,187,185,86,6,228,160,111,45,122,79,195,199,30,106,201,237,247,201,72,29,186,226,152,54,181,133,122,118,173,45,197,146,66,247,83,25,176,166,117,207,96,149,25,198,175,41,154,59,109,225,184,107,236,150,25,34,202,13,43,92,96,249,85,73,54,22,172,123,6,174,149,76,59,19,63,195,115,254,12,143,253,116,104,204,9,75,118,200,248,23,178,80,193,61,48,128,232,47,156,18,18,158,148,238,118,11,167,70,41,112,70,48,178,1,174,82,110,22,16,245,110,154,81,216,72,211,37,132,72,185,141,157,102,134,78,205,218,30,109,59,236,102,25,38,221,165,122,246,131,120,66,137,225,50,184,190,74,195,26,32,155,162,132,142,156,141,32,2,209,213,139,198,59,168,211,168,199,46,189,74,203,219,127,108,221,43,124,185,228,187,250,239,195,8,221,31,84,161,24,80,133,244,168,50,199,166,173,233,92,225,38,1,54,145,7,160,122,105,0,32,234,2,89,146,61,78,132,101,85,31,164,226,151,123,53,197,167,56,212,124,226,220,252,7,161,98,26,46,140,40,136,42,158,224,188,172,96,245,228,231,224,19,99,127,123,162,38,111,213,4,169,9,38,253,178,208,83,237,62,224,131,82,90,104,235,13,88,191,215,199,182,210,225,25,64,216,125,102,252,136,181,125,116,52,11,149,169,37,139,183,73,178,160,210,184,143,164,202,70,104,52,141,189,138,2,105,15,214,66,15,180,136,205,72,233,248,103,141,34,179,176,190,125,226,194,107,143,147,164,226,90,12,17,135,137,31,15,146,15,78,151,177,88,61,19,157,195,159,16,1,144,109,100,25,169,117,64,189,36,176,62,18,199,120,137,164,82,192,209,137,235,200,110,117,175,96,198,31,20,251,69,110,100,70,184,11,0,0,206,99,32,159,72,64,138,233,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; diff --git a/test/test_ota/test_ota_core.cpp b/test/test_ota/test_ota_core.cpp index ebb55972..a43defbc 100644 --- a/test/test_ota/test_ota_core.cpp +++ b/test/test_ota/test_ota_core.cpp @@ -64,6 +64,12 @@ TEST(OtaPolicy, TrustedAutoInstallIsStrictlyForwardOnly) { EXPECT_TRUE(ota_trusted_auto_version_allows(0x01170103u, 0x01170104u)); } +TEST(OtaCapability, LiveMaximumTracksTheApplicationReassemblyLimit) { + EXPECT_EQ(ota_max_block_capability(), 2048u); + EXPECT_EQ(ota_max_block_capability(), OTA_MAX_BLOCK); + EXPECT_LE(ota_max_block_capability(), OTA_DATA_V2_MAX_ENCODED); +} + TEST(OtaBootPackage, StorageProfilesAndSdGeometryAreExact) { EXPECT_EQ(OTA_BL_PROFILE_SD_BOOT_UPDATE, 0x09u); EXPECT_EQ(OTA_BL_PROFILE_INTERNAL_BOOT_UPDATE, 0x0Au); @@ -461,6 +467,7 @@ TEST(OtaBootPackage, ParserSeparatesV2ApplicationsFromStrictV3Bootloader) { auto m = boot_manifest_bytes(); ASSERT_TRUE(mota_parse_manifest(m.data(), m.size(), parsed)); EXPECT_TRUE(parsed.is_bootloader()); + EXPECT_EQ(parsed.block_size(), 1024u); EXPECT_EQ(parsed.block_count, 40u); m[0] = MOTA_APP_FORMAT_VER; @@ -469,6 +476,8 @@ TEST(OtaBootPackage, ParserSeparatesV2ApplicationsFromStrictV3Bootloader) { EXPECT_FALSE(mota_parse_manifest(m.data(), m.size(), parsed)); m = boot_manifest_bytes(); m[19] = 9; EXPECT_FALSE(mota_parse_manifest(m.data(), m.size(), parsed)); + m = boot_manifest_bytes(); m[19] = 11; + EXPECT_FALSE(mota_parse_manifest(m.data(), m.size(), parsed)); m = boot_manifest_bytes(); memset(m.data() + 7, 0, 4); EXPECT_FALSE(mota_parse_manifest(m.data(), m.size(), parsed)); m = boot_manifest_bytes(); m[MOTA_OFF_BASE_HASH] = 1; @@ -1098,6 +1107,22 @@ TEST(OtaParse, RejectsTooManyBlocks) { EXPECT_EQ(mm.block_count, 65535u); } +TEST(OtaParse, BlockCountCeilingDoesNotOverflowAtUint32Maximum) { + std::vector manifest(MOTA_MFL, 0); + manifest[0] = MOTA_FORMAT_VER; + manifest[1] = MFLAG_FULL; + manifest[2] = HASH_ALGO_SHA256; + manifest[15] = 0xFF; + manifest[16] = 0xFF; + manifest[17] = 0xFF; + manifest[18] = 0xFF; + manifest[19] = 24; // largest parser-valid block: ceil(UINT32_MAX / 2^24) = 256 + + MotaManifest parsed; + ASSERT_TRUE(mota_parse_manifest(manifest.data(), manifest.size(), parsed)); + EXPECT_EQ(parsed.block_count, 256u); +} + TEST(OtaMerkle, RootMatchesVectorAndLeaves) { MotaManifest m; ASSERT_TRUE(mota_parse(MOTA_VEC, MOTA_VEC_LEN, m)); @@ -1680,12 +1705,21 @@ TEST(OtaProtocol, CodecRoundTrips) { EXPECT_EQ(0, memcmp(pm2.proof, proof, 12)); } -TEST(OtaProtocol, V2ProfilePacks171ByteFragmentsInsideExistingMessages) { +TEST(OtaProtocol, V2ProfilesPackLegacyAndExtendedLengthBoundaries) { const uint16_t request = ota_req_make_v2(0x007Fu, true); EXPECT_TRUE(ota_req_is_v2(request)); EXPECT_EQ(ota_req_v2_fragments(request), 0x007Fu); EXPECT_NE(request & OTA_REQ_V2_ALLOW_DEFLATE, 0u); - EXPECT_FALSE(ota_req_is_v2((uint16_t)(request | OTA_REQ_V2_RESERVED))); + EXPECT_FALSE(ota_req_v2_extended_length(request)); + + // Extended v2 needs at most twelve 171-byte fragments. Keeping fragment bit 12 clear leaves the + // deployed legacy "send all" value 0xFFFF unambiguous. + const uint16_t extended_request = ota_req_make_v2(0x0FFFu, true, true); + EXPECT_TRUE(ota_req_is_v2(extended_request)); + EXPECT_TRUE(ota_req_v2_extended_length(extended_request)); + EXPECT_EQ(ota_req_v2_fragments(extended_request), 0x0FFFu); + EXPECT_EQ(extended_request, 0xEFFFu); + EXPECT_FALSE(ota_req_is_v2(0xFFFFu)); // deployed legacy full-block request EXPECT_FALSE(ota_req_is_v2(0x007Fu)); uint16_t descriptor = 0; @@ -1697,10 +1731,44 @@ TEST(OtaProtocol, V2ProfilePacks171ByteFragmentsInsideExistingMessages) { EXPECT_EQ(fragment, 5u); EXPECT_EQ(encoded_len, 1024u); EXPECT_TRUE(deflated); + EXPECT_FALSE(ota_data_v2_unpack((uint16_t)(descriptor & ~OTA_DATA_V2_MARK), + fragment, encoded_len, deflated)); EXPECT_FALSE(ota_data_v2_pack(16, 1024, false, descriptor)); EXPECT_FALSE(ota_data_v2_pack(0, 0, false, descriptor)); EXPECT_FALSE(ota_data_v2_pack(0, 1025, false, descriptor)); + ASSERT_TRUE(ota_data_v2_pack_extended(15, 2048, descriptor)); + EXPECT_EQ(descriptor, 0xFFFFu); + ASSERT_TRUE(ota_data_v2_unpack_extended(descriptor, 2048, + fragment, encoded_len, deflated)); + EXPECT_EQ(fragment, 15u); + EXPECT_EQ(encoded_len, 2048u); + EXPECT_FALSE(deflated); + EXPECT_FALSE(ota_data_v2_unpack_extended(descriptor, 2047, + fragment, encoded_len, deflated)); + + ASSERT_TRUE(ota_data_v2_pack_extended(0, 1024, descriptor)); + ASSERT_TRUE(ota_data_v2_unpack_extended(descriptor, 2048, + fragment, encoded_len, deflated)); + EXPECT_EQ(fragment, 0u); + EXPECT_EQ(encoded_len, 1024u); + EXPECT_TRUE(deflated); + + ASSERT_TRUE(ota_data_v2_pack_extended(0, 193, descriptor)); + ASSERT_TRUE(ota_data_v2_unpack_extended(descriptor, 193, + fragment, encoded_len, deflated)); + EXPECT_EQ(encoded_len, 193u); + EXPECT_FALSE(deflated); // raw short tail in a 2 KiB manifest + EXPECT_FALSE(ota_data_v2_pack_extended(16, 2048, descriptor)); + EXPECT_FALSE(ota_data_v2_pack_extended(0, 0, descriptor)); + EXPECT_FALSE(ota_data_v2_pack_extended(0, 2049, descriptor)); + EXPECT_FALSE(ota_data_v2_unpack_extended(0, 2048, + fragment, encoded_len, deflated)); + EXPECT_FALSE(ota_data_v2_unpack_extended(OTA_DATA_V2_MARK, 0, + fragment, encoded_len, deflated)); + EXPECT_FALSE(ota_data_v2_unpack_extended(OTA_DATA_V2_MARK, 2049, + fragment, encoded_len, deflated)); + std::array payload{}; ASSERT_TRUE(ota_data_v2_pack(0, 1024, false, descriptor)); DataMsg message{{1, 2, 3, 4}, 9, descriptor, payload.data(), @@ -1724,6 +1792,57 @@ static bool sim_send(void* ctx, const uint8_t* msg, uint16_t len, bool /*flood*/ g_q.push_back({((SendTo*)ctx)->dest, std::vector(msg, msg + len)}); return true; } +struct ExtendedV2Trace { + const MotaManifest* manifest = nullptr; + uint32_t extended_requests = 0; + uint32_t legacy_requests = 0; + bool full_raw = false; + bool tail_raw = false; + bool full_deflated = false; + bool tail_deflated = false; +}; +struct TracedSendTo { OtaManager* dest; ExtendedV2Trace* trace; }; +static bool traced_sim_send(void* ctx, const uint8_t* msg, uint16_t len, bool /*flood*/) { + TracedSendTo* route = (TracedSendTo*)ctx; + ExtendedV2Trace* trace = route->trace; + if (trace && trace->manifest && ota_msg_type(msg, len) == OTA_REQ) { + ReqWindowMsg request{}; + if (decode_req_window(msg, len, request)) { + for (uint8_t i = 0; i < request.n_items; i++) { + if (ota_req_is_v2(request.items[i].want_mask) && + ota_req_v2_extended_length(request.items[i].want_mask)) { + trace->extended_requests++; + } else if (!ota_req_is_v2(request.items[i].want_mask)) { + trace->legacy_requests++; + } + } + } + } else if (trace && trace->manifest && ota_msg_type(msg, len) == OTA_DATA) { + DataMsg data{}; + if (decode_data(msg, len, data) && (data.frag_off & OTA_DATA_V2_MARK) != 0 && + data.block_idx < trace->manifest->block_count) { + const uint32_t block_offset = (uint32_t)data.block_idx * trace->manifest->block_size(); + const uint16_t raw_len = (uint16_t)std::min( + trace->manifest->block_size(), trace->manifest->payload_size - block_offset); + uint8_t fragment = 0; + uint16_t encoded_len = 0; + bool deflated = false; + if (ota_data_v2_unpack_extended(data.frag_off, raw_len, + fragment, encoded_len, deflated)) { + const bool full = raw_len == trace->manifest->block_size(); + if (deflated) { + if (full) trace->full_deflated = true; + else trace->tail_deflated = true; + } else { + if (full) trace->full_raw = true; + else trace->tail_raw = true; + } + } + } + } + g_q.push_back({route->dest, std::vector(msg, msg + len)}); + return true; +} struct CapturedMessages { std::vector> items; }; static bool capture_send(void* ctx, const uint8_t* msg, uint16_t len, bool /*flood*/) { ((CapturedMessages*)ctx)->items.emplace_back(msg, msg + len); @@ -1783,6 +1902,67 @@ static bool test_wire_decode(void* context, const uint8_t* source, uint16_t sour return true; } +struct VectorDeflateCodec { + const MotaManifest* manifest = nullptr; + uint32_t calls = 0; +}; + +static bool vector_2k_wire_encode(void* context, const uint8_t* source, uint16_t source_len, + uint8_t* destination, uint16_t capacity, + uint16_t* encoded_len) { + VectorDeflateCodec* codec = (VectorDeflateCodec*)context; + if (!codec || !codec->manifest || !source || !destination || !encoded_len || + codec->manifest->block_count != SIM_MOTA_2K_BLOCKS) return false; + for (uint32_t block = 0; block < codec->manifest->block_count; block++) { + const uint32_t offset = block * codec->manifest->block_size(); + const uint16_t length = (uint16_t)std::min( + codec->manifest->block_size(), codec->manifest->payload_size - offset); + if (length != source_len || memcmp(source, codec->manifest->payload + offset, length) != 0) continue; + const uint16_t representation_len = SIM_MOTA_2K_DEFLATED_LENGTHS[block]; + if (representation_len > capacity) return false; + memcpy(destination, + SIM_MOTA_2K_DEFLATED + SIM_MOTA_2K_DEFLATED_OFFSETS[block], + representation_len); + *encoded_len = representation_len; + codec->calls++; + return true; + } + return false; +} + +struct RejectWireDecode { + uint32_t calls = 0; +}; + +static bool reject_wire_decode(void* context, const uint8_t*, uint16_t, + uint8_t*, uint16_t, uint16_t* decoded_len) { + RejectWireDecode* decoder = (RejectWireDecode*)context; + if (decoder) decoder->calls++; + if (decoded_len) *decoded_len = 0; + return false; +} + +struct SyntheticRepresentationDecode { + const uint8_t* representation = nullptr; + uint16_t representation_len = 0; + const uint8_t* decoded = nullptr; + uint16_t decoded_len = 0; + uint32_t calls = 0; +}; + +static bool synthetic_representation_decode(void* context, const uint8_t* source, + uint16_t source_len, uint8_t* destination, + uint16_t capacity, uint16_t* decoded_len) { + SyntheticRepresentationDecode* decoder = (SyntheticRepresentationDecode*)context; + if (!decoder || !source || !destination || !decoded_len || + source_len != decoder->representation_len || capacity != decoder->decoded_len || + memcmp(source, decoder->representation, source_len) != 0) return false; + memcpy(destination, decoder->decoded, capacity); + *decoded_len = capacity; + decoder->calls++; + return true; +} + static bool test_representation_decode(void* context, const uint8_t* source, uint16_t source_len, uint8_t* destination, uint16_t capacity, uint16_t* decoded_len) { @@ -2000,10 +2180,11 @@ TEST(OtaTransfer, TwoManagersFullTransfer) { EXPECT_TRUE(mota_check_image_hash_full(m)); } -TEST(OtaTransfer, TransportCodecInflatesBeforeProofAndStagesOriginalContainer) { +TEST(OtaTransfer, LegacyOneKilobyteV2CodecStillStagesOriginalContainer) { g_q.clear(); MotaManifest manifest; ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); + ASSERT_EQ(manifest.block_size(), 1024u); OtaManager server, client; OtaStoreRam<4096> store; SendTo to_client{&client}, to_server{&server}; @@ -2030,6 +2211,78 @@ TEST(OtaTransfer, TransportCodecInflatesBeforeProofAndStagesOriginalContainer) { EXPECT_EQ(0, memcmp(store.data(), SIM_MOTA_1K, SIM_MOTA_1K_LEN)); } +TEST(OtaTransfer, TwoKilobyteExtendedV2TransfersFullAndShortRawBlocks) { + g_q.clear(); + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA_2K, SIM_MOTA_2K_LEN, manifest)); + ASSERT_EQ(manifest.block_size(), 2048u); + ASSERT_EQ(manifest.block_count, SIM_MOTA_2K_BLOCKS); + ASSERT_EQ(manifest.block_count, 2u); + ASSERT_GT(manifest.payload_size, manifest.block_size()); + ASSERT_LT(manifest.payload_size - manifest.block_size(), manifest.block_size()); + + OtaManager server, client; + OtaStoreRam<4096> store; + ExtendedV2Trace trace{}; + trace.manifest = &manifest; + TracedSendTo to_client{&client, &trace}, to_server{&server, &trace}; + server.begin(0, traced_sim_send, &to_client); + client.begin(SIM_TARGET_ID, traced_sim_send, &to_server); + client.set_fetch_store(&store); + client.set_autofetch(OtaManager::AUTOFETCH_ANY); + ASSERT_TRUE(server.serve(SIM_MOTA_2K, SIM_MOTA_2K_LEN)); + + server.announce(); + pump(client, &server); + + EXPECT_EQ(client.fetchState(), OtaManager::COMPLETE); + EXPECT_GT(trace.extended_requests, 0u); + EXPECT_EQ(trace.legacy_requests, 0u); + EXPECT_TRUE(trace.full_raw); + EXPECT_TRUE(trace.tail_raw); + EXPECT_FALSE(trace.full_deflated); + EXPECT_FALSE(trace.tail_deflated); + ASSERT_EQ(store.staged_size(), SIM_MOTA_2K_LEN); + EXPECT_EQ(0, memcmp(store.data(), SIM_MOTA_2K, SIM_MOTA_2K_LEN)); +} + +TEST(OtaTransfer, TwoKilobyteExtendedV2InflatesFullAndShortBlocksBeforeProof) { + g_q.clear(); + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA_2K, SIM_MOTA_2K_LEN, manifest)); + ASSERT_EQ(manifest.block_size(), 2048u); + ASSERT_EQ(manifest.block_count, SIM_MOTA_2K_BLOCKS); + + OtaManager server, client; + OtaStoreRam<4096> store; + ExtendedV2Trace trace{}; + trace.manifest = &manifest; + TracedSendTo to_client{&client, &trace}, to_server{&server, &trace}; + VectorDeflateCodec encoder{}; + encoder.manifest = &manifest; + server.begin(0, traced_sim_send, &to_client); + server.set_transport_deflate_encoder(vector_2k_wire_encode, &encoder); + client.begin(SIM_TARGET_ID, traced_sim_send, &to_server); + client.set_transport_deflate_decoder(ota_transport_inflate); + client.set_fetch_store(&store); + client.set_autofetch(OtaManager::AUTOFETCH_ANY); + ASSERT_TRUE(server.serve(SIM_MOTA_2K, SIM_MOTA_2K_LEN)); + + server.announce(); + pump(client, &server); + + EXPECT_EQ(client.fetchState(), OtaManager::COMPLETE); + EXPECT_EQ(encoder.calls, manifest.block_count); + EXPECT_GT(trace.extended_requests, 0u); + EXPECT_EQ(trace.legacy_requests, 0u); + EXPECT_TRUE(trace.full_deflated); + EXPECT_TRUE(trace.tail_deflated); + EXPECT_FALSE(trace.full_raw); + EXPECT_FALSE(trace.tail_raw); + ASSERT_EQ(store.staged_size(), SIM_MOTA_2K_LEN); + EXPECT_EQ(0, memcmp(store.data(), SIM_MOTA_2K, SIM_MOTA_2K_LEN)); +} + static void deliver_manifest_fragment(OtaManager& client, const uint8_t mid[4], uint8_t frag_idx, const uint8_t* bytes, uint16_t len) { uint8_t wire[MAX_PACKET_PAYLOAD]; @@ -2103,7 +2356,11 @@ static void deliver_verified_v2_raw_block(OtaManager& client, const MotaManifest DataMsg data{}; memcpy(data.manifest_id, manifest.merkle_root, 4); data.block_idx = (uint16_t)block; - ASSERT_TRUE(ota_data_v2_pack((uint8_t)fragment, block_len, false, data.frag_off)); + if (block_size > OTA_DATA_V2_LEGACY_MAX_ENCODED) { + ASSERT_TRUE(ota_data_v2_pack_extended((uint8_t)fragment, block_len, data.frag_off)); + } else { + ASSERT_TRUE(ota_data_v2_pack((uint8_t)fragment, block_len, false, data.frag_off)); + } data.data = body; data.data_len = (uint16_t)(sizeof(stream_id) + length); const uint16_t wire_len = encode_data(wire, sizeof(wire), data); @@ -2124,6 +2381,65 @@ static void deliver_verified_v2_raw_block(OtaManager& client, const MotaManifest ASSERT_TRUE(client.on_message(wire, wire_len)); } +static void deliver_v2_representation(OtaManager& client, const MotaManifest& manifest, + uint32_t block, const uint8_t* representation, + uint16_t representation_len) { + ASSERT_LT(block, manifest.block_count); + ASSERT_NE(representation, nullptr); + const uint32_t block_offset = block * manifest.block_size(); + const uint16_t raw_len = (uint16_t)std::min( + manifest.block_size(), manifest.payload_size - block_offset); + ASSERT_GT(representation_len, 0u); + ASSERT_LT(representation_len, raw_len); + + uint8_t stream_id[OTA_DATA_V2_STREAM_ID_BYTES]; + mh4(stream_id, representation, representation_len); + const uint32_t fragments = + (representation_len + OTA_FRAG_DATA_V2 - 1) / OTA_FRAG_DATA_V2; + uint8_t wire[MAX_PACKET_PAYLOAD]; + for (uint32_t fragment = 0; fragment < fragments; fragment++) { + const uint32_t offset = fragment * OTA_FRAG_DATA_V2; + const uint16_t length = (uint16_t)std::min( + OTA_FRAG_DATA_V2, representation_len - offset); + uint8_t body[OTA_DATA_V2_STREAM_ID_BYTES + OTA_FRAG_DATA_V2]; + memcpy(body, stream_id, sizeof(stream_id)); + memcpy(body + sizeof(stream_id), representation + offset, length); + DataMsg data{}; + memcpy(data.manifest_id, manifest.merkle_root, 4); + data.block_idx = (uint16_t)block; + if (manifest.block_size() > OTA_DATA_V2_LEGACY_MAX_ENCODED) { + ASSERT_TRUE(ota_data_v2_pack_extended( + (uint8_t)fragment, representation_len, data.frag_off)); + } else { + ASSERT_TRUE(ota_data_v2_pack( + (uint8_t)fragment, representation_len, true, data.frag_off)); + } + data.data = body; + data.data_len = (uint16_t)(sizeof(stream_id) + length); + const uint16_t wire_len = encode_data(wire, sizeof(wire), data); + ASSERT_GT(wire_len, 0); + ASSERT_TRUE(client.on_message(wire, wire_len)); + } +} + +static void deliver_block_proof(OtaManager& client, const MotaManifest& manifest, + uint32_t block) { + ASSERT_LT(block, manifest.block_count); + std::vector scratch(manifest.block_count * 4); + uint8_t siblings[32 * 4]; + const uint8_t sibling_count = merkle_gen_proof( + manifest.leaves, manifest.block_count, block, scratch.data(), siblings); + ProofMsg proof{}; + memcpy(proof.manifest_id, manifest.merkle_root, 4); + proof.block_idx = (uint16_t)block; + proof.n_proof = sibling_count; + proof.proof = siblings; + uint8_t wire[MAX_PACKET_PAYLOAD]; + const uint16_t wire_len = encode_proof(wire, sizeof(wire), proof); + ASSERT_GT(wire_len, 0); + ASSERT_TRUE(client.on_message(wire, wire_len)); +} + TEST(OtaTransfer, ServerPacesOneKilobyteBlockAndProactiveProofWithBackpressure) { MotaManifest manifest; ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); @@ -2177,6 +2493,59 @@ TEST(OtaTransfer, ServerPacesOneKilobyteBlockAndProactiveProofWithBackpressure) EXPECT_EQ(server.pendingServeJobs(), 0u); } +TEST(OtaTransfer, LiteralLegacyFullMaskServesTwoKilobyteBlockInThirteenFragments) { + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA_2K, SIM_MOTA_2K_LEN, manifest)); + ASSERT_EQ(manifest.block_size(), 2048u); + + OtaManager server; + CapturedMessages sent; + server.begin(0, capture_send, &sent); + ASSERT_TRUE(server.serve(SIM_MOTA_2K, SIM_MOTA_2K_LEN)); + + ReqMsg request{}; + memcpy(request.manifest_id, manifest.merkle_root, 4); + request.block_idx = 0; + request.want_mask = 0xFFFFu; // deployed legacy "all fragments" + EXPECT_FALSE(ota_req_is_v2(request.want_mask)); + uint8_t wire[MAX_PACKET_PAYLOAD]; + const uint16_t request_len = encode_req(wire, sizeof(wire), request); + ASSERT_GT(request_len, 0); + ASSERT_TRUE(server.on_message(wire, request_len)); + + const uint32_t fragment_count = + (manifest.block_size() + OTA_FRAG_DATA - 1) / OTA_FRAG_DATA; + ASSERT_EQ(fragment_count, 13u); + for (uint32_t fragment = 0; fragment < fragment_count; fragment++) { + server.serviceEgress(); + ASSERT_EQ(sent.items.size(), fragment + 1); + DataMsg data{}; + ASSERT_TRUE(decode_data(sent.items.back().data(), + (uint16_t)sent.items.back().size(), data)); + EXPECT_EQ(data.block_idx, 0u); + EXPECT_EQ(data.frag_off, fragment * OTA_FRAG_DATA); + const uint16_t expected_len = (uint16_t)std::min( + OTA_FRAG_DATA, manifest.block_size() - fragment * OTA_FRAG_DATA); + EXPECT_EQ(data.data_len, expected_len); + EXPECT_EQ(0, memcmp(data.data, + manifest.payload + fragment * OTA_FRAG_DATA, + expected_len)); + } + EXPECT_EQ(sent.items.size(), 13u); // no v2 12-fragment interpretation + EXPECT_EQ(server.pendingServeJobs(), 1u); + + server.serviceEgress(); // establish proof turnaround deadline + ASSERT_EQ(sent.items.size(), 13u); + server.set_clock(OTA_MANIFEST_EGRESS_MIN_GAP_MS); + server.serviceEgress(); + ASSERT_EQ(sent.items.size(), 14u); + ProofMsg proof{}; + ASSERT_TRUE(decode_proof(sent.items.back().data(), + (uint16_t)sent.items.back().size(), proof)); + EXPECT_EQ(proof.block_idx, 0u); + EXPECT_EQ(server.pendingServeJobs(), 0u); +} + TEST(OtaTransfer, NegotiatedV2Carries171RawBytesAndRepeatsRepresentationId) { MotaManifest manifest; ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); @@ -2498,6 +2867,55 @@ TEST(OtaTransfer, NewClientRequestsV2ThenRetriesLegacyWhenNoV2DataArrives) { EXPECT_EQ(fallback.want_mask, 0x007Fu); } +TEST(OtaTransfer, TwoKilobyteClientAcceptsCanonicalLegacyFallbackThroughBitTwelve) { + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA_2K, SIM_MOTA_2K_LEN, manifest)); + ASSERT_EQ(manifest.block_size(), 2048u); + ASSERT_EQ(manifest.block_count, 2u); + + OtaManager client; + OtaStoreRam<4096> store; + CapturedMessages sent; + client.begin(SIM_TARGET_ID, capture_send, &sent); + client.set_fetch_store(&store); + client.set_clock(100); + ASSERT_EQ(client.pull(manifest.merkle_root, manifest.target_id), OtaManager::PULL_STARTED); + sent.items.clear(); + deliver_manifest_fragment(client, manifest.merkle_root, 0, + manifest.manifest_start, OTA_MF_FRAG); + deliver_manifest_fragment(client, manifest.merkle_root, 1, + manifest.manifest_start + OTA_MF_FRAG, + (uint16_t)(MOTA_MFL - OTA_MF_FRAG)); + + ASSERT_EQ(sent.items.size(), 1u); + ReqMsg extended{}; + ASSERT_TRUE(decode_req(sent.items[0].data(), + (uint16_t)sent.items[0].size(), extended)); + EXPECT_TRUE(ota_req_is_v2(extended.want_mask)); + EXPECT_TRUE(ota_req_v2_extended_length(extended.want_mask)); + EXPECT_EQ(ota_req_v2_fragments(extended.want_mask), 0x0FFFu); + + // The first canonical legacy slice selects the fallback geometry. Completing the 2 KiB block then + // necessarily receives fragment bit 12 (offset 1920) before its proof can authenticate and commit it. + ASSERT_EQ((manifest.block_size() + OTA_FRAG_DATA - 1) / OTA_FRAG_DATA, 13u); + deliver_verified_block(client, manifest, 0); + ASSERT_EQ(client.blocksHave(), 1u); + + ASSERT_EQ(sent.items.size(), 2u); + ReqMsg next{}; + ASSERT_TRUE(decode_req(sent.items.back().data(), + (uint16_t)sent.items.back().size(), next)); + EXPECT_EQ(next.block_idx, 1u); + EXPECT_FALSE(ota_req_is_v2(next.want_mask)); + EXPECT_EQ(next.want_mask, 0x0003u); // short tail remains legacy after downgrade + + deliver_verified_block(client, manifest, 1); + EXPECT_EQ(client.fetchState(), OtaManager::COMPLETE); + EXPECT_EQ(client.blocksHave(), client.blocksTotal()); + ASSERT_EQ(store.staged_size(), SIM_MOTA_2K_LEN); + EXPECT_EQ(0, memcmp(store.data(), SIM_MOTA_2K, SIM_MOTA_2K_LEN)); +} + TEST(OtaTransfer, MalformedLegacyDataCannotForceV2SessionDowngrade) { MotaManifest manifest; ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); @@ -2597,6 +3015,78 @@ TEST(OtaTransfer, PartialV2SeederCanDisappearAndLegacySeederTakesOver) { EXPECT_EQ(client.blocksHave(), 1u); } +TEST(OtaTransfer, FailedTwoKilobyteInflateRetriesRawWithoutLeavingExtendedV2) { + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA_2K, SIM_MOTA_2K_LEN, manifest)); + ASSERT_EQ(manifest.block_size(), 2048u); + ASSERT_EQ(manifest.block_count, 2u); + + OtaManager client; + OtaStoreRam<4096> store; + CapturedMessages sent; + RejectWireDecode decoder{}; + client.begin(SIM_TARGET_ID, capture_send, &sent); + client.set_transport_deflate_decoder(reject_wire_decode, &decoder); + client.set_fetch_store(&store); + client.set_clock(100); + ASSERT_EQ(client.pull(manifest.merkle_root, manifest.target_id), OtaManager::PULL_STARTED); + sent.items.clear(); + deliver_manifest_fragment(client, manifest.merkle_root, 0, + manifest.manifest_start, OTA_MF_FRAG); + deliver_manifest_fragment(client, manifest.merkle_root, 1, + manifest.manifest_start + OTA_MF_FRAG, + (uint16_t)(MOTA_MFL - OTA_MF_FRAG)); + ASSERT_EQ(sent.items.size(), 1u); + ReqMsg first{}; + ASSERT_TRUE(decode_req(sent.items[0].data(), + (uint16_t)sent.items[0].size(), first)); + EXPECT_TRUE(ota_req_is_v2(first.want_mask)); + EXPECT_TRUE(ota_req_v2_extended_length(first.want_mask)); + EXPECT_NE(first.want_mask & OTA_REQ_V2_ALLOW_DEFLATE, 0u); + + sent.items.clear(); + deliver_v2_representation( + client, manifest, 0, + SIM_MOTA_2K_DEFLATED + SIM_MOTA_2K_DEFLATED_OFFSETS[0], + SIM_MOTA_2K_DEFLATED_LENGTHS[0]); + EXPECT_EQ(decoder.calls, 1u); + EXPECT_EQ(client.blocksHave(), 0u); + EXPECT_TRUE(sent.items.empty()); // retry is paced by the normal deadline + + const uint32_t timeout = client.fetchRetryTimeoutMs(); + client.set_clock(100 + timeout); + client.loop(); + ASSERT_EQ(sent.items.size(), 1u); + ReqMsg retry{}; + ASSERT_TRUE(decode_req(sent.items[0].data(), + (uint16_t)sent.items[0].size(), retry)); + EXPECT_EQ(retry.block_idx, 0u); + EXPECT_TRUE(ota_req_is_v2(retry.want_mask)); + EXPECT_TRUE(ota_req_v2_extended_length(retry.want_mask)); + EXPECT_EQ(retry.want_mask & OTA_REQ_V2_ALLOW_DEFLATE, 0u); + EXPECT_EQ(ota_req_v2_fragments(retry.want_mask), 0x0FFFu); + + sent.items.clear(); + deliver_verified_v2_raw_block(client, manifest, 0); + ASSERT_EQ(client.blocksHave(), 1u); + EXPECT_EQ(decoder.calls, 1u); // raw v2 bypasses the failed decoder + ASSERT_EQ(sent.items.size(), 1u); + ReqMsg next{}; + ASSERT_TRUE(decode_req(sent.items[0].data(), + (uint16_t)sent.items[0].size(), next)); + EXPECT_EQ(next.block_idx, 1u); + EXPECT_TRUE(ota_req_is_v2(next.want_mask)); + EXPECT_TRUE(ota_req_v2_extended_length(next.want_mask)); + EXPECT_EQ(next.want_mask & OTA_REQ_V2_ALLOW_DEFLATE, 0u); + + deliver_verified_v2_raw_block(client, manifest, 1); + EXPECT_EQ(client.fetchState(), OtaManager::COMPLETE); + EXPECT_EQ(client.blocksHave(), client.blocksTotal()); + EXPECT_EQ(decoder.calls, 1u); + ASSERT_EQ(store.staged_size(), SIM_MOTA_2K_LEN); + EXPECT_EQ(0, memcmp(store.data(), SIM_MOTA_2K, SIM_MOTA_2K_LEN)); +} + TEST(OtaTransfer, ConfirmedV2SeederCanDisappearBetweenBlocksAndLegacySeederTakesOver) { MotaManifest manifest; ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); @@ -2723,6 +3213,59 @@ TEST(OtaTransfer, V2NeverMixesDifferentCompressedRepresentationsAcrossSeeders) { EXPECT_EQ(0, memcmp(store.data() + payload_offset, manifest.payload, manifest.block_size())); } +TEST(OtaTransfer, ExtendedCompressedRepresentationReassemblesThroughFragmentEight) { + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA_2K, SIM_MOTA_2K_LEN, manifest)); + ASSERT_EQ(manifest.block_size(), 2048u); + + std::vector representation(1500); + for (uint32_t i = 0; i < representation.size(); i++) { + representation[i] = (uint8_t)(i * 29u + 7u); + } + ASSERT_GT(representation.size(), 1024u); + ASSERT_LT(representation.size(), manifest.block_size()); + ASSERT_EQ((representation.size() + OTA_FRAG_DATA_V2 - 1) / OTA_FRAG_DATA_V2, 9u); + + SyntheticRepresentationDecode decoder{}; + decoder.representation = representation.data(); + decoder.representation_len = (uint16_t)representation.size(); + decoder.decoded = manifest.payload; + decoder.decoded_len = (uint16_t)manifest.block_size(); + + OtaManager client; + OtaStoreRam<4096> store; + CapturedMessages sent; + client.begin(SIM_TARGET_ID, capture_send, &sent); + client.set_transport_deflate_decoder(synthetic_representation_decode, &decoder); + client.set_fetch_store(&store); + ASSERT_EQ(client.pull(manifest.merkle_root, manifest.target_id), OtaManager::PULL_STARTED); + sent.items.clear(); + deliver_manifest_fragment(client, manifest.merkle_root, 0, + manifest.manifest_start, OTA_MF_FRAG); + deliver_manifest_fragment(client, manifest.merkle_root, 1, + manifest.manifest_start + OTA_MF_FRAG, + (uint16_t)(MOTA_MFL - OTA_MF_FRAG)); + ASSERT_EQ(sent.items.size(), 1u); + ReqMsg request{}; + ASSERT_TRUE(decode_req(sent.items[0].data(), + (uint16_t)sent.items[0].size(), request)); + EXPECT_TRUE(ota_req_v2_extended_length(request.want_mask)); + EXPECT_NE(request.want_mask & OTA_REQ_V2_ALLOW_DEFLATE, 0u); + + sent.items.clear(); + deliver_v2_representation(client, manifest, 0, + representation.data(), (uint16_t)representation.size()); + EXPECT_EQ(decoder.calls, 1u); // fragment 8 completed reassembly + decode + EXPECT_EQ(client.blocksHave(), 0u); // proof still gates the decoded bytes + EXPECT_TRUE(sent.items.empty()); + + deliver_block_proof(client, manifest, 0); + EXPECT_EQ(client.blocksHave(), 1u); + const uint32_t payload_offset = (uint32_t)(manifest.payload - SIM_MOTA_2K); + EXPECT_EQ(0, memcmp(store.data() + payload_offset, + manifest.payload, manifest.block_size())); +} + TEST(OtaTransfer, NewClientFallsBackWhenLegacySourceServesOnlyFirstWindowRow) { MotaManifest manifest; ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); @@ -3244,7 +3787,7 @@ TEST(OtaCatalog, RetainsProtocolMaximumRows) { TEST(OtaCatalog, RejectsAdvertisedSourceWithOversizedBlocks) { OtaManager server; server.begin(0, nullptr, nullptr); - SyntheticCatalogSource source(1, 11); // 2048-byte blocks exceed OTA_MAX_BLOCK + SyntheticCatalogSource source(1, 12); // 4096-byte blocks exceed OTA_MAX_BLOCK ASSERT_TRUE(server.add_source(&source)); EXPECT_EQ(server.servedCount(), 0); } diff --git a/tools/ble_mota/ble_mota_seeder.py b/tools/ble_mota/ble_mota_seeder.py index 33e5eb7d..f2a82541 100755 --- a/tools/ble_mota/ble_mota_seeder.py +++ b/tools/ble_mota/ble_mota_seeder.py @@ -57,6 +57,7 @@ STATUS_OK = 0 STATUS_ERR = 1 MOTA_READ_MAX = 192 MOTA_DEFLATE_CHUNK_MAX = 190 +MOTA_DEFLATE_BLOCK_MAX = 2048 MOTA_DESC_WIRE = 38 MOTA_SOURCE_CAP_DEFLATE_BLOCK = 0x01 MOTA_HEADER_LEN = 8 @@ -88,14 +89,15 @@ class MotaFile: return None raw_offset = block_index * block_size raw_length = min(block_size, payload_size - raw_offset) - if raw_length <= 0 or raw_length > 1024 or self.path.stat().st_size != self.size: + if (raw_length <= 0 or raw_length > MOTA_DEFLATE_BLOCK_MAX + or self.path.stat().st_size != self.size): return None with self.path.open("rb") as stream: stream.seek(payload_offset + raw_offset) raw = stream.read(raw_length) if len(raw) != raw_length: return None - compressor = zlib.compressobj(level=9, method=zlib.DEFLATED, wbits=-10) + compressor = zlib.compressobj(level=9, method=zlib.DEFLATED, wbits=-11) encoded = compressor.compress(raw) + compressor.flush() return encoded if 0 < len(encoded) < len(raw) else None diff --git a/tools/ble_mota/test_ble_mota_seeder.py b/tools/ble_mota/test_ble_mota_seeder.py index 86b86136..f1487134 100644 --- a/tools/ble_mota/test_ble_mota_seeder.py +++ b/tools/ble_mota/test_ble_mota_seeder.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import hashlib import struct import tempfile import unittest @@ -24,16 +25,21 @@ class TransportDeflateTests(unittest.TestCase): def setUp(self): self.directory = tempfile.TemporaryDirectory() self.path = Path(self.directory.name) / "test.mota" - # This distribution makes zlib level 9 choose a dynamic-Huffman block, proving the - # host is not silently constrained to the bootloader's separate fixed-only profile. - self.raw = bytes(i % 31 if i % 7 else 0 for i in range(1024)) + # The repeated 512-byte prefix is 1536 bytes behind its second copy. A 2 KiB + # application window can encode that match; the legacy 1 KiB window cannot. + noise = b"".join( + hashlib.sha256(index.to_bytes(4, "little")).digest() + for index in range(48) + ) + self.raw = noise[:512] + noise[512:1536] + noise[:512] + self.assertEqual(len(self.raw), seeder.MOTA_DEFLATE_BLOCK_MAX) payload_offset = 256 self.path.write_bytes(bytes(payload_offset) + self.raw) descriptor = bytearray(seeder.MOTA_DESC_WIRE) struct.pack_into("> 1) & 0x03, 2) # dynamic Huffman (BTYPE=2) - self.assertEqual(zlib.decompress(bytes(encoded), wbits=-10), self.raw) + self.assertEqual(zlib.decompress(bytes(encoded), wbits=-11), self.raw) + + compressor_1k = zlib.compressobj( + level=9, method=zlib.DEFLATED, wbits=-10 + ) + encoded_1k = compressor_1k.compress(self.raw) + compressor_1k.flush() + self.assertLess(len(encoded) + 400, len(encoded_1k)) def test_rejects_bad_block_range_and_oversized_chunk(self): self.assertEqual(self.exchange(1, 0, 0)[0], seeder.STATUS_ERR) diff --git a/tools/lora_ota/build_rak3401_compact_bundle.py b/tools/lora_ota/build_rak3401_compact_bundle.py index e49a8ff1..017f933d 100755 --- a/tools/lora_ota/build_rak3401_compact_bundle.py +++ b/tools/lora_ota/build_rak3401_compact_bundle.py @@ -43,7 +43,10 @@ STAGE_CEILING = 0xD4000 FIXED_WORKSPACE = 0x98000 FLASH_PAGE = 4096 SEGMENT_SIZE = 4096 -BLOCK_SIZE = 1024 +LEGACY_BLOCK_SIZE = 1024 +DEFLATE_BLOCK_SIZE = 2048 +# Schema-1 and the pinned bootstrap always use the historical geometry. +BLOCK_SIZE = LEGACY_BLOCK_SIZE UF2_BLOCK_SIZE = 512 UF2_DATA_OFFSET = 32 UF2_DATA_CAPACITY = 476 @@ -54,19 +57,32 @@ UF2_FLAG_NOT_MAIN_FLASH = 0x00000001 UF2_FLAG_FAMILY_ID = 0x00002000 UF2_NRF52840_FAMILY_ID = 0xADA52840 REQUIRED_SIMULATORS = { - "preview5": { - "sha256": "c6b3c6bc0b284b0b2c7a9eef1b3ea7afeb05ce4241d81309d4ab6381a575b897", - "tag": "0.9.2-OTAFIX2.4.1-preview.5", - "commit": "f8d5649ccf8b4a482c43cfdfeb61b29ab816ca48", - }, - "current": { - "sha256": "bf57fc4b62f11d6d9ee1d8b7a863d22d3e8048ad750f6bee5ceffd7b2d4d83a7", - "commit": "ffb1580d8fa59d333e1e944a198ec9b231be5a40", + "otafix2.4": { + "sha256": "9dacf24b1023fe2f4c620419417649ccc9538dd4a611b47bc19b7fedb97ceba6", + "tag": "0.9.2-OTAFIX2.4", + "commit": "d73de8372e89b8ef352747c8bc7a1aaeab80fbfe", }, } MOTATOOL_VERSION = "0.1.0" -MOTATOOL_COMMIT = "b0bfa1c16fa758f622cacaea6e427756921c4271" +MOTATOOL_COMMIT = "abdabec012cb53883a01da53b3e7b604ee0fd070" +TRANSPORT_ROUTE_ENCODER_SHA256 = ( + "53e3ca7e82f2f95b62c65d0858a83f5c54bd238a955b3a01fbe41c08bb4962e8" +) DETOOLS_VERSION = "0.53.0" +CONTAINER_ROUTE_OBJECTIVE = "minimum packages, then minimum total container bytes" +TRANSPORT_ROUTE_OBJECTIVE = ( + "minimum packages, then minimum ideal linear-path serialized MeshCore bytes" +) +TRANSPORT_STEP_FIELDS = ( + "profile", "block_size", "payload_bytes", "wire_bytes", "deflate_bytes", + "deflate_blocks", "data_packets", "request_packets", + "manifest_packets", "request_pipeline", "proof_request_packets", + "proof_packets", "packets", "manifest_bytes", "data_bytes", + "block_request_bytes", "proof_request_bytes", "proof_response_bytes", + "origin_mesh_bytes", "linear_path_bytes", +) +LEGACY_RELEASE_TRANSPORT_BYTES = 1_563_957 +LEGACY_RELEASE_TRANSPORT_PACKETS = 13_490 PHYSICAL_VALIDATION_KIND = "rak3401-mota-exact-chain" MAX_PHYSICAL_VALIDATION_BYTES = 256 * 1024 PHYSICAL_VALIDATION_FIELDS = { @@ -141,9 +157,11 @@ def parse_simulators(values: list[str]) -> list[tuple[str, Path]]: if len(actual) != len(simulators): raise CompactBuildError("apply simulator labels must be unique") if actual != expected: + required = " and ".join( + f"{label}={digest}" for label, digest in expected.items() + ) raise CompactBuildError( - "legacy RAK3401 builds require exactly preview5=" - f"{expected['preview5']} and current={expected['current']} apply simulators" + f"legacy RAK3401 builds require exactly {required} apply simulators" ) return simulators @@ -285,6 +303,76 @@ def read_route(path: Path) -> tuple[list[dict[str, object]], dict[str, object]]: else: node_count = ENDPOINT_NODE + 1 endpoint_node = ENDPOINT_NODE + search = document.get("search") if schema == 1 else document + if not isinstance(search, dict): + raise CompactBuildError("route plan is missing its exhaustive-search summary") + objective = search.get("objective") + allowed_objectives = ( + (CONTAINER_ROUTE_OBJECTIVE,) + if schema == 1 + else (CONTAINER_ROUTE_OBJECTIVE, TRANSPORT_ROUTE_OBJECTIVE) + ) + if objective not in allowed_objectives: + raise CompactBuildError("route plan has the wrong search objective") + transport_accounting: dict[str, object] | None = None + transport_encoder_sha: str | None = None + relay_hops = 0 + if objective == TRANSPORT_ROUTE_OBJECTIVE: + raw_accounting = search.get("transport_accounting") + if not isinstance(raw_accounting, dict): + raise CompactBuildError("route plan transport accounting is missing") + transport_accounting = dict(raw_accounting) + relay_hops = parse_int( + raw_accounting.get("relay_hops"), "route transport relay_hops" + ) + if relay_hops > route_search.OTA_MAX_HOPS: + raise CompactBuildError("route transport relay_hops is out of range") + if ( + raw_accounting.get("source_capability_field") + != route_search.TRANSPORT_CAPABILITY + or raw_accounting.get("source_pipeline_field") + != route_search.TRANSPORT_PIPELINE_FIELD + or raw_accounting.get("source_max_block_field") + != route_search.TRANSPORT_MAX_BLOCK_FIELD + or raw_accounting.get("source_profile_matrix") + != route_search.TRANSPORT_PROFILE_MATRIX + or raw_accounting.get("first_bootstrap_profile") != "legacy-160-raw" + or raw_accounting.get("legacy_block_size") != LEGACY_BLOCK_SIZE + or raw_accounting.get("supported_block_sizes") + != [LEGACY_BLOCK_SIZE, DEFLATE_BLOCK_SIZE] + or raw_accounting.get("comparison_baseline_block_size") + != LEGACY_BLOCK_SIZE + or raw_accounting.get("included") + != ( + "OTA messages, repeated 4-byte v2 stream IDs, 171-byte DATA " + "slicing, adaptive requests, manifest, exact per-block proofs, " + "and MeshCore framing" + ) + or raw_accounting.get("excluded") + != ( + "discovery, retries, flood fan-out, and radio-dependent LoRa " + "PHY coding/preamble" + ) + ): + raise CompactBuildError("route transport capability contract is invalid") + encoder_sha = raw_accounting.get("encoder_sha256") + if encoder_sha is not None and encoder_sha != TRANSPORT_ROUTE_ENCODER_SHA256: + raise CompactBuildError( + "route transport encoder does not match the audited release binary" + ) + transport_encoder_sha = encoder_sha + transport_accounting = { + key: raw_accounting[key] + for key in ( + "relay_hops", "source_capability_field", "source_pipeline_field", + "source_max_block_field", "source_profile_matrix", + "first_bootstrap_profile", "legacy_block_size", + "supported_block_sizes", "comparison_baseline_block_size", + "included", "excluded", + ) + } + if transport_encoder_sha is not None: + transport_accounting["encoder_sha256"] = transport_encoder_sha if parse_int(document.get("app_base"), "route app_base") != APP_BASE: raise CompactBuildError("route plan has the wrong application base") if parse_int(document.get("stage_ceiling"), "route stage_ceiling") != STAGE_CEILING: @@ -329,6 +417,118 @@ def read_route(path: Path) -> tuple[list[dict[str, object]], dict[str, object]]: reuse = raw.get("reuse_baseline_package", False) if not isinstance(reuse, bool): raise CompactBuildError(f"route step {number} reuse flag must be boolean") + # Older all-legacy schema-2 plans predate the explicit field. Treat an + # omission as their historical 1 KiB geometry; new route output always + # writes the field and transport routes also pin it in accounting. + raw_block_size = raw.get("block_size", LEGACY_BLOCK_SIZE) + block_size = parse_int(raw_block_size, f"route step {number} block_size") + if block_size not in (LEGACY_BLOCK_SIZE, DEFLATE_BLOCK_SIZE): + raise CompactBuildError(f"route step {number} block_size is unsupported") + raw_transport = raw.get("transport") + transport = None + if objective == TRANSPORT_ROUTE_OBJECTIVE: + if not isinstance(raw_transport, dict): + raise CompactBuildError( + f"route step {number} is missing transport accounting" + ) + if raw_transport.get("profile") not in ( + "legacy-160-raw", "v2-171-raw", "v2-171-deflate" + ): + raise CompactBuildError( + f"route step {number} has an invalid transport profile" + ) + transport = dict(raw_transport) + for field in TRANSPORT_STEP_FIELDS[1:]: + transport[field] = parse_int( + raw_transport.get(field), + f"route step {number} transport {field}", + ) + payload_sha = raw_transport.get("payload_sha256") + encoder_sha = raw_transport.get("encoder_sha256") + profile = str(transport["profile"]) + compressed_profile = profile == "v2-171-deflate" + if ( + profile == "legacy-160-raw" + and block_size != LEGACY_BLOCK_SIZE + ) or ( + profile == "v2-171-raw" + and block_size != DEFLATE_BLOCK_SIZE + ): + raise CompactBuildError( + f"route step {number} block_size disagrees with its " + "transport profile" + ) + if ( + compressed_profile + and (payload_sha is None or encoder_sha is None) + ) or ( + not compressed_profile + and (payload_sha is not None or encoder_sha is not None) + ): + raise CompactBuildError( + f"route step {number} transport hashes do not match its profile" + ) + for field, value in ( + ("payload_sha256", payload_sha), + ("encoder_sha256", encoder_sha), + ): + if value is not None: + if not isinstance(value, str) or not re.fullmatch( + r"[0-9a-f]{64}", value + ): + raise CompactBuildError( + f"route step {number} transport {field} is invalid" + ) + transport[field] = value + if encoder_sha is not None and encoder_sha != TRANSPORT_ROUTE_ENCODER_SHA256: + raise CompactBuildError( + f"route step {number} transport encoder is not the audited binary" + ) + try: + if compressed_profile: + calculated = route_search.v2_transport_cost( + int(transport["payload_bytes"]), + int(transport["wire_bytes"]), + int(transport["deflate_bytes"]), + int(transport["deflate_blocks"]), + int(transport["data_packets"]), + int(transport["request_pipeline"]), + block_size, + ) + elif profile == "v2-171-raw": + calculated = route_search.v2_raw_transport_cost( + int(transport["payload_bytes"]), + int(transport["request_pipeline"]), + block_size, + ) + else: + calculated = route_search.legacy_transport_cost( + int(transport["payload_bytes"]) + ) + calculated["linear_path_bytes"] = route_search.linear_path_bytes( + int(calculated["origin_mesh_bytes"]), + int(calculated["packets"]), + relay_hops, + ) + except route_search.RouteSearchError as exc: + raise CompactBuildError( + f"route step {number} transport accounting is invalid: {exc}" + ) from exc + for field in TRANSPORT_STEP_FIELDS: + if transport[field] != calculated[field]: + raise CompactBuildError( + f"route step {number} transport {field} is inconsistent" + ) + if route_search.container_size( + int(transport["payload_bytes"]), block_size + ) != expected_size: + raise CompactBuildError( + f"route step {number} transport payload does not match its container" + ) + elif raw_transport is not None: + raise CompactBuildError( + f"route step {number} unexpectedly contains transport accounting" + ) if source >= target or target >= node_count or memory % FLASH_PAGE: raise CompactBuildError(f"route step {number} has invalid nodes or workspace alignment") if memory < FLASH_PAGE or APP_BASE + memory >= STAGE_CEILING: @@ -338,16 +538,20 @@ def read_route(path: Path) -> tuple[list[dict[str, object]], dict[str, object]]: elif source != steps[-1]["target_node"]: raise CompactBuildError(f"route step {number} is discontinuous") nodes.append(target) - steps.append({ + step = { "source_node": source, "target_node": target, "inplace_memory": memory, + "block_size": block_size, "reuse_baseline_package": reuse, "expected_container_size": expected_size, "expected_staging_margin": expected_margin, "expected_target_sha256": expected_sha, "expected_target_version": expected_version, - }) + } + if transport is not None: + step["transport"] = transport + steps.append(step) if not nodes or nodes[0] != 0 or nodes[-1] != endpoint_node: raise CompactBuildError( f"route nodes must run from 0 through {endpoint_node}: {tuple(nodes)}" @@ -362,11 +566,6 @@ def read_route(path: Path) -> tuple[list[dict[str, object]], dict[str, object]]: raise CompactBuildError("the pinned first bridge requires a 0x98000 workspace") if steps[1]["inplace_memory"] != FIXED_WORKSPACE: raise CompactBuildError("the fixed-receiver second step requires a 0x98000 workspace") - search = document.get("search") if schema == 1 else document - if not isinstance(search, dict): - raise CompactBuildError("route plan is missing its exhaustive-search summary") - if search.get("objective") != "minimum packages, then minimum total container bytes": - raise CompactBuildError("route plan has the wrong search objective") if search.get("shortest_package_count") != expected_steps: raise CompactBuildError("route plan search package count is inconsistent") if schema == 1: @@ -381,6 +580,29 @@ def read_route(path: Path) -> tuple[list[dict[str, object]], dict[str, object]]: != selected_bytes ): raise CompactBuildError("route plan search byte total is inconsistent") + if objective == TRANSPORT_ROUTE_OBJECTIVE: + has_compressed_step = any( + step["transport"]["profile"] == "v2-171-deflate" + for step in steps + ) + if has_compressed_step != (transport_encoder_sha is not None): + raise CompactBuildError( + "route transport encoder presence does not match its profiles" + ) + selected_transport_bytes = sum( + int(step["transport"]["linear_path_bytes"]) for step in steps + ) + if ( + parse_int( + search.get("selected_total_transport_bytes"), + "route search selected_total_transport_bytes", + ) + != selected_transport_bytes + ): + raise CompactBuildError( + "route plan search transport-byte total is inconsistent" + ) + assert transport_accounting is not None search_aliases = { "candidate_geometries": ( "candidate_geometries", "shorter_path_workspace_candidates" @@ -403,9 +625,13 @@ def read_route(path: Path) -> tuple[list[dict[str, object]], dict[str, object]]: "candidate_geometries", "feasible_edges", "objective", "selected_total_bytes", "shortest_package_count", "shortest_route_count", "search_complete", "node_count", + "selected_total_transport_bytes", "transport_accounting", ) if key in search } + if transport_accounting is not None: + document["transport_accounting"] = transport_accounting + document["search"]["transport_accounting"] = transport_accounting return steps, document @@ -511,10 +737,49 @@ def write_inventory_provenance(source: Path, destination: Path) -> None: raise CompactBuildError("cannot normalize invalid image inventory") allowed = ( "node", "size", "sha256", "body_hash", "version", - "source_commit", "baseline_container_size", + "source_commit", "baseline_container_size", "version_rank", + "source_kind", "capability_class", "ota_send_abi", + "ota_transport_deflate", "ota_fetch_pipeline", + "transport_max_block_bytes", + "transport_impl", "transport_fragment_bytes", + "accepted_transport_codecs", "prohibit_same_version_hop", + "variant_id", "variant_rank", "variant_count", ) + normalized_top = { + key: document[key] + for key in ( + "kind", "source_inventory_sha256", "image_count", "start_node", + "bootstrap_node", "endpoint_node", "app_base", "stage_ceiling", + ) + if key in document + } + capability_contract = document.get("capability_contract") + if isinstance(capability_contract, dict): + normalized_top["capability_contract"] = { + key: capability_contract[key] + for key in ( + "compatibility_alias", "cost_capability_owner", + "ota_fetch_pipeline", "ota_transport_deflate", + "transport_max_block_bytes", + "request_cost_rule", + ) + if key in capability_contract + } + edge_policy = document.get("edge_policy") + if isinstance(edge_policy, dict): + normalized_top["edge_policy"] = { + key: edge_policy[key] + for key in ( + "bootstrap_outbound_workspace", "bootstrap_outbound_workspace_hex", + "capability_owner", "eligibility_expression", "endpoint_must_be_last", + "pinned_first_edge", "prohibit_same_version_hops", "reason", + "require_target_version_rank_gt_source_version_rank", + ) + if key in edge_policy + } document = { "schema": 2, + **normalized_top, "images": [ {key: record[key] for key in allowed if key in record} for record in raw_images @@ -562,6 +827,7 @@ def public_route_search( "candidate_geometries", "feasible_edges", "objective", "selected_total_bytes", "shortest_package_count", "shortest_route_count", "search_complete", "node_count", + "selected_total_transport_bytes", "transport_accounting", ) return {key: search[key] for key in allowed if key in search} if schema != 1 or not isinstance(raw_search, dict): @@ -596,10 +862,12 @@ def write_route_provenance( endpoint_version: str, ) -> None: """Emit a reusable route plan containing only validated public fields.""" - normalized_steps = [ - { + normalized_steps = [] + for step in steps: + normalized_step = { "source_node": int(step["source_node"]), "target_node": int(step["target_node"]), + "block_size": int(step["block_size"]), "inplace_memory": f"0x{int(step['inplace_memory']):X}", "reuse_baseline_package": bool(step["reuse_baseline_package"]), "expected_container_size": int(step["expected_container_size"]), @@ -607,15 +875,22 @@ def write_route_provenance( "expected_target_sha256": str(step["expected_target_sha256"]), "expected_target_version": str(step["expected_target_version"]), } - for step in steps - ] + raw_transport = step.get("transport") + if isinstance(raw_transport, dict): + normalized_step["transport"] = { + key: raw_transport[key] + for key in (*TRANSPORT_STEP_FIELDS, "payload_sha256", "encoder_sha256") + if key in raw_transport + } + normalized_steps.append(normalized_step) if schema == 2: allowed = ( "schema", "status", "search_complete", "app_base", "stage_ceiling", "node_count", "candidate_geometries", "feasible_edges", "objective", "nodes", "shortest_package_count", "shortest_route_count", - "selected_total_bytes", + "selected_total_bytes", "selected_total_transport_bytes", + "transport_accounting", ) normalized = {key: evidence[key] for key in allowed if key in evidence} else: @@ -651,7 +926,7 @@ def validate_geometry_results( raise CompactBuildError(f"invalid route-search inventory: {exc}") from exc jobs = route_search.all_jobs(inventory) required = { - route_search.cache_key(job[5], job[6], job[2]): (job[0], job[1]) + route_search.cache_key(job[5], job[6], job[2]): (job[0], job[1], job[7]) for job in jobs } rows: list[dict[str, object]] = [] @@ -694,12 +969,16 @@ def validate_geometry_results( and target_sha == str(inventory[target_node]["sha256"]) ) if ( - (key not in required or required[key] != (source_node, target_node)) + ( + key not in required + or required[key][:2] != (source_node, target_node) + ) and not legacy_pinned_source_measurement ): raise CompactBuildError(f"extra or misindexed geometry result at line {line}") try: payload = int(raw["payload"]) + block_size = int(raw["block_size"]) container = int(raw["container"]) stage_start = int(raw["stage_start"]) margin = int(raw["margin"]) @@ -724,7 +1003,18 @@ def validate_geometry_results( "an exhaustive route search" ) else: - expected_container = route_search.container_size(payload) + expected_block_size = ( + LEGACY_BLOCK_SIZE + if legacy_pinned_source_measurement + else required[key][2] + ) + if block_size != expected_block_size: + raise CompactBuildError( + f"geometry line {line} has the wrong block size" + ) + expected_container = route_search.container_size( + payload, block_size + ) expected_stage = route_search.align_down(STAGE_CEILING - expected_container) expected_margin = expected_stage - (APP_BASE + memory) if ( @@ -737,9 +1027,20 @@ def validate_geometry_results( raise CompactBuildError(f"geometry line {line} has inconsistent geometry") row = dict(raw) row.update(source=source_node, target=target_node, memory=memory, - payload=payload, container=container, stage_start=stage_start, + payload=payload, block_size=block_size, + container=container, stage_start=stage_start, margin=margin, feasible=feasible, error=error, source_sha256=source_sha, target_sha256=target_sha) + try: + row.update( + route_search.normalize_transport_stats( + row, payload, f"geometry line {line}", block_size + ) + ) + except route_search.RouteSearchError as exc: + raise CompactBuildError( + f"geometry line {line} has invalid transport statistics: {exc}" + ) from exc rows.append(row) missing = set(required) - seen if missing: @@ -748,24 +1049,40 @@ def validate_geometry_results( inventory[1].get("baseline_container_size"), "inventory node 1 baseline_container_size", ) - try: - with tempfile.TemporaryDirectory() as directory: - recomputed = route_search.select_route( - rows, inventory, baseline_size, Path(directory) / "route.json", - complete=True, - ) - except route_search.RouteSearchError as exc: - raise CompactBuildError(f"route recomputation failed: {exc}") from exc try: declared = json.loads(route_path.read_text(encoding="ascii")) except (OSError, UnicodeError, json.JSONDecodeError) as exc: raise CompactBuildError(f"cannot reread route plan: {exc}") from exc - compared = ( + declared_objective = declared.get("objective") if isinstance(declared, dict) else None + if declared_objective == TRANSPORT_ROUTE_OBJECTIVE: + objective = "transport" + raw_accounting = declared.get("transport_accounting") + if not isinstance(raw_accounting, dict): + raise CompactBuildError("route plan transport accounting is missing") + relay_hops = parse_int( + raw_accounting.get("relay_hops"), "route transport relay_hops" + ) + elif declared_objective == CONTAINER_ROUTE_OBJECTIVE: + objective = "container" + relay_hops = 0 + else: + raise CompactBuildError("route plan has the wrong search objective") + try: + with tempfile.TemporaryDirectory() as directory: + recomputed = route_search.select_route( + rows, inventory, baseline_size, Path(directory) / "route.json", + complete=True, objective=objective, relay_hops=relay_hops, + ) + except route_search.RouteSearchError as exc: + raise CompactBuildError(f"route recomputation failed: {exc}") from exc + compared = [ "schema", "status", "search_complete", "app_base", "stage_ceiling", "node_count", "candidate_geometries", "feasible_edges", "objective", "nodes", "shortest_package_count", "shortest_route_count", "selected_total_bytes", "steps", - ) + ] + if objective == "transport": + compared.extend(("selected_total_transport_bytes", "transport_accounting")) for field in compared: if declared.get(field) != recomputed.get(field): raise CompactBuildError(f"route plan does not match recomputed {field}") @@ -879,12 +1196,146 @@ def patch_geometry(detools: Path, payload_path: Path, label: str) -> tuple[int, return values["memory"], values["segment"], values["from_size"], values["to_size"] +def validate_selected_transport( + motatool: Path, + payload_path: Path, + plan_step: dict[str, object], + inventory: list[dict[str, object]], + relay_hops: int, + number: int, +) -> dict[str, object]: + """Re-measure a selected payload and match every declared radio-cost field.""" + declared = plan_step.get("transport") + if not isinstance(declared, dict): + raise CompactBuildError(f"step {number} has no transport accounting") + source_node = int(plan_step["source_node"]) + uses_v2 = route_search.source_uses_v2(inventory, source_node) + uses_deflate = route_search.source_uses_deflate(inventory, source_node) + expected_block_size = route_search.source_block_size(inventory, source_node) + if int(plan_step.get("block_size", 0)) != expected_block_size: + raise CompactBuildError( + f"step {number} logical block size disagrees with source capability" + ) + expected_profile = route_search.source_transport_profile( + inventory, source_node + ) + if declared.get("profile") != expected_profile: + raise CompactBuildError( + f"step {number} transport profile disagrees with source capability" + ) + payload_size = payload_path.stat().st_size + payload_sha = common.sha256_file(payload_path) + verification_encoder_sha: str | None = None + try: + if uses_deflate: + measured = route_search.measure_transport_size( + motatool, payload_path, expected_block_size + ) + if measured["payload_sha256"] != payload_sha: + raise CompactBuildError( + f"step {number} transport tool measured a different payload" + ) + verification_encoder_sha = str(measured["transport_encoder_sha256"]) + if verification_encoder_sha != declared.get("encoder_sha256"): + raise CompactBuildError( + f"step {number} transport measurement used a different " + "encoder binary" + ) + calculated = route_search.v2_transport_cost( + payload_size, + int(measured["v2_wire_bytes"]), + int(measured["v2_deflate_bytes"]), + int(measured["v2_deflate_blocks"]), + int(measured["v2_data_packets"]), + route_search.source_transport_pipeline(inventory, source_node), + expected_block_size, + ) + if declared.get("payload_sha256") != payload_sha: + raise CompactBuildError( + f"step {number} payload does not match transport route pin" + ) + if declared.get("encoder_sha256") != TRANSPORT_ROUTE_ENCODER_SHA256: + raise CompactBuildError( + f"step {number} route encoder is not the audited release binary" + ) + elif uses_v2: + calculated = route_search.v2_raw_transport_cost( + payload_size, + route_search.source_transport_pipeline(inventory, source_node), + expected_block_size, + ) + if "payload_sha256" in declared or "encoder_sha256" in declared: + raise CompactBuildError( + f"step {number} raw transport unexpectedly has encoder pins" + ) + else: + calculated = route_search.legacy_transport_cost(payload_size) + if "payload_sha256" in declared or "encoder_sha256" in declared: + raise CompactBuildError( + f"step {number} legacy transport unexpectedly has encoder pins" + ) + calculated["linear_path_bytes"] = route_search.linear_path_bytes( + int(calculated["origin_mesh_bytes"]), + int(calculated["packets"]), + relay_hops, + ) + if uses_v2: + one_kib_raw = route_search.v2_raw_transport_cost( + payload_size, + route_search.source_transport_pipeline(inventory, source_node), + LEGACY_BLOCK_SIZE, + ) + else: + one_kib_raw = route_search.legacy_transport_cost(payload_size) + one_kib_raw["linear_path_bytes"] = route_search.linear_path_bytes( + int(one_kib_raw["origin_mesh_bytes"]), + int(one_kib_raw["packets"]), + relay_hops, + ) + except route_search.RouteSearchError as exc: + raise CompactBuildError( + f"step {number} transport verification failed: {exc}" + ) from exc + for field in TRANSPORT_STEP_FIELDS: + if declared.get(field) != calculated[field]: + raise CompactBuildError( + f"step {number} regenerated transport {field} does not match route" + ) + return { + "status": "passed", + **{field: calculated[field] for field in TRANSPORT_STEP_FIELDS}, + "payload_sha256": payload_sha, + "route_encoder_sha256": ( + TRANSPORT_ROUTE_ENCODER_SHA256 if uses_deflate else None + ), + "verification_encoder_sha256": verification_encoder_sha, + "one_kib_no_compression": { + field: one_kib_raw[field] + for field in TRANSPORT_STEP_FIELDS + }, + } + + def write_chain(path: Path, rows: list[dict[str, object]]) -> None: fields = [ "step", "from_version", "to_version", "mota_file", "mota_size", + "mota_block_size", "inplace_memory", "stage_start", "workspace_end", "staging_margin", "target_image_size", "base_body_hash", "target_body_hash", - "target_sha256", "source_commit", + "target_sha256", "source_commit", "transport_profile", + "transport_relay_hops", "transport_request_pipeline", + "transport_payload_sha256", "transport_encoder_sha256", + "transport_payload_bytes", "transport_wire_bytes", + "transport_deflate_bytes", "transport_deflate_blocks", + "transport_data_packets", "transport_request_packets", + "transport_manifest_packets", "transport_proof_request_packets", + "transport_proof_packets", "transport_packets", + "transport_manifest_bytes", "transport_data_bytes", + "transport_block_request_bytes", "transport_proof_request_bytes", + "transport_proof_response_bytes", + "transport_origin_mesh_bytes", "transport_linear_path_bytes", + "baseline_1k_raw_packets", "baseline_1k_raw_origin_mesh_bytes", + "baseline_1k_raw_linear_path_bytes", ] with path.open("w", newline="", encoding="ascii") as output: writer = csv.DictWriter(output, fieldnames=fields, lineterminator="\n") @@ -1055,6 +1506,133 @@ def write_docs( largest = max(int(row["mota_size"]) for row in rows) minimum_margin = min(int(row["staging_margin"]) for row in rows) expected_steps = len(rows) + transport_rows = [row for row in rows if row.get("transport_profile")] + if transport_rows and len(transport_rows) != expected_steps: + raise CompactBuildError("generated chain has partial transport accounting") + if transport_rows: + relay_hop_values = { + int(row["transport_relay_hops"]) for row in transport_rows + } + if len(relay_hop_values) != 1: + raise CompactBuildError("generated chain mixes transport relay-hop models") + relay_hops = relay_hop_values.pop() + model_label = ( + "ideal direct-source" + if relay_hops == 0 + else f"ideal linear path with {relay_hops} relay hops" + ) + transport_bytes = sum( + int(row["transport_linear_path_bytes"]) for row in transport_rows + ) + transport_packets = sum( + int(row["transport_packets"]) for row in transport_rows + ) + payload_bytes = sum( + int(row["transport_payload_bytes"]) for row in transport_rows + ) + wire_bytes = sum( + int(row["transport_wire_bytes"]) for row in transport_rows + ) + deflate_bytes = sum( + int(row["transport_deflate_bytes"]) for row in transport_rows + ) + deflate_blocks = sum( + int(row["transport_deflate_blocks"]) for row in transport_rows + ) + data_packets = sum( + int(row["transport_data_packets"]) for row in transport_rows + ) + baseline_bytes = sum( + int(row["baseline_1k_raw_linear_path_bytes"]) + for row in transport_rows + ) + baseline_packets = sum( + int(row["baseline_1k_raw_packets"]) + for row in transport_rows + ) + baseline_byte_savings = baseline_bytes - transport_bytes + baseline_packet_savings = baseline_packets - transport_packets + baseline_byte_percent = 100.0 * baseline_byte_savings / baseline_bytes + baseline_packet_percent = 100.0 * baseline_packet_savings / baseline_packets + legacy_release_transport_bytes = route_search.linear_path_bytes( + LEGACY_RELEASE_TRANSPORT_BYTES, + LEGACY_RELEASE_TRANSPORT_PACKETS, + relay_hops, + ) + transport_savings = legacy_release_transport_bytes - transport_bytes + packet_savings = LEGACY_RELEASE_TRANSPORT_PACKETS - transport_packets + transport_percent = ( + 100.0 * transport_savings / legacy_release_transport_bytes + ) + packet_percent = 100.0 * packet_savings / LEGACY_RELEASE_TRANSPORT_PACKETS + deflate_steps = [ + str(row["step"]) + for row in transport_rows + if row["transport_profile"] == "v2-171-deflate" + ] + raw_v2_steps = [ + str(row["step"]) + for row in transport_rows + if row["transport_profile"] == "v2-171-raw" + ] + legacy_steps = [ + str(row["step"]) + for row in transport_rows + if row["transport_profile"] == "legacy-160-raw" + ] + transport_summary = f""" +- Staged container bytes: {total_bytes:,} + +| Same-route radio model | Logical block geometry | Origin packets | {model_label.capitalize()} serialized MeshCore bytes | +|---|---:|---:|---:| +| No DEFLATE baseline; packet profiles preserved | 1 KiB all steps | {baseline_packets:,} | {baseline_bytes:,} | +| Selected capability-aware route | Per-source 1/2 KiB signed geometry | {transport_packets:,} | {transport_bytes:,} | + +- Selected block streams: {payload_bytes:,} raw payload bytes become + {wire_bytes:,} radio bytes, including {deflate_bytes:,} DEFLATE bytes across + {deflate_blocks:,} compressed blocks and {data_packets:,} DATA packets +- Same-route savings: {baseline_packet_savings:,} packets + ({baseline_packet_percent:.2f}%) and {baseline_byte_savings:,} bytes + ({baseline_byte_percent:.2f}%) +- The baseline preserves each step's packet profile (171-byte negotiated v2 + or 160-byte legacy); it changes only logical blocks to 1 KiB and disables + DEFLATE. +- Serialized-byte reduction versus the published b40d2e6c chain: + {transport_savings:,} ({transport_percent:.2f}%) +- Origin packet reduction versus the published b40d2e6c chain: + {packet_savings:,} ({packet_percent:.2f}%) +""" + mode_descriptions = [] + if deflate_steps: + mode_descriptions.append( + f"Steps {', '.join(deflate_steps)} use negotiated 171-byte DATA " + "and independent per-block DEFLATE at each source's declared " + "signed-container block size." + ) + if raw_v2_steps: + mode_descriptions.append( + f"Steps {', '.join(raw_v2_steps)} use negotiated 171-byte raw " + "DATA with 2 KiB signed-container blocks; they require no " + "compression measurement or encoder hash." + ) + if legacy_steps: + mode_descriptions.append( + f"Steps {', '.join(legacy_steps)} use 1 KiB logical blocks and " + "the legacy 160-byte raw transport." + ) + transport_description = " ".join(mode_descriptions) + ( + " The protocol remains multi-hop; this report uses the " + f"{model_label} serialization model, excluding retries and flood " + "fan-out." + ) + package_description = ( + f"{expected_steps}; exhaustive route search proved that {expected_steps} " + "is the minimum for the declared firmware inventory" + ) + else: + transport_summary = f"\n- Staged container bytes: {total_bytes:,}\n" + transport_description = "This legacy route has no audited transport-byte accounting." + package_description = str(expected_steps) physical_text = ( "The publisher supplied an unsigned operator assertion that the exact " "package sequence passed physical/SWD validation; every hash in that " @@ -1080,8 +1658,8 @@ below its original `0x{STAGE_CEILING:X}` scan ceiling, and every encoded detools workspace ends at or below that package. No byte from `0x{STAGE_CEILING:X}` through `0xED000` is used by this chain. -- Packages: {expected_steps} (the replaced chain used 30) -- Total transferred bytes: {total_bytes:,} +- Packages: {package_description} +{transport_summary.rstrip()} - Largest package: {largest:,} bytes - Smallest workspace-to-stage margin: {minimum_margin:,} bytes - Endpoint image SHA-256: `{endpoint_sha}` @@ -1092,11 +1670,14 @@ The next package leaves the fixed-workspace receiver while still obeying its old `0x{FIXED_WORKSPACE:X}` limit. Every later package uses the page-aligned workspace selected in `ROUTE.json`. +{transport_description} When DEFLATE is selected, it exists only on the radio +transfer. The staged `.mota` container and bootloader input remain unchanged +and uncompressed, so the deployed bootloader does not need an inflater. + All {expected_steps} transitions passed container verification, zero-filled and -erased-workspace reconstruction, OTAFIX 2.4.1 Preview 5 simulator validation -(tag `0.9.2-OTAFIX2.4.1-preview.5`, commit -`f8d5649ccf8b4a482c43cfdfeb61b29ab816ca48`) and current simulator validation -(commit `ffb1580d8fa59d333e1e944a198ec9b231be5a40`). +erased-workspace reconstruction, and validation with the simulator built from +the exact deployed OTAFIX 2.4 source (tag `0.9.2-OTAFIX2.4`, commit +`d73de8372e89b8ef352747c8bc7a1aaeab80fbfe`). {physical_text} Use the pinned chain runner and do not install packages out of order. """, @@ -1112,7 +1693,9 @@ Endpoint profile name: `halo-keymind-cascade-dev` Legacy stage ceiling: `0x{STAGE_CEILING:X}` Application base: `0x{APP_BASE:X}` Segment size: `{SEGMENT_SIZE}` -Logical mOTA block size: `{BLOCK_SIZE}` +Logical mOTA block size: independently selected per running source from its +explicit `{route_search.TRANSPORT_MAX_BLOCK_FIELD}` capability (`{LEGACY_BLOCK_SIZE}` +or `{DEFLATE_BLOCK_SIZE}` bytes); DEFLATE permission is a separate capability Pinned 30-step reconstruction input SHA-256: `{previous_sha}` Dynamic bridge-image manifest SHA-256: `{bridge_sha}` @@ -1121,6 +1704,7 @@ Geometry-results SHA-256: `{geometry_sha or "not applicable (schema 1)"}` Bundled route-selection/verifier source SHA-256: `{route_search_sha or "not applicable (schema 1)"}` Executing bundle-builder source SHA-256: `{builder_source_sha}` +Audited route transport encoder SHA-256: `{TRANSPORT_ROUTE_ENCODER_SHA256}` Tool provenance: @@ -1132,19 +1716,23 @@ Tool provenance: Required bootloader simulators: -- OTAFIX 2.4.1 Preview 5 (`0.9.2-OTAFIX2.4.1-preview.5`, commit - `f8d5649ccf8b4a482c43cfdfeb61b29ab816ca48`): - `c6b3c6bc0b284b0b2c7a9eef1b3ea7afeb05ce4241d81309d4ab6381a575b897` -- Current OTAFIX (commit `ffb1580d8fa59d333e1e944a198ec9b231be5a40`): - `bf57fc4b62f11d6d9ee1d8b7a863d22d3e8048ad750f6bee5ceffd7b2d4d83a7` +- Exact deployed OTAFIX 2.4 (`0.9.2-OTAFIX2.4`, commit + `d73de8372e89b8ef352747c8bc7a1aaeab80fbfe`): + `9dacf24b1023fe2f4c620419417649ccc9538dd4a611b47bc19b7fedb97ceba6` -For schema 2, the route plan is independently selected again from every -successful row in the complete, content-pinned `GEOMETRY.csv` table. Failed -tool jobs are not admissible evidence. The minimum-package, then minimum-byte -claim applies only to that declared firmware inventory and geometry table; it -does not claim that no other conceivable bridge firmware could improve the -route. The bundle generator rebuilds every selected package and checks it -against the old bootloader ceiling before invoking either simulator. +For schema 2, the route plan is independently selected again from every row in +the complete, content-pinned `GEOMETRY.csv` table. Transport measurements are +required only for feasible capable-source edges on the shortest-route DAG. +Failed tool jobs and missing relevant transport rows are not admissible +evidence. The secondary objective is exact ideal linear-path serialized +MeshCore bytes, after minimizing package count. It applies only to the declared +firmware inventory and geometry table; it does not claim that no other +conceivable bridge firmware could improve the route. + +The bundle generator rebuilds every selected package, matches every v2 payload +SHA-256 and transport counter against `ROUTE.json`, checks the old bootloader +ceiling, reconstructs under both erased and zero-filled workspace assumptions, +then invokes the exact deployed bootloader simulator. """, encoding="ascii", ) @@ -1309,6 +1897,24 @@ def main() -> int: route, route_document = read_route(args.route_plan) route_evidence = route_document route_schema = int(route_document["schema"]) + route_search_summary = route_document.get("search") + transport_route = ( + route_schema == 2 + and isinstance(route_search_summary, dict) + and route_search_summary.get("objective") == TRANSPORT_ROUTE_OBJECTIVE + ) + transport_relay_hops = 0 + if transport_route: + transport_accounting = route_search_summary.get("transport_accounting") + assert isinstance(transport_accounting, dict) + transport_relay_hops = int(transport_accounting["relay_hops"]) + if ( + tools["motatool"]["executable_sha256"] + != TRANSPORT_ROUTE_ENCODER_SHA256 + ): + raise CompactBuildError( + "motatool executable does not match the audited route encoder" + ) if route_schema == 1 and args.accelerated_images is None: raise CompactBuildError("schema-1 routes require --accelerated-images") if route_schema == 2 and args.image_inventory is None: @@ -1317,6 +1923,7 @@ def main() -> int: raise CompactBuildError("schema-2 routes require --geometry-results") if route_schema == 1 and args.geometry_results is not None: raise CompactBuildError("--geometry-results is only valid for schema-2 routes") + route_inventory: list[dict[str, object]] | None = None if route_schema == 1: assert args.accelerated_images is not None args.accelerated_images = snapshot_schema1_manifest( @@ -1404,6 +2011,12 @@ def main() -> int: else: assert args.image_inventory is not None assert args.geometry_results is not None + try: + route_inventory = route_search.load_inventory(args.image_inventory) + except route_search.RouteSearchError as exc: + raise CompactBuildError( + f"invalid route-search inventory: {exc}" + ) from exc images, sources = read_image_inventory( args.image_inventory, endpoint_node + 1 ) @@ -1443,6 +2056,7 @@ def main() -> int: source_node = int(plan_step["source_node"]) target_node = int(plan_step["target_node"]) memory_size = int(plan_step["inplace_memory"]) + block_size = int(plan_step["block_size"]) base_image = images[source_node] target_image = images[target_node] base_packed, base_body, base_version, _base_body_size = firmware_identity(base_image) @@ -1466,7 +2080,7 @@ def main() -> int: "--base", str(base_path), "--patch-type", "in-place", "--inplace-memory", hex(memory_size), "--segment-size", str(SEGMENT_SIZE), - "--block-size", str(BLOCK_SIZE), "--out", str(output_path), + "--block-size", str(block_size), "--out", str(output_path), ], f"build compact step {number}", ) @@ -1475,7 +2089,19 @@ def main() -> int: build_kind = "freshly generated" run([str(args.motatool), "verify", str(output_path)], f"verify compact step {number}") parsed = motalib.parse_container(output_path.read_bytes()) + if parsed.manifest.block_size != block_size: + raise CompactBuildError( + f"step {number} manifest block size is " + f"{parsed.manifest.block_size}, expected {block_size}" + ) payload_path.write_bytes(parsed.payload) + transport_validation = None + if transport_route: + assert route_inventory is not None + transport_validation = validate_selected_transport( + args.motatool, payload_path, plan_step, route_inventory, + transport_relay_hops, number, + ) patch_memory, patch_segment, patch_from, patch_to = patch_geometry( args.detools, payload_path, f"compact step {number}" ) @@ -1548,12 +2174,13 @@ def main() -> int: "sha256": common.sha256_file(simulator), }) - output_rows.append({ + output_row = { "step": number, "from_version": from_version, "to_version": to_version, "mota_file": f"motas/{filename}", "mota_size": package_size, + "mota_block_size": block_size, "inplace_memory": f"0x{memory_size:X}", "stage_start": f"0x{stage_start:X}", "workspace_end": f"0x{workspace_end:X}", @@ -1563,8 +2190,80 @@ def main() -> int: "target_body_hash": target_body, "target_sha256": target_sha, "source_commit": sources[target_node], - }) - validation_steps.append({ + "transport_profile": "", + "transport_relay_hops": "", + "transport_request_pipeline": "", + "transport_payload_sha256": "", + "transport_encoder_sha256": "", + "transport_payload_bytes": "", + "transport_wire_bytes": "", + "transport_deflate_bytes": "", + "transport_deflate_blocks": "", + "transport_data_packets": "", + "transport_request_packets": "", + "transport_manifest_packets": "", + "transport_proof_request_packets": "", + "transport_proof_packets": "", + "transport_packets": "", + "transport_manifest_bytes": "", + "transport_data_bytes": "", + "transport_block_request_bytes": "", + "transport_proof_request_bytes": "", + "transport_proof_response_bytes": "", + "transport_origin_mesh_bytes": "", + "transport_linear_path_bytes": "", + "baseline_1k_raw_packets": "", + "baseline_1k_raw_origin_mesh_bytes": "", + "baseline_1k_raw_linear_path_bytes": "", + } + if transport_validation is not None: + output_row.update({ + "transport_profile": transport_validation["profile"], + "transport_relay_hops": transport_relay_hops, + "transport_request_pipeline": transport_validation[ + "request_pipeline" + ], + "transport_payload_sha256": transport_validation["payload_sha256"], + "transport_encoder_sha256": ( + transport_validation["route_encoder_sha256"] or "" + ), + "transport_payload_bytes": transport_validation["payload_bytes"], + "transport_wire_bytes": transport_validation["wire_bytes"], + "transport_deflate_bytes": transport_validation["deflate_bytes"], + "transport_deflate_blocks": transport_validation["deflate_blocks"], + "transport_data_packets": transport_validation["data_packets"], + "transport_request_packets": transport_validation["request_packets"], + "transport_manifest_packets": transport_validation["manifest_packets"], + "transport_proof_request_packets": transport_validation[ + "proof_request_packets" + ], + "transport_proof_packets": transport_validation["proof_packets"], + "transport_packets": transport_validation["packets"], + "transport_manifest_bytes": transport_validation["manifest_bytes"], + "transport_data_bytes": transport_validation["data_bytes"], + "transport_block_request_bytes": transport_validation[ + "block_request_bytes" + ], + "transport_proof_request_bytes": transport_validation[ + "proof_request_bytes" + ], + "transport_proof_response_bytes": transport_validation[ + "proof_response_bytes" + ], + "transport_origin_mesh_bytes": transport_validation["origin_mesh_bytes"], + "transport_linear_path_bytes": transport_validation["linear_path_bytes"], + "baseline_1k_raw_packets": transport_validation[ + "one_kib_no_compression" + ]["packets"], + "baseline_1k_raw_origin_mesh_bytes": transport_validation[ + "one_kib_no_compression" + ]["origin_mesh_bytes"], + "baseline_1k_raw_linear_path_bytes": transport_validation[ + "one_kib_no_compression" + ]["linear_path_bytes"], + }) + output_rows.append(output_row) + validation_step = { "step": number, "source_node": source_node, "target_node": target_node, @@ -1575,14 +2274,24 @@ def main() -> int: "staging_margin": staging_margin, "mota_sha256": common.sha256_file(output_path), "mota_size": package_size, + "block_size": block_size, "target_sha256": target_sha, "zero_fill_apply": "passed", "erased_fill_apply": "passed", "apply_simulators": simulator_results, - }) + } + if transport_validation is not None: + validation_step["transport"] = transport_validation + validation_steps.append(validation_step) + transport_text = ( + f" radio={transport_validation['profile']} " + f"bytes={transport_validation['linear_path_bytes']}" + if transport_validation is not None else "" + ) print( f"[compact] {number:02d}/{expected_steps} nodes={source_node}->{target_node} " - f"memory=0x{memory_size:X} size={package_size} margin={staging_margin}", + f"memory=0x{memory_size:X} size={package_size} margin={staging_margin}" + f"{transport_text}", flush=True, ) @@ -1668,13 +2377,134 @@ def main() -> int: "app_base": APP_BASE, "stage_ceiling": STAGE_CEILING, "segment_size": SEGMENT_SIZE, - "block_size": BLOCK_SIZE, + "block_size_policy": { + "source_max_block_field": "transport_max_block_bytes", + "supported_block_sizes": [LEGACY_BLOCK_SIZE, DEFLATE_BLOCK_SIZE], + "source_deflate_field": "ota_transport_deflate", + "source_profile_matrix": dict(route_search.TRANSPORT_PROFILE_MATRIX), + }, "steps": validation_steps, } if route_schema == 2: validation["geometry_results_sha256"] = geometry_sha validation["route_search_source_sha256"] = route_search_sha validation["image_inventory_sha256"] = bridge_sha + if transport_route: + transport_validations = [ + step["transport"] for step in validation_steps + if isinstance(step.get("transport"), dict) + ] + if len(transport_validations) != expected_steps: + raise CompactBuildError("not every selected step has transport validation") + selected_container_bytes = sum( + int(row["mota_size"]) for row in output_rows + ) + selected_transport_bytes = sum( + int(item["linear_path_bytes"]) for item in transport_validations + ) + baseline_transport_bytes = sum( + int(item["one_kib_no_compression"]["linear_path_bytes"]) + for item in transport_validations + ) + route_encoder_hashes = { + str(item["route_encoder_sha256"]) + for item in transport_validations + if item.get("route_encoder_sha256") is not None + } + if len(route_encoder_hashes) > 1: + raise CompactBuildError("selected steps mix route transport encoders") + selected_route_encoder = next(iter(route_encoder_hashes), None) + selected_packets = sum( + int(item["packets"]) for item in transport_validations + ) + baseline_packets = sum( + int(item["one_kib_no_compression"]["packets"]) + for item in transport_validations + ) + component_totals = { + field: sum(int(item[field]) for item in transport_validations) + for field in ( + "manifest_packets", "data_packets", "request_packets", + "proof_request_packets", "proof_packets", "manifest_bytes", + "data_bytes", "block_request_bytes", "proof_request_bytes", + "proof_response_bytes", + ) + } + if selected_packets != sum( + component_totals[field] + for field in ( + "manifest_packets", "data_packets", "request_packets", + "proof_request_packets", "proof_packets", + ) + ) or sum( + component_totals[field] + for field in ( + "manifest_bytes", "data_bytes", "block_request_bytes", + "proof_request_bytes", "proof_response_bytes", + ) + ) != sum( + int(item["origin_mesh_bytes"]) for item in transport_validations + ): + raise CompactBuildError("transport component totals are inconsistent") + if ( + selected_container_bytes != route_evidence.get("selected_total_bytes") + or selected_transport_bytes + != route_evidence.get("selected_total_transport_bytes") + ): + raise CompactBuildError("selected transport totals do not match route evidence") + validation["transport_validation"] = { + "status": "passed", + "relay_hops": transport_relay_hops, + "route_encoder_sha256": selected_route_encoder, + "selected_total_container_bytes": selected_container_bytes, + "selected_total_transport_bytes": selected_transport_bytes, + "selected_total_packets": selected_packets, + "selected_total_payload_bytes": sum( + int(item["payload_bytes"]) for item in transport_validations + ), + "selected_total_wire_bytes": sum( + int(item["wire_bytes"]) for item in transport_validations + ), + "selected_total_deflate_bytes": sum( + int(item["deflate_bytes"]) for item in transport_validations + ), + "selected_total_deflate_blocks": sum( + int(item["deflate_blocks"]) for item in transport_validations + ), + "component_totals": component_totals, + "one_kib_no_compression_baseline": { + "description": ( + "same selected route with 1 KiB logical blocks and no " + "DEFLATE, preserving each step's negotiated 171-byte or " + "legacy 160-byte packet profile" + ), + "block_size": LEGACY_BLOCK_SIZE, + "total_payload_bytes": sum( + int(item["payload_bytes"]) for item in transport_validations + ), + "total_wire_bytes": sum( + int(item["one_kib_no_compression"]["wire_bytes"]) + for item in transport_validations + ), + "total_transport_bytes": baseline_transport_bytes, + "total_packets": baseline_packets, + "saved_transport_bytes": ( + baseline_transport_bytes - selected_transport_bytes + ), + "saved_packets": baseline_packets - selected_packets, + "saved_transport_percent": round( + 100.0 + * (baseline_transport_bytes - selected_transport_bytes) + / baseline_transport_bytes, + 6, + ), + "saved_packet_percent": round( + 100.0 * (baseline_packets - selected_packets) + / baseline_packets, + 6, + ), + }, + } if physical_validation is not None: validation["physical_validation_record_sha256"] = common.sha256_file( physical_validation_path diff --git a/tools/lora_ota/build_rak3401_inflate_bridges.py b/tools/lora_ota/build_rak3401_inflate_bridges.py index 929ec662..359f326b 100644 --- a/tools/lora_ota/build_rak3401_inflate_bridges.py +++ b/tools/lora_ota/build_rak3401_inflate_bridges.py @@ -29,7 +29,7 @@ import motalib # noqa: E402 ENV_NAME = "RAK_3401_repeater_lora_ota_no_external_sensors" -VERSION_SUFFIX = "halo-keymind-cascade-mota-inflate" +VERSION_SUFFIX = "halo-keymind-cascade-mota-inflate2k" EXPECTED_TARGET_ID = 0x2FA509C1 EXPECTED_HARDWARE = "RAK_3401" COMMON_PATCH = SCRIPT_DIR / "rak3401_terminal_ota_backport.patch" @@ -39,16 +39,18 @@ VERSION_PATCH = SCRIPT_DIR / "rak3401_four_component_endf_backport.patch" WORKSPACE_PATCH = SCRIPT_DIR / "rak3401_dynamic_workspace_backport.patch" SELECTIVE_OS_HOOK = SCRIPT_DIR / "rak3401_selective_os.py" INFLATE_ASSET_ROOT = SCRIPT_DIR / "rak3401_inflate_assets" -INFLATE_SNAPSHOT_COMMIT = "add51bf00c46c15ef54318ca766a6daf08a147ee" +# Provenance of the receive shim these pinned assets extend. The exact amended +# sources are authenticated independently by INFLATE_ASSETS below. +INFLATE_BASE_SNAPSHOT_COMMIT = "add51bf00c46c15ef54318ca766a6daf08a147ee" INFLATE_ASSETS = { "src/helpers/ota/OtaDeflate.cpp": ( "OtaDeflate.cpp", - "7cc464fc3c304cc65f52973593068c8d85a783853da121476c4bc2e196798e8e", + "524328f798c931ce483916cf2c517f04df933d6d9d50d868abc907dc0544bbae", ), "src/helpers/ota/OtaDeflate.h": ( "OtaDeflate.h", - "f6cd5b0fe8b2164178881d93664df37722d6434f8ef61c37d08255bce12f842c", + "39c3e5ca4d90b56f1631c75bb1dd4d8b9589aaa8705b936f46bfa2f890e0c154", ), "src/helpers/ota/OtaTinf.c": ( "OtaTinf.c", @@ -244,10 +246,13 @@ def validate_inflate_source(source: Path) -> None: "void manager receive ABI": "void on_message(const uint8_t* msg, uint16_t len)", "requested block hook declaration": "uint16_t requestedBlockLength(", "requested block hook implementation": "uint16_t OtaManager::requestedBlockLength(", + "nominal block geometry hook": "uint16_t* nominal_size = nullptr", "inflater context member": "OtaTransportInflateReceiver transport_inflate;", "inflater context begin": "transport_inflate.begin(manager, target_id, send, ctx);", "v2 request marker": "OTA_REQ_V2_MARK = 0x8000u", + "2 KiB logical block maximum": "#define OTA_MAX_BLOCK 2048", "v2 fragment size": "#define OTA_FRAG_DATA_V2 171", + "v2 extended length flag": "OTA_REQ_V2_EXTENDED_LENGTH = 0x2000u", "tinf build filter": "+", } haystacks = { @@ -255,10 +260,13 @@ def validate_inflate_source(source: Path) -> None: "void manager receive ABI": manager_h, "requested block hook declaration": manager_h, "requested block hook implementation": manager_cpp, + "nominal block geometry hook": manager_h, "inflater context member": context_h, "inflater context begin": context_h, "v2 request marker": protocol_h, + "2 KiB logical block maximum": manager_h, "v2 fragment size": manager_h, + "v2 extended length flag": protocol_h, "tinf build filter": platformio, } for label, needle in required.items(): @@ -277,6 +285,16 @@ def apply_inflate_transforms(source: Path) -> None: """Apply the add51bf0 receive shim through exact, fail-closed anchors.""" manager_h = source / "src/helpers/ota/OtaManager.h" + replace_exact( + manager_h, + b"#ifndef OTA_MAX_BLOCK\n" + b"#define OTA_MAX_BLOCK 1024 // largest logical block (merkle leaf unit) = reassembly buffer size\n" + b"#endif\n", + b"#ifndef OTA_MAX_BLOCK\n" + b"#define OTA_MAX_BLOCK 2048 // receive bridge accepts extended 2 KiB logical blocks\n" + b"#endif\n", + "OTA logical block maximum", + ) replace_exact( manager_h, b"#ifndef OTA_FRAG_DATA\n" @@ -297,7 +315,8 @@ def apply_inflate_transforms(source: Path) -> None: b" bool terminallyConsumes(const uint8_t* msg, uint16_t len);\n" b" // Exact logical length of a block this receiver currently requested, or zero for unsolicited DATA.\n" b" // Used by the historical transport shim without exposing or changing the manager's reassembly buffers.\n" - b" uint16_t requestedBlockLength(const uint8_t* manifest_id, uint16_t block) const;\n" + b" uint16_t requestedBlockLength(const uint8_t* manifest_id, uint16_t block,\n" + b" uint16_t* nominal_size = nullptr) const;\n" b" void on_message(const uint8_t* msg, uint16_t len); // feed one received OTA message\n", "OtaManager receive shim declaration", ) @@ -306,10 +325,13 @@ def apply_inflate_transforms(source: Path) -> None: replace_exact( manager_cpp, b"bool OtaManager::terminallyConsumes(const uint8_t* msg, uint16_t len) {\n", - b"uint16_t OtaManager::requestedBlockLength(const uint8_t* manifest_id, uint16_t block) const {\n" + b"uint16_t OtaManager::requestedBlockLength(const uint8_t* manifest_id, uint16_t block,\n" + b" uint16_t* nominal_size) const {\n" + b" if (nominal_size) *nominal_size = 0;\n" b" if (!manifest_id || !_fetch || _fstate != FETCHING || block >= _fbc ||\n" b" memcmp(manifest_id, _fid, sizeof(_fid)) != 0 || findReassemblySlot(block) < 0) return 0;\n" b" const uint32_t len = blockLen(block);\n" + b" if (nominal_size && _fbs <= OTA_MAX_BLOCK) *nominal_size = (uint16_t)_fbs;\n" b" return len <= OTA_MAX_BLOCK ? (uint16_t)len : 0;\n" b"}\n\n" b"bool OtaManager::terminallyConsumes(const uint8_t* msg, uint16_t len) {\n", @@ -326,37 +348,54 @@ def apply_inflate_transforms(source: Path) -> None: b"};\n" ) v2_protocol = data_message + ( - b"\n// OTA_REQ want_mask extension. These bits sit outside every valid fragment bit for <=1 KiB blocks. A first\n" - b"// v2 request deliberately includes all seven legacy fragment bits so an old source can answer it completely.\n" + b"\n// OTA_REQ want_mask extension. These bits sit outside every valid fragment bit: a 2 KiB block needs at\n" + b"// most 13 legacy or 12 v2 fragments. A first request includes all legacy fragment bits so an old source\n" + b"// can answer a 1 KiB block completely.\n" b"static const uint16_t OTA_REQ_V2_MARK = 0x8000u;\n" b"static const uint16_t OTA_REQ_V2_ALLOW_DEFLATE = 0x4000u;\n" - b"static const uint16_t OTA_REQ_V2_RESERVED = 0x2000u;\n" + b"static const uint16_t OTA_REQ_V2_EXTENDED_LENGTH = 0x2000u;\n" b"static const uint16_t OTA_REQ_V2_FRAGMENT_MASK = 0x1FFFu;\n" - b"\n// OTA_DATA v2 frag_off packing:\n" - b"// bit 15 v2 marker\n" - b"// bit 14 data[] is one raw-RFC1951-DEFLATE stream fragment (clear = raw block bytes)\n" - b"// bits 13..10 fragment index (0..15; byte offset = index * OTA_FRAG_DATA_V2)\n" - b"// bits 9..0 total encoded block length minus one (1..1024 bytes)\n" + b"static const uint16_t OTA_REQ_V2_EXT_FRAGMENT_MASK = 0x0FFFu;\n" + b"static const uint16_t OTA_REQ_V2_EXT_RESERVED = 0x1000u;\n" + b"\n// The 1 KiB OTA_DATA layout remains byte-for-byte compatible. An extended request selects a 2 KiB layout\n" + b"// with fragment index in bits 14..11 and encoded length minus one in bits 10..0. In that layout an\n" + b"// encoded length shorter than the manifest-derived raw length unambiguously identifies DEFLATE.\n" b"static const uint16_t OTA_DATA_V2_MARK = 0x8000u;\n" b"static const uint16_t OTA_DATA_V2_DEFLATED = 0x4000u;\n" b"static const uint16_t OTA_DATA_V2_FRAGMENT_BITS = 0x3C00u;\n" b"static const uint16_t OTA_DATA_V2_LENGTH_BITS = 0x03FFu;\n" b"static const uint8_t OTA_DATA_V2_FRAGMENT_SHIFT = 10;\n" - b"static const uint16_t OTA_DATA_V2_MAX_ENCODED = 1024;\n" + b"static const uint16_t OTA_DATA_V2_LEGACY_MAX_ENCODED = 1024;\n" + b"static const uint16_t OTA_DATA_V2_EXT_FRAGMENT_BITS = 0x7800u;\n" + b"static const uint16_t OTA_DATA_V2_EXT_LENGTH_BITS = 0x07FFu;\n" + b"static const uint8_t OTA_DATA_V2_EXT_FRAGMENT_SHIFT = 11;\n" + b"static const uint16_t OTA_DATA_V2_MAX_ENCODED = 2048;\n" b"static const uint8_t OTA_DATA_V2_STREAM_ID_BYTES = 4;\n" b"\ninline bool ota_req_is_v2(uint16_t want_mask) {\n" - b" return (want_mask & OTA_REQ_V2_MARK) != 0 && (want_mask & OTA_REQ_V2_RESERVED) == 0;\n" + b" if ((want_mask & OTA_REQ_V2_MARK) == 0) return false;\n" + b" return (want_mask & OTA_REQ_V2_EXTENDED_LENGTH) == 0 ||\n" + b" (want_mask & OTA_REQ_V2_EXT_RESERVED) == 0;\n" b"}\n" b"\ninline uint16_t ota_req_v2_fragments(uint16_t want_mask) {\n" - b" return (uint16_t)(want_mask & OTA_REQ_V2_FRAGMENT_MASK);\n" + b" const uint16_t mask = (want_mask & OTA_REQ_V2_EXTENDED_LENGTH)\n" + b" ? OTA_REQ_V2_EXT_FRAGMENT_MASK : OTA_REQ_V2_FRAGMENT_MASK;\n" + b" return (uint16_t)(want_mask & mask);\n" b"}\n" - b"\ninline uint16_t ota_req_make_v2(uint16_t fragments, bool allow_deflate) {\n" - b" return (uint16_t)((fragments & OTA_REQ_V2_FRAGMENT_MASK) | OTA_REQ_V2_MARK |\n" - b" (allow_deflate ? OTA_REQ_V2_ALLOW_DEFLATE : 0));\n" + b"\ninline bool ota_req_v2_extended_length(uint16_t want_mask) {\n" + b" return (want_mask & OTA_REQ_V2_EXTENDED_LENGTH) != 0;\n" + b"}\n" + b"\ninline uint16_t ota_req_make_v2(uint16_t fragments, bool allow_deflate,\n" + b" bool extended_length = false) {\n" + b" const uint16_t fragment_mask = extended_length\n" + b" ? OTA_REQ_V2_EXT_FRAGMENT_MASK : OTA_REQ_V2_FRAGMENT_MASK;\n" + b" return (uint16_t)((fragments & fragment_mask) | OTA_REQ_V2_MARK |\n" + b" (allow_deflate ? OTA_REQ_V2_ALLOW_DEFLATE : 0) |\n" + b" (extended_length ? OTA_REQ_V2_EXTENDED_LENGTH : 0));\n" b"}\n" b"\ninline bool ota_data_v2_pack(uint8_t fragment, uint16_t encoded_len, bool deflated,\n" b" uint16_t& packed) {\n" - b" if (fragment >= 16 || encoded_len == 0 || encoded_len > OTA_DATA_V2_MAX_ENCODED) return false;\n" + b" if (fragment >= 16 || encoded_len == 0 ||\n" + b" encoded_len > OTA_DATA_V2_LEGACY_MAX_ENCODED) return false;\n" b" packed = (uint16_t)(OTA_DATA_V2_MARK |\n" b" (deflated ? OTA_DATA_V2_DEFLATED : 0) |\n" b" ((uint16_t)fragment << OTA_DATA_V2_FRAGMENT_SHIFT) |\n" @@ -371,6 +410,26 @@ def apply_inflate_transforms(source: Path) -> None: b" deflated = (packed & OTA_DATA_V2_DEFLATED) != 0;\n" b" return true;\n" b"}\n" + b"\ninline bool ota_data_v2_pack_extended(uint8_t fragment, uint16_t encoded_len,\n" + b" uint16_t& packed) {\n" + b" if (fragment >= 16 || encoded_len == 0 || encoded_len > OTA_DATA_V2_MAX_ENCODED) return false;\n" + b" packed = (uint16_t)(OTA_DATA_V2_MARK |\n" + b" ((uint16_t)fragment << OTA_DATA_V2_EXT_FRAGMENT_SHIFT) |\n" + b" (encoded_len - 1u));\n" + b" return true;\n" + b"}\n" + b"\ninline bool ota_data_v2_unpack_extended(uint16_t packed, uint16_t raw_len,\n" + b" uint8_t& fragment, uint16_t& encoded_len,\n" + b" bool& deflated) {\n" + b" if ((packed & OTA_DATA_V2_MARK) == 0 || raw_len == 0 ||\n" + b" raw_len > OTA_DATA_V2_MAX_ENCODED) return false;\n" + b" fragment = (uint8_t)((packed & OTA_DATA_V2_EXT_FRAGMENT_BITS) >>\n" + b" OTA_DATA_V2_EXT_FRAGMENT_SHIFT);\n" + b" encoded_len = (uint16_t)((packed & OTA_DATA_V2_EXT_LENGTH_BITS) + 1u);\n" + b" if (encoded_len > raw_len) return false;\n" + b" deflated = encoded_len < raw_len;\n" + b" return true;\n" + b"}\n" ) replace_exact(protocol_h, data_message, v2_protocol, "OTA v2 protocol helpers") @@ -679,7 +738,7 @@ def main() -> int: "target_id": f"{EXPECTED_TARGET_ID:08X}", "hardware": EXPECTED_HARDWARE, "version_suffix": VERSION_SUFFIX, - "inflate_snapshot_commit": INFLATE_SNAPSHOT_COMMIT, + "inflate_base_snapshot_commit": INFLATE_BASE_SNAPSHOT_COMMIT, "inflate_assets": asset_metadata, "builder_script": Path(__file__).name, "builder_script_sha256": sha256_file(Path(__file__)), @@ -696,6 +755,8 @@ def main() -> int: "selective_os_hook": SELECTIVE_OS_HOOK.name, "selective_os_hook_sha256": sha256_file(SELECTIVE_OS_HOOK), "transport_fragment_bytes": 171, + "transport_max_block_bytes": 2048, + "transport_descriptor_layouts": ["v2-1k", "v2-2k-extended"], "transport_codec": "raw-deflate-receive-only", "fetch_pipeline": 1, "targets": records, diff --git a/tools/lora_ota/lora_ota.py b/tools/lora_ota/lora_ota.py index ed92aaa0..6a5fe571 100755 --- a/tools/lora_ota/lora_ota.py +++ b/tools/lora_ota/lora_ota.py @@ -57,7 +57,8 @@ ENDF_MAGIC = b"EndF" ENDF_SIZE = 56 MAX_ARCHIVE_MEMBER_SIZE = 64 * 1024 * 1024 MAX_FIRMWARE_IMAGE_SIZE = 64 * 1024 * 1024 -MOTA_MAX_BLOCK_SIZE = 1024 +LEGACY_TARGET_MAX_BLOCK_SIZE = 1024 +MOTA_MAX_BLOCK_SIZE = 2048 TRANSMISSION_RETRY_LIMIT = 3 TRANSMISSION_RETRY_WINDOW_SECONDS = 90 TRANSMISSION_RETRY_DELAY_SECONDS = 2 @@ -230,6 +231,8 @@ class TargetInfo: current_version: str | None = None current_version_source: str | None = None nrf_qspi: bool = False + # Firmware predating the live maxblk marker had 1 KiB receive buffers. + max_block_size: int = LEGACY_TARGET_MAX_BLOCK_SIZE @property def nrf_external(self) -> bool: @@ -832,7 +835,7 @@ def parse_mota(blob: bytes, path: Path | None = None) -> MotaInfo: target_id, fw_version, image_size, payload_size = struct.unpack_from(" 10: + if block_size_log2 == 0 or block_size_log2 > 11: raise OtaError( f"invalid mOTA block size; firmware supports at most " f"{MOTA_MAX_BLOCK_SIZE} bytes" @@ -1002,6 +1005,11 @@ def compatible_mota(info: MotaInfo, target: TargetInfo) -> tuple[bool, str]: return False, f"target {info.target_id:08X}, need {target.target_id:08X}" if info.hw_id and target.hw_id and info.hw_id != target.hw_id: return False, f"hardware {info.hw_id!r}, destination is {target.hw_id!r}" + if info.block_size > target.max_block_size: + return False, ( + f"block size {info.block_size} bytes exceeds destination " + f"maxblk:{target.max_block_size}" + ) if info.is_full: if target.platform == "nrf52" and not target.nrf_external: return False, "internal-flash nRF52 accepts only an in-place delta" @@ -1382,6 +1390,8 @@ def prepare_package( "build", "--fw", str(new_image), + "--block-size", + str(target.max_block_size), "--out", str(output), ] @@ -2359,6 +2369,33 @@ def split_host_port(value: str, default_port: int) -> tuple[str, int]: return host, port +def parse_target_max_block_size(status: str, self_status: str) -> int: + """Read the live application receive limit, with a safe deployed fallback.""" + markers = re.findall( + r"\bmaxblk:(\d+)\b", f"{status} {self_status}", re.IGNORECASE + ) + if not markers: + return LEGACY_TARGET_MAX_BLOCK_SIZE + values = {int(marker) for marker in markers} + if len(values) != 1: + rendered = ", ".join(str(value) for value in sorted(values)) + raise OtaError( + f"destination reports conflicting maxblk capabilities: {rendered}" + ) + value = values.pop() + if ( + value <= 0 + or value > MOTA_MAX_BLOCK_SIZE + or value & (value - 1) + ): + raise OtaError( + f"destination reports unsupported maxblk:{value}; this runner " + f"supports power-of-two application blocks through " + f"{MOTA_MAX_BLOCK_SIZE} bytes" + ) + return value + + def query_target( controller: Controller, args: argparse.Namespace, @@ -2381,6 +2418,7 @@ def query_target( if not hash_match: raise OtaError("could not read destination base hash from `ota self`") base_hash = bytes.fromhex(hash_match.group(1)) + max_block_size = parse_target_max_block_size(status, self_status) combined = f"{status} {self_status}" if reported_platform is None: platform = ( @@ -2479,6 +2517,7 @@ def query_target( current_version=current_version, current_version_source=current_version_source, nrf_qspi=nrf_qspi, + max_block_size=max_block_size, ) @@ -3894,6 +3933,7 @@ def confirm_update( print("\nValidated update plan:") print(f" destination : {target.name} ({target.target_id:08X}, {target.platform})") print(f" running base: {target.base_hash.hex().upper()}") + print(f" max block : {target.max_block_size} bytes") if target.platform == "nrf52": version = target.bootloader_version or "unknown" print( @@ -7023,6 +7063,8 @@ def offline_target(args: argparse.Namespace) -> TargetInfo: status="offline", self_status="offline", nrf_qspi=args.nrf_qspi, + # There is no live query in prepare-only mode; retain the current application default. + max_block_size=MOTA_MAX_BLOCK_SIZE, ) diff --git a/tools/lora_ota/rak3401_inflate_assets/OtaDeflate.cpp b/tools/lora_ota/rak3401_inflate_assets/OtaDeflate.cpp index 6e98b2cd..efbd2857 100644 --- a/tools/lora_ota/rak3401_inflate_assets/OtaDeflate.cpp +++ b/tools/lora_ota/rak3401_inflate_assets/OtaDeflate.cpp @@ -115,7 +115,12 @@ void OtaTransportInflateReceiver::send(const uint8_t* msg, uint16_t len, bool fl } if (!_legacy_session) { - request.want_mask = ota_req_make_v2(fragments, _allow_deflate); + uint16_t nominal_block_size = 0; + (void)_manager->requestedBlockLength(request.manifest_id, request.block_idx, + &nominal_block_size); + request.want_mask = ota_req_make_v2( + fragments, _allow_deflate, + nominal_block_size > OTA_DATA_V2_LEGACY_MAX_ENCODED); uint8_t wire[16]; const uint16_t wire_len = encode_req(wire, sizeof(wire), request); if (wire_len != 0) { @@ -162,13 +167,20 @@ bool OtaTransportInflateReceiver::handle_v2_data(const uint8_t* msg, uint16_t le if (!decode_data(msg, len, data) || !is_active(data.manifest_id, data.block_idx) || data.data_len <= OTA_DATA_V2_STREAM_ID_BYTES || _legacy_session) return false; - const uint16_t block_len = _manager->requestedBlockLength(data.manifest_id, data.block_idx); + uint16_t nominal_block_size = 0; + const uint16_t block_len = _manager->requestedBlockLength( + data.manifest_id, data.block_idx, &nominal_block_size); if (block_len == 0) return false; uint8_t fragment = 0; uint16_t encoded_len = 0; bool deflated = false; - if (!ota_data_v2_unpack(data.frag_off, fragment, encoded_len, deflated)) return false; + const bool extended_length = nominal_block_size > OTA_DATA_V2_LEGACY_MAX_ENCODED; + const bool unpacked = extended_length + ? ota_data_v2_unpack_extended(data.frag_off, block_len, + fragment, encoded_len, deflated) + : ota_data_v2_unpack(data.frag_off, fragment, encoded_len, deflated); + if (!unpacked) return false; if (encoded_len > block_len || (deflated ? encoded_len >= block_len : encoded_len != block_len) || (deflated && !_allow_deflate)) return false; diff --git a/tools/lora_ota/rak3401_inflate_assets/OtaDeflate.h b/tools/lora_ota/rak3401_inflate_assets/OtaDeflate.h index 8e54e068..921ccf1a 100644 --- a/tools/lora_ota/rak3401_inflate_assets/OtaDeflate.h +++ b/tools/lora_ota/rak3401_inflate_assets/OtaDeflate.h @@ -15,10 +15,11 @@ bool ota_transport_inflate(void* context, const uint8_t* src, uint16_t src_len, uint8_t* dst, uint16_t dst_cap, uint16_t* dst_len); // Receive-only compatibility shim for historical OTA managers. The wrapped manager continues to reassemble -// canonical 160-byte DATA, verify the Merkle proof, and stage the original `.mota`. This shim only negotiates -// the v2 wire profile, locks and reassembles one encoded representation, exact-inflates it, and injects the -// resulting logical block back into that unchanged path. Bridge builds use OTA_FETCH_PIPELINE=1, so one fixed -// encoded block and one fixed decoded block cover every in-flight request without heap allocation. +// canonical 160-byte DATA, verify the Merkle proof, and stage the original `.mota`. This shim negotiates both +// the original 1 KiB and extended 2 KiB v2 wire descriptors, locks and reassembles one encoded representation, +// exact-inflates it, and injects the resulting logical block back into that unchanged path. Bridge builds use +// OTA_FETCH_PIPELINE=1, so one fixed encoded block and one fixed decoded block cover every in-flight request +// without heap allocation. class OtaTransportInflateReceiver { public: void begin(OtaManager& manager, uint32_t target_id, OtaSend send, void* send_ctx); diff --git a/tools/lora_ota/rak3401_mota_chain.py b/tools/lora_ota/rak3401_mota_chain.py index 40f1e25a..c0b65582 100755 --- a/tools/lora_ota/rak3401_mota_chain.py +++ b/tools/lora_ota/rak3401_mota_chain.py @@ -29,24 +29,42 @@ except ImportError: import lora_ota as ota -# The published b40d2e6c prerelease keeps the first nine package bytes from the +# The published 3f6eddd5 prerelease keeps the first nine package bytes from the # physically passed fd98bc90 chain and replaces its endpoint package. The new -# step 10 and this hardened host runner have exhaustive offline/simulator -# evidence, but not a clean physical transition run, so live use remains gated. +# step 10 and this updated host runner have exhaustive offline/exact-bootloader +# simulator evidence, but not a clean physical transition run, so live use +# remains gated. RELEASE_URL = ( "https://github.com/mikecarper/MeshCore/releases/tag/" - "rak3401-mota-v1.16.07-c1caa5ad-to-v1.17.1.5-b40d2e6c" + "rak3401-mota-v1.16.07-c1caa5ad-to-v1.17.1.6-3f6eddd5" +) +ASSET_NAME = "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.6-3f6eddd5.zip" +ASSET_SHA256 = "135783dd8777490db2422a7543f7d80e3cf03f621d5ac362da930abbc1850dc4" +CHECKSUM_LIST_SHA256 = "93835ed497c579be0a2322292e7f15ab095e7c8ab7892833fe7df3130dbdaba4" +BUNDLE_ROOT_NAME = "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.6-3f6eddd5" + +# Preserve the exact b40d2e6c prerelease that was primary before 3f6eddd5. It +# remains a separately recognized, hidden-lab-ack-gated superseded artifact; +# sharing its first nine targets with the current release must never make the +# archives interchangeable. +SUPERSEDED_B40_ASSET_NAME = ( + "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-b40d2e6c.zip" +) +SUPERSEDED_B40_ASSET_SHA256 = ( + "8e7f0c2565f37d4372fa51dc3ba72b591e7f38b74be09303e48641623f99bce4" +) +SUPERSEDED_B40_CHECKSUM_LIST_SHA256 = ( + "29a02a6c4c83eafa80168a80562b0ded5d8432e6cc02cfaca289cb2ed426681a" +) +SUPERSEDED_B40_ROOT_NAME = ( + "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-b40d2e6c" ) -ASSET_NAME = "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-b40d2e6c.zip" -ASSET_SHA256 = "8e7f0c2565f37d4372fa51dc3ba72b591e7f38b74be09303e48641623f99bce4" -CHECKSUM_LIST_SHA256 = "29a02a6c4c83eafa80168a80562b0ded5d8432e6cc02cfaca289cb2ed426681a" -BUNDLE_ROOT_NAME = "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-b40d2e6c" # Preserve the exact fd98bc90 candidate whose ten package transitions passed # directly on the physical RAK3401 and whose endpoint passed independent SWD # readback. It remains a separately identified, hidden-lab-ack-gated legacy -# artifact; sharing its version and first nine targets with b40d2e6c must never -# make the two archives interchangeable. +# artifact; sharing its version and first nine targets with later candidates +# must never make the archives interchangeable. PHYSICALLY_PASSED_FD98_ASSET_NAME = ( "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-fd98bc90.zip" ) @@ -160,6 +178,7 @@ MAX_CHECKSUM_LIST_BYTES = 1024 * 1024 MAX_PROGRESS_BYTES = 1024 * 1024 PINNED_ARCHIVE_CHECKSUMS = { ASSET_SHA256: CHECKSUM_LIST_SHA256, + SUPERSEDED_B40_ASSET_SHA256: SUPERSEDED_B40_CHECKSUM_LIST_SHA256, PHYSICALLY_PASSED_FD98_ASSET_SHA256: ( PHYSICALLY_PASSED_FD98_CHECKSUM_LIST_SHA256 ), @@ -185,9 +204,12 @@ EXPECTED_HARDWARE = "RAK_3401" EXPECTED_STEP_COUNT = 10 PHYSICALLY_QUALIFIED_9_STEP_COUNT = 9 EXPECTED_START_VERSION = "1.16.7.0" -EXPECTED_FINAL_VERSION = "1.17.1.5" +EXPECTED_FINAL_VERSION = "1.17.1.6" +SUPERSEDED_B40_FINAL_VERSION = "1.17.1.5" +PHYSICALLY_PASSED_FD98_FINAL_VERSION = "1.17.1.5" PHYSICALLY_QUALIFIED_9_FINAL_VERSION = "1.17.1.02" SUPERSEDED_FINAL_VERSION = "1.17.1.0" +EXPECTED_BOOTLOADER_VERSION = "0.9.2-OTAFIX2.4" WATCHDOG_RESET_WAIT_SECONDS = 90 WATCHDOG_STABILITY_WAIT_SECONDS = 90 TRANSFER_SETTINGS_FILE = "target-transfer-settings.json" @@ -265,8 +287,8 @@ COMPACT_RELEASE_ANCHORS = ( ) # Every target in the exact legacy fd98bc90 sequence whose package transitions -# passed directly on the physical RAK3401. The b40d2e6c prerelease reuses the -# first nine package bytes exactly, but has its own endpoint pin below. +# passed directly on the physical RAK3401. Later prereleases reuse the first +# nine package bytes exactly, but each has its own endpoint pin below. PHYSICALLY_PASSED_FD98_ANCHORS = ( (1, "8364257a2b3a219905e870fad6fbb2040a96ca4b4bb7201b2867534cc2b45530"), (2, "ac5f50e5028378ccfe6ea08bbf32f227f50fdcf5285a7deb866e309fbdd0a88f"), @@ -280,11 +302,16 @@ PHYSICALLY_PASSED_FD98_ANCHORS = ( (10, "31c182c888ceb1135e5afb2376610d93cee2e807b556c838e07fd4486c79d095"), ) -CURRENT_10_CANDIDATE_ANCHORS = ( +SUPERSEDED_B40_ANCHORS = ( *PHYSICALLY_PASSED_FD98_ANCHORS[:9], (10, "a7761da8af1f2447d7f01d72e2b1456b99c0e874a165d6483f48ab1f58e8d905"), ) +CURRENT_10_CANDIDATE_ANCHORS = ( + *PHYSICALLY_PASSED_FD98_ANCHORS[:9], + (10, "cd1e9b819f9e918f9f2ceaf37b8a74bf242e2be78f319af2cf02a42d7729d0f4"), +) + # Exact anchors for every image in the accelerated 30-step release. The outer # asset and inner checksum-list pins cover every byte as well; the complete # anchor table keeps structural validation fail-closed if those layers are @@ -363,13 +390,21 @@ SUPERSEDED_30_MESSAGE = ( "the deployed bootloader. Use --verify-only for the older bundle." ) CURRENT_10_CANDIDATE_MESSAGE = ( - "live installation of the exact published b40d2e6c ten-step prerelease is " + "live installation of the exact published 3f6eddd5 ten-step prerelease is " "disabled: steps 1-9 are byte-identical to package transitions that " - "completed directly on the physical RAK3401, but the new b40d2e6c step 10 " - "and this hardened host runner have passed offline reconstruction, motatool " - "verification, and both bootloader simulators only. The exact final " - "transition has not had a clean physical run. Use --verify-only, or the " - "hidden controlled-lab override, until that qualification is complete." + "completed directly on the physical RAK3401, but the new 3f6eddd5 step 10 " + "and this updated host runner have passed offline reconstruction, motatool " + "verification, and the exact OTAFIX2.4 bootloader simulator only. The exact " + "final transition has not had a clean physical run. Use --verify-only, or " + "the hidden controlled-lab override, until that qualification is complete." +) +SUPERSEDED_B40_MESSAGE = ( + "live installation of the exact superseded b40d2e6c ten-step prerelease is " + "disabled: steps 1-9 are byte-identical to package transitions that " + "completed directly on the physical RAK3401, but its step 10 and this " + "updated host runner have not had a clean physical run. Use --verify-only " + "with the explicit superseded bundle path, or the hidden controlled-lab " + "override." ) PHYSICALLY_PASSED_FD98_MESSAGE = ( "live installation of the exact legacy fd98bc90 ten-step candidate is " @@ -398,7 +433,9 @@ def matches_exact_anchors( def ten_step_candidate_kind(steps: list[ChainStep]) -> str | None: if matches_exact_anchors(steps, CURRENT_10_CANDIDATE_ANCHORS): - return "current-b40d2e6c" + return "current-3f6eddd5" + if matches_exact_anchors(steps, SUPERSEDED_B40_ANCHORS): + return "superseded-b40d2e6c" if matches_exact_anchors(steps, PHYSICALLY_PASSED_FD98_ANCHORS): return "legacy-fd98bc90" return None @@ -406,15 +443,25 @@ def ten_step_candidate_kind(steps: list[ChainStep]) -> str | None: def ten_step_verification_message(steps: list[ChainStep]) -> str: candidate_kind = ten_step_candidate_kind(steps) - if candidate_kind == "current-b40d2e6c": + if candidate_kind == "current-3f6eddd5": return ( - "Current b40d2e6c ten-step prerelease verified offline: exact " + "Current 3f6eddd5 ten-step prerelease verified offline: exact " "archive/checksum pins, all target anchors, zero-filled and " "erased-workspace reconstruction, independent motatool verification, " - "and both bootloader simulations passed. Steps 1-9 are byte-identical " - "to packages that passed directly on the RAK3401. The new step 10 and " - "this hardened host runner have not had a clean physical run; live " - "use remains gated." + "and the exact OTAFIX2.4 bootloader simulation passed. Steps 1-9 are " + "byte-identical to packages that passed directly on the RAK3401. The " + "new step 10 and this updated host runner have not had a clean " + "physical run; live use remains gated." + ) + if candidate_kind == "superseded-b40d2e6c": + return ( + "Superseded b40d2e6c ten-step prerelease verified offline: exact " + "archive/checksum pins, all target anchors, zero-filled and " + "erased-workspace reconstruction, independent motatool verification, " + "and its pinned bootloader simulations passed. Steps 1-9 are " + "byte-identical to packages that passed directly on the RAK3401. Its " + "step 10 and this updated host runner have not had a clean physical " + "run; live use remains gated." ) if candidate_kind == "legacy-fd98bc90": return ( @@ -435,10 +482,14 @@ def require_live_release_safe( ) -> None: if len(steps) == EXPECTED_STEP_COUNT: candidate_kind = ten_step_candidate_kind(steps) - if candidate_kind == "current-b40d2e6c": + if candidate_kind == "current-3f6eddd5": if args.accept_test_candidate: return raise KnownUnsafeReleaseError(CURRENT_10_CANDIDATE_MESSAGE) + if candidate_kind == "superseded-b40d2e6c": + if args.accept_test_candidate: + return + raise KnownUnsafeReleaseError(SUPERSEDED_B40_MESSAGE) if candidate_kind == "legacy-fd98bc90": if args.accept_test_candidate: return @@ -1008,6 +1059,7 @@ def locate_bundle(args: argparse.Namespace, work_dir: Path) -> Path: ) if actual not in { ASSET_SHA256, + SUPERSEDED_B40_ASSET_SHA256, PHYSICALLY_PASSED_FD98_ASSET_SHA256, PHYSICALLY_QUALIFIED_9_ASSET_SHA256, SUPERSEDED_30_ASSET_SHA256, @@ -1033,6 +1085,7 @@ def read_checksum_entries(bundle_root: Path) -> dict[PurePosixPath, str]: checksum_digest = hashlib.sha256(checksum_bytes).hexdigest() expected_lists = { CHECKSUM_LIST_SHA256, + SUPERSEDED_B40_CHECKSUM_LIST_SHA256, PHYSICALLY_PASSED_FD98_CHECKSUM_LIST_SHA256, PHYSICALLY_QUALIFIED_9_CHECKSUM_LIST_SHA256, SUPERSEDED_30_CHECKSUM_LIST_SHA256, @@ -1292,6 +1345,10 @@ def parse_chain(bundle_root: Path) -> tuple[list[ChainStep], bytes]: relative = safe_relative_path(row["mota_file"], "mOTA path") path = bundle_root.joinpath(*relative.parts) size = int(row["mota_size"]) + declared_block_size = row.get("mota_block_size", "") + block_size = ( + int(declared_block_size) if declared_block_size else None + ) target_image_size = int(row["target_image_size"]) base_hash = bytes.fromhex(row["base_body_hash"]) target_sha256 = row["target_sha256"].lower() @@ -1306,6 +1363,10 @@ def parse_chain(bundle_root: Path) -> tuple[list[ChainStep], bytes]: if not path.is_file() or path.stat().st_size != size: raise ota.OtaError(f"chain package size/path mismatch at step {number}: {path}") package = ota.parse_mota(path.read_bytes(), path) + if block_size is not None and package.block_size != block_size: + raise ota.OtaError( + f"step {number} block size does not match CHAIN.csv" + ) expected_version = ota.parse_version(to_version) if expected_version is None or package.fw_version != expected_version: raise ota.OtaError(f"step {number} version does not match its mOTA manifest") @@ -1333,10 +1394,15 @@ def parse_chain(bundle_root: Path) -> tuple[list[ChainStep], bytes]: candidate_kind = ten_step_candidate_kind(steps) if candidate_kind is None: raise ota.OtaError( - "ten-step candidate does not match the exact b40d2e6c or " - "legacy fd98bc90 audited image pins" + "ten-step candidate does not match the exact 3f6eddd5, " + "superseded b40d2e6c, or legacy fd98bc90 audited image pins" ) - expected_final_version = EXPECTED_FINAL_VERSION + if candidate_kind == "current-3f6eddd5": + expected_final_version = EXPECTED_FINAL_VERSION + elif candidate_kind == "superseded-b40d2e6c": + expected_final_version = SUPERSEDED_B40_FINAL_VERSION + else: + expected_final_version = PHYSICALLY_PASSED_FD98_FINAL_VERSION elif len(steps) == PHYSICALLY_QUALIFIED_9_STEP_COUNT: for number, expected_sha256 in COMPACT_RELEASE_ANCHORS: if steps[number - 1].target_sha256 != expected_sha256: @@ -1537,6 +1603,12 @@ def query_live_target( ) if target.platform != "nrf52" or target.nrf_external: raise ota.OtaError("live target is not the expected internal-flash nRF52 node") + if target.bootloader_version != EXPECTED_BOOTLOADER_VERSION: + raise ota.OtaError( + "live target bootloader is " + f"{target.bootloader_version or 'unknown'!r}, expected exact deployed " + f"version {EXPECTED_BOOTLOADER_VERSION!r}" + ) if target.bootloader_abi is None or target.bootloader_abi < 2: raise ota.OtaError("live target does not report OTAFIX mOTA ABI 2") if target.bootloader_codecs is None or not target.bootloader_codecs & (1 << ota.MOTA_CODEC_IN_PLACE): @@ -2327,10 +2399,10 @@ def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description=( "Verify the exact published ten-step RAK3401 prerelease from " - "c1caa5ad to b40d2e6c. Its first nine package bytes passed directly " - "on hardware; the new final transition and this hardened host runner " - "have offline/simulator qualification only, so live use remains " - "disabled without the controlled-lab override." + "c1caa5ad to 3f6eddd5. Its first nine package bytes passed directly " + "on hardware; the new final transition and this updated host runner " + "have offline/exact-OTAFIX2.4-simulator qualification only, so live " + "use remains disabled without the controlled-lab override." ) ) parser.add_argument( diff --git a/tools/lora_ota/rak3401_route_search.py b/tools/lora_ota/rak3401_route_search.py index b84b0ede..0d666f28 100755 --- a/tools/lora_ota/rak3401_route_search.py +++ b/tools/lora_ota/rak3401_route_search.py @@ -7,9 +7,12 @@ be migrated safely when new candidate images are inserted. The default secondary objective remains container size. ``--objective transport`` instead measures the exact live motatool encoder and minimizes clean serialized -MeshCore bytes. Inventory boolean ``ota_transport_deflate`` and integer -``ota_fetch_pipeline`` belong to the currently running/source image; the pinned -first bootstrap is always legacy raw. +MeshCore bytes. Inventory fields ``ota_transport_deflate``, +``ota_fetch_pipeline``, and ``transport_max_block_bytes`` belong to the +currently running/source image; the pinned first bootstrap is always legacy +raw. The explicit maximum controls signed-container block geometry independently +of whether DEFLATE is allowed; either DEFLATE or 2 KiB geometry negotiates the +171-byte v2 wire profile. """ from __future__ import annotations @@ -43,9 +46,21 @@ EXPECTED_TARGET_ID = 0x2FA509C1 EXPECTED_HARDWARE = "RAK_3401" TRANSPORT_CAPABILITY = "ota_transport_deflate" TRANSPORT_PIPELINE_FIELD = "ota_fetch_pipeline" +TRANSPORT_MAX_BLOCK_FIELD = "transport_max_block_bytes" TRANSPORT_CAPABILITY_ALIAS = "transport_inflate" TRANSPORT_PIPELINE_ALIAS = "fetch_pipeline" -BLOCK_SIZE = 1024 +LEGACY_BLOCK_SIZE = 1024 +DEFLATE_BLOCK_SIZE = 2048 +TRANSPORT_PROFILE_MATRIX = { + "1024,deflate=false": "legacy-160-raw", + "1024,deflate=true": "v2-171-deflate", + "2048,deflate=false": "v2-171-raw", + "2048,deflate=true": "v2-171-deflate", +} +# Retain the historical name for callers that are specifically dealing with +# the immutable first package. New route geometry must choose a block size from +# the currently running/source image via ``source_block_size``. +BLOCK_SIZE = LEGACY_BLOCK_SIZE LEGACY_DATA_BYTES = 160 V2_DATA_BYTES = 171 MANIFEST_BYTES = 197 @@ -58,7 +73,7 @@ TRANSPORT_FIELDS = [ ] FIELDS = [ "source", "target", "source_sha256", "target_sha256", "memory", - "payload", "container", "stage_start", "margin", "feasible", "error", + "payload", "block_size", "container", "stage_start", "margin", "feasible", "error", ] + TRANSPORT_FIELDS @@ -74,27 +89,46 @@ def align_down(value: int, unit: int = PAGE) -> int: return value // unit * unit -def container_size(payload_size: int) -> int: - return payload_size + 210 + 4 * ((payload_size + 1023) // 1024) +def _validate_block_size(block_size: int) -> int: + if block_size not in (LEGACY_BLOCK_SIZE, DEFLATE_BLOCK_SIZE): + raise RouteSearchError( + f"logical block size must be {LEGACY_BLOCK_SIZE} or {DEFLATE_BLOCK_SIZE}" + ) + return block_size -def payload_size_from_container(total: int) -> int: +def container_size( + payload_size: int, block_size: int = LEGACY_BLOCK_SIZE +) -> int: + block_size = _validate_block_size(block_size) + if payload_size <= 0: + raise RouteSearchError("container payload must be positive") + return payload_size + 210 + 4 * ( + (payload_size + block_size - 1) // block_size + ) + + +def payload_size_from_container( + total: int, block_size: int = LEGACY_BLOCK_SIZE +) -> int: """Invert the fixed mOTA geometry used by the pinned bootstrap package.""" + block_size = _validate_block_size(block_size) if total <= 210: raise RouteSearchError("container is too small to contain a payload") - max_blocks = (total - 210 + BLOCK_SIZE - 1) // BLOCK_SIZE + max_blocks = (total - 210 + block_size - 1) // block_size for blocks in range(1, max_blocks + 1): payload = total - 210 - 4 * blocks - if payload > 0 and (payload + BLOCK_SIZE - 1) // BLOCK_SIZE == blocks: + if payload > 0 and (payload + block_size - 1) // block_size == blocks: return payload raise RouteSearchError(f"container size {total} has no valid payload geometry") -def _block_lengths(payload_size: int) -> list[int]: +def _block_lengths(payload_size: int, block_size: int) -> list[int]: if payload_size <= 0: raise RouteSearchError("transport payload must be positive") - full, tail = divmod(payload_size, BLOCK_SIZE) - result = [BLOCK_SIZE] * full + block_size = _validate_block_size(block_size) + full, tail = divmod(payload_size, block_size) + result = [block_size] * full if tail: result.append(tail) return result @@ -136,7 +170,7 @@ def _common_transport_cost(block_count: int) -> tuple[int, int, int]: def legacy_transport_cost(payload_size: int) -> dict[str, int | str]: """Ideal clean legacy transfer, including requests, DATA, and proofs.""" - blocks = _block_lengths(payload_size) + blocks = _block_lengths(payload_size, LEGACY_BLOCK_SIZE) block_count = len(blocks) data_packets = sum( (length + LEGACY_DATA_BYTES - 1) // LEGACY_DATA_BYTES @@ -151,6 +185,7 @@ def legacy_transport_cost(payload_size: int) -> dict[str, int | str]: manifest_bytes = common_bytes - proof_bytes return { "profile": "legacy-160-raw", + "block_size": LEGACY_BLOCK_SIZE, "payload_bytes": payload_size, "wire_bytes": payload_size, "deflate_bytes": 0, @@ -197,9 +232,10 @@ def v2_transport_cost( deflate_blocks: int, data_packets: int, pipeline_capacity: int = RAK3401_PIPELINE_MAX, + block_size: int = DEFLATE_BLOCK_SIZE, ) -> dict[str, int | str]: """Ideal clean negotiated v2 transfer using measured seeder output.""" - blocks = _block_lengths(payload_size) + blocks = _block_lengths(payload_size, block_size) block_count = len(blocks) raw_data_packets = sum( (length + V2_DATA_BYTES - 1) // V2_DATA_BYTES for length in blocks @@ -226,6 +262,7 @@ def v2_transport_cost( manifest_bytes = common_bytes - proof_bytes return { "profile": "v2-171-deflate", + "block_size": block_size, "payload_bytes": payload_size, "wire_bytes": wire_bytes, "deflate_bytes": deflate_bytes, @@ -247,6 +284,29 @@ def v2_transport_cost( } +def v2_raw_transport_cost( + payload_size: int, + pipeline_capacity: int = RAK3401_PIPELINE_MAX, + block_size: int = LEGACY_BLOCK_SIZE, +) -> dict[str, int | str]: + """Ideal negotiated v2 transfer without compression for a comparison base.""" + blocks = _block_lengths(payload_size, block_size) + data_packets = sum( + (length + V2_DATA_BYTES - 1) // V2_DATA_BYTES for length in blocks + ) + result = v2_transport_cost( + payload_size, + payload_size, + 0, + 0, + data_packets, + pipeline_capacity, + block_size, + ) + result["profile"] = "v2-171-raw" + return result + + def linear_path_bytes(origin_mesh_bytes: int, packets: int, relay_hops: int) -> int: """Serialized bytes for an ideal single path; real flood fan-out is external.""" if origin_mesh_bytes < 0 or packets < 0: @@ -302,6 +362,13 @@ def load_inventory(path: Path) -> list[dict[str, object]]: isinstance(record, dict) and TRANSPORT_CAPABILITY in record for record in images ) + strict_block_contract = strict_transport_contract or ( + isinstance(capability_contract, dict) + and TRANSPORT_MAX_BLOCK_FIELD in capability_contract + ) or any( + isinstance(record, dict) and TRANSPORT_MAX_BLOCK_FIELD in record + for record in images + ) edge_policy = document.get("edge_policy", {}) strict_version_ranks = ( isinstance(edge_policy, dict) @@ -401,6 +468,25 @@ def load_inventory(path: Path) -> list[dict[str, object]]: f"{TRANSPORT_CAPABILITY}" ) record[TRANSPORT_CAPABILITY] = capability + raw_max_block = record.get(TRANSPORT_MAX_BLOCK_FIELD) + if raw_max_block is None: + if strict_block_contract: + raise RouteSearchError( + f"image {index} {TRANSPORT_MAX_BLOCK_FIELD} is required by " + "the inventory capability contract" + ) + max_block = LEGACY_BLOCK_SIZE + else: + max_block = _integer( + raw_max_block, f"image {index} {TRANSPORT_MAX_BLOCK_FIELD}" + ) + if max_block not in (LEGACY_BLOCK_SIZE, DEFLATE_BLOCK_SIZE): + raise RouteSearchError( + f"image {index} {TRANSPORT_MAX_BLOCK_FIELD} must be " + f"{LEGACY_BLOCK_SIZE} or {DEFLATE_BLOCK_SIZE}" + ) + record[TRANSPORT_MAX_BLOCK_FIELD] = max_block + uses_v2 = capability or max_block > LEGACY_BLOCK_SIZE raw_pipeline = record.get(TRANSPORT_PIPELINE_FIELD) if ( raw_pipeline is None @@ -408,12 +494,12 @@ def load_inventory(path: Path) -> list[dict[str, object]]: and TRANSPORT_PIPELINE_ALIAS in record ): raw_pipeline = record[TRANSPORT_PIPELINE_ALIAS] - if capability and raw_pipeline is None: + if uses_v2 and raw_pipeline is None: raise RouteSearchError( - f"image {index} {TRANSPORT_PIPELINE_FIELD} is required when " - f"{TRANSPORT_CAPABILITY} is true" + f"image {index} {TRANSPORT_PIPELINE_FIELD} is required for " + "v2 transport" ) - if capability: + if uses_v2: pipeline = _integer( raw_pipeline, f"image {index} {TRANSPORT_PIPELINE_FIELD}" ) @@ -426,7 +512,7 @@ def load_inventory(path: Path) -> list[dict[str, object]]: elif raw_pipeline is not None and strict_transport_contract: raise RouteSearchError( f"image {index} {TRANSPORT_PIPELINE_FIELD} must be null when " - f"{TRANSPORT_CAPABILITY} is false" + "neither DEFLATE nor 2 KiB blocks are enabled" ) else: record[TRANSPORT_PIPELINE_FIELD] = None @@ -532,7 +618,8 @@ def transport_stats_complete(row: dict[str, object]) -> bool: def normalize_transport_stats( - row: dict[str, object], payload_size: int, label: str + row: dict[str, object], payload_size: int, label: str, + block_size: int = DEFLATE_BLOCK_SIZE, ) -> dict[str, object]: present = [row.get(field) not in (None, "") for field in TRANSPORT_FIELDS] if any(present) and not all(present): @@ -554,16 +641,22 @@ def normalize_transport_stats( int(normalized["v2_deflate_bytes"]), int(normalized["v2_deflate_blocks"]), int(normalized["v2_data_packets"]), + block_size=block_size, ) return normalized -def measure_transport_size(tool: Path, payload: Path) -> dict[str, object]: +def measure_transport_size( + tool: Path, + payload: Path, + block_size: int = DEFLATE_BLOCK_SIZE, +) -> dict[str, object]: """Ask motatool's live-seeder encoder for exact per-block byte counts.""" + block_size = _validate_block_size(block_size) try: completed = subprocess.run( [str(tool), "transport-size", "--payload", str(payload), - "--block-size", str(BLOCK_SIZE)], + "--block-size", str(block_size)], check=False, capture_output=True, text=True, encoding="utf-8", timeout=60, ) @@ -585,9 +678,9 @@ def measure_transport_size(tool: Path, payload: Path) -> dict[str, object]: expected_payload = payload.stat().st_size if measured.get("payload_bytes") != expected_payload: raise RouteSearchError("transport-size tool measured the wrong payload") - if measured.get("block_size") != BLOCK_SIZE: + if measured.get("block_size") != block_size: raise RouteSearchError("transport-size tool used the wrong block size") - expected_blocks = (expected_payload + BLOCK_SIZE - 1) // BLOCK_SIZE + expected_blocks = (expected_payload + block_size - 1) // block_size if measured.get("block_count") != expected_blocks: raise RouteSearchError("transport-size tool returned the wrong block count") row = { @@ -598,7 +691,9 @@ def measure_transport_size(tool: Path, payload: Path) -> dict[str, object]: "v2_deflate_blocks": measured.get("deflate_blocks"), "v2_data_packets": measured.get("data_packets"), } - return normalize_transport_stats(row, expected_payload, str(payload)) + return normalize_transport_stats( + row, expected_payload, str(payload), block_size + ) def migrate_csv( @@ -664,6 +759,7 @@ def migrate_csv( memory = _integer(row["memory"], f"memory at {csv_path}:{line}") try: payload = int(row.get("payload", "")) + recorded_block_size = row.get("block_size", "") total = int(row.get("container", "")) stage_start = int(row.get("stage_start", "")) margin = int(row.get("margin", "")) @@ -681,11 +777,24 @@ def migrate_csv( f"successful cached geometry carries an error at " f"{csv_path}:{line}; regenerate it" ) - expected_total = container_size(payload) - expected_stage = align_down(STAGE_CEILING - expected_total) - expected_margin = expected_stage - (APP_BASE + memory) + expected_block_size = source_block_size(images, source_node) + # Geometry caches predating per-source block sizing have no column + # and therefore contain 1 KiB container arithmetic. The detools + # payload itself does not depend on the Merkle block size, so first + # authenticate that historical arithmetic and then migrate only + # the derived container/staging fields. Transport counters do + # depend on block boundaries and are deliberately invalidated. + if recorded_block_size in (None, ""): + recorded_block_size_int = LEGACY_BLOCK_SIZE + else: + recorded_block_size_int = _integer( + recorded_block_size, f"block_size at {csv_path}:{line}" + ) + recorded_total = container_size(payload, recorded_block_size_int) + recorded_stage = align_down(STAGE_CEILING - recorded_total) + recorded_margin = recorded_stage - (APP_BASE + memory) if (total, stage_start, margin) != ( - expected_total, expected_stage, expected_margin + recorded_total, recorded_stage, recorded_margin ): raise RouteSearchError( f"inconsistent cached geometry at {csv_path}:{line}" @@ -694,16 +803,30 @@ def migrate_csv( raise RouteSearchError( f"cached feasibility is inconsistent at {csv_path}:{line}" ) + block_size = expected_block_size + total = container_size(payload, block_size) + stage_start = align_down(STAGE_CEILING - total) + margin = stage_start - (APP_BASE + memory) + feasible = payload >= 0 and margin >= 0 key = cache_key(source_sha, target_sha, memory) normalized = {field: row.get(field, "") for field in FIELDS} normalized.update(source=source_node, target=target_node, source_sha256=source_sha, target_sha256=target_sha, - memory=memory) - normalized.update(normalize_transport_stats( - row, payload, f"{csv_path}:{line}" - )) + memory=memory, payload=payload, + block_size=block_size, container=total, + stage_start=stage_start, margin=margin, + feasible=feasible, error="") + if recorded_block_size_int == block_size: + normalized.update(normalize_transport_stats( + row, payload, f"{csv_path}:{line}", block_size + )) + else: + normalized.update({field: "" for field in TRANSPORT_FIELDS}) previous = migrated.get(key) - comparable = ("payload", "container", "stage_start", "margin", "feasible", "error") + comparable = ( + "payload", "block_size", "container", "stage_start", "margin", + "feasible", "error", + ) if previous is not None and any( str(previous.get(item, "")) != str(normalized.get(item, "")) for item in comparable @@ -723,7 +846,7 @@ def migrate_csv( return migrated -Job = tuple[int, int, int, str, str, str, str] +Job = tuple[int, int, int, str, str, str, str, int] def is_forward_progress( @@ -745,7 +868,8 @@ def all_jobs(images: list[dict[str, object]]) -> list[Job]: continue jobs.append((1, target, FIXED_MEMORY, str(images[1]["path"]), str(images[target]["path"]), str(images[1]["sha256"]), - str(images[target]["sha256"]))) + str(images[target]["sha256"]), + source_block_size(images, 1))) for source in range(2, endpoint): source_size = int(images[source]["size"]) for target in range(source + 1, endpoint + 1): @@ -756,7 +880,8 @@ def all_jobs(images: list[dict[str, object]]) -> list[Job]: for memory_page in range(first_page, AVAILABLE_PAGES): jobs.append((source, target, memory_page * PAGE, str(images[source]["path"]), str(images[target]["path"]), - str(images[source]["sha256"]), str(images[target]["sha256"]))) + str(images[source]["sha256"]), str(images[target]["sha256"]), + source_block_size(images, source))) return jobs @@ -826,7 +951,7 @@ def job_needs_transport_stats( ) -> bool: return ( objective == "transport" - and source_uses_v2(images, job[0]) + and source_uses_deflate(images, job[0]) and (relevant_pairs is None or (job[0], job[1]) in relevant_pairs) ) @@ -835,8 +960,12 @@ def cache_satisfies_job( row: dict[str, object] | None, needs_transport: bool, expected_encoder_sha256: str | None = None, + expected_block_size: int | None = None, ) -> bool: return row is not None and ( + expected_block_size is None + or int(row.get("block_size") or LEGACY_BLOCK_SIZE) == expected_block_size + ) and ( not needs_transport or not truth(row.get("feasible", False)) or ( @@ -857,13 +986,17 @@ def geometry_job(args: tuple[object, ...]) -> dict[str, object]: "detools is required to generate geometry; run this tool in the " "detools pipx environment" ) from exc - if len(args) not in (8, 9): + if len(args) not in (9, 10): raise RouteSearchError("invalid geometry job") - source, target, memory, from_raw, to_raw, source_sha, target_sha, patches_raw = args[:8] - transport_tool = str(args[8]) if len(args) == 9 and args[8] else "" + ( + source, target, memory, from_raw, to_raw, source_sha, target_sha, + block_size, patches_raw, + ) = args[:9] + transport_tool = str(args[9]) if len(args) == 10 and args[9] else "" source = int(source) target = int(target) memory = int(memory) + block_size = _validate_block_size(int(block_size)) from_raw = str(from_raw) to_raw = str(to_raw) source_sha = str(source_sha) @@ -877,12 +1010,12 @@ def geometry_job(args: tuple[object, ...]) -> dict[str, object]: memory_size=memory, segment_size=SEGMENT, use_mmap=True, ) payload = patch.stat().st_size - total = container_size(payload) + total = container_size(payload, block_size) stage_start = align_down(STAGE_CEILING - total) margin = stage_start - (APP_BASE + memory) error = "" transport = ( - measure_transport_size(Path(transport_tool), patch) + measure_transport_size(Path(transport_tool), patch, block_size) if transport_tool and margin >= 0 else {field: "" for field in TRANSPORT_FIELDS} ) @@ -895,7 +1028,8 @@ def geometry_job(args: tuple[object, ...]) -> dict[str, object]: patch.unlink(missing_ok=True) return {"source": source, "target": target, "source_sha256": source_sha, "target_sha256": target_sha, "memory": memory, "payload": payload, - "container": total, "stage_start": stage_start, "margin": margin, + "block_size": block_size, "container": total, + "stage_start": stage_start, "margin": margin, "feasible": payload >= 0 and margin >= 0, "error": error, **transport} @@ -915,6 +1049,12 @@ def project_cache(cache: dict[tuple[str, str, int], dict[str, object]], ) row.update(source=index[key[0]], target=index[key[1]], source_sha256=key[0], target_sha256=key[1], memory=key[2]) + expected_block_size = source_block_size(images, int(row["source"])) + if int(row.get("block_size") or LEGACY_BLOCK_SIZE) != expected_block_size: + raise RouteSearchError( + f"required cache geometry has the wrong block size: {key}" + ) + row["block_size"] = expected_block_size rows.append(row) return rows @@ -980,11 +1120,50 @@ def write_csv(path: Path, rows: list[dict[str, object]]) -> None: raise -def source_uses_v2(images: list[dict[str, object]], source: int) -> bool: - """Capability is the currently running/source image; bootstrap is raw.""" +def source_block_size(images: list[dict[str, object]], source: int) -> int: + """Logical package/Merkle block size accepted by the running receiver.""" + if source == 0: + return LEGACY_BLOCK_SIZE + block_size = int(images[source].get(TRANSPORT_MAX_BLOCK_FIELD, 0)) + if block_size not in (LEGACY_BLOCK_SIZE, DEFLATE_BLOCK_SIZE): + raise RouteSearchError( + f"source image {source} has no explicit supported application " + "block size" + ) + return block_size + + +def source_uses_deflate(images: list[dict[str, object]], source: int) -> bool: + """Whether the running receiver permits compressed v2 DATA blocks.""" return source != 0 and bool(images[source][TRANSPORT_CAPABILITY]) +def source_uses_v2(images: list[dict[str, object]], source: int) -> bool: + """Whether either independent capability requires the negotiated v2 wire.""" + return source != 0 and ( + source_uses_deflate(images, source) + or source_block_size(images, source) > LEGACY_BLOCK_SIZE + ) + + +def source_transport_profile( + images: list[dict[str, object]], source: int +) -> str: + """Select the profile from the independent compression/block-size axes.""" + if source == 0: + return "legacy-160-raw" + key = ( + f"{source_block_size(images, source)},deflate=" + f"{str(source_uses_deflate(images, source)).lower()}" + ) + try: + return TRANSPORT_PROFILE_MATRIX[key] + except KeyError as exc: + raise RouteSearchError( + f"source image {source} has an unsupported transport capability matrix" + ) from exc + + def source_transport_pipeline(images: list[dict[str, object]], source: int) -> int: """Return the request-flight capacity compiled into the source image.""" pipeline = images[source][TRANSPORT_PIPELINE_FIELD] @@ -1002,7 +1181,7 @@ def transport_encoder_for_rows( str(row["transport_encoder_sha256"]) for row in rows if truth(row.get("feasible", False)) - and source_uses_v2(images, int(row["source"])) + and source_uses_deflate(images, int(row["source"])) and ( relevant_pairs is None or (int(row["source"]), int(row["target"])) in relevant_pairs @@ -1020,13 +1199,26 @@ def edge_transport_cost( row: dict[str, object], images: list[dict[str, object]], relay_hops: int ) -> dict[str, int | str]: source = int(row["source"]) + block_size = source_block_size(images, source) + recorded_block_size = int(row.get("block_size") or block_size) + if recorded_block_size != block_size: + raise RouteSearchError( + f"geometry edge {source}->{int(row['target'])} uses " + f"{recorded_block_size}-byte blocks, expected {block_size}" + ) payload = int(row.get("payload", 0)) if payload <= 0: - payload = payload_size_from_container(int(row["container"])) - if source_uses_v2(images, source): + payload = payload_size_from_container(int(row["container"]), block_size) + elif int(row["container"]) != container_size(payload, block_size): + raise RouteSearchError( + f"geometry edge {source}->{int(row['target'])} container does not " + "match its logical block size" + ) + profile = source_transport_profile(images, source) + if profile == "v2-171-deflate": if not transport_stats_complete(row): raise RouteSearchError( - f"missing transport statistics for capable source edge " + f"missing transport statistics for compressed source edge " f"{source}->{int(row['target'])} memory=0x{int(row['memory']):X}" ) cost = v2_transport_cost( @@ -1036,6 +1228,13 @@ def edge_transport_cost( int(row["v2_deflate_blocks"]), int(row["v2_data_packets"]), source_transport_pipeline(images, source), + block_size, + ) + elif profile == "v2-171-raw": + cost = v2_raw_transport_cost( + payload, + source_transport_pipeline(images, source), + block_size, ) else: cost = legacy_transport_cost(payload) @@ -1064,6 +1263,7 @@ def select_route(rows: list[dict[str, object]], images: list[dict[str, object]], raise RouteSearchError("pinned first package does not fit") baseline_row: dict[str, object] = { "source": 0, "target": 1, "memory": FIXED_MEMORY, + "block_size": LEGACY_BLOCK_SIZE, "container": baseline_size, "stage_start": baseline_stage, "margin": baseline_margin, "feasible": True, } @@ -1160,8 +1360,13 @@ def select_route(rows: list[dict[str, object]], images: list[dict[str, object]], ) common["transport_accounting"] = { "relay_hops": relay_hops, + "legacy_block_size": LEGACY_BLOCK_SIZE, + "supported_block_sizes": [LEGACY_BLOCK_SIZE, DEFLATE_BLOCK_SIZE], + "comparison_baseline_block_size": LEGACY_BLOCK_SIZE, "source_capability_field": TRANSPORT_CAPABILITY, "source_pipeline_field": TRANSPORT_PIPELINE_FIELD, + "source_max_block_field": TRANSPORT_MAX_BLOCK_FIELD, + "source_profile_matrix": dict(TRANSPORT_PROFILE_MATRIX), "first_bootstrap_profile": "legacy-160-raw", "included": ( "OTA messages, repeated 4-byte v2 stream IDs, 171-byte DATA slicing, " @@ -1186,6 +1391,7 @@ def select_route(rows: list[dict[str, object]], images: list[dict[str, object]], for source, target in zip(nodes, nodes[1:]): row = edges[source, target] step = {"source_node": source, "target_node": target, + "block_size": source_block_size(images, source), "inplace_memory": f"0x{int(row['memory']):X}", "reuse_baseline_package": source == 0 and target == 1, "expected_container_size": int(row["container"]), @@ -1197,14 +1403,14 @@ def select_route(rows: list[dict[str, object]], images: list[dict[str, object]], step["transport"] = { key: cost[key] for key in ( "profile", "payload_bytes", "wire_bytes", "deflate_bytes", - "deflate_blocks", "data_packets", "request_packets", + "deflate_blocks", "block_size", "data_packets", "request_packets", "manifest_packets", "request_pipeline", "proof_request_packets", "proof_packets", "packets", "manifest_bytes", "data_bytes", "block_request_bytes", "proof_request_bytes", "proof_response_bytes", "origin_mesh_bytes", "linear_path_bytes", ) } - if source_uses_v2(images, source): + if source_uses_deflate(images, source): step["transport"]["payload_sha256"] = row["payload_sha256"] step["transport"]["encoder_sha256"] = row[ "transport_encoder_sha256" @@ -1284,7 +1490,8 @@ def main(argv: list[str] | None = None) -> int: ) cache: dict[tuple[str, str, int], dict[str, object]] = {} comparable = ( - "payload", "container", "stage_start", "margin", "feasible", "error" + "payload", "block_size", "container", "stage_start", "margin", + "feasible", "error", ) for reuse_number, value in enumerate(args.reuse, 1): left, separator, right = value.partition("=") @@ -1324,7 +1531,11 @@ def main(argv: list[str] | None = None) -> int: required = {cache_key(job[5], job[6], job[2]) for job in jobs} geometry_remaining = [ job for job in jobs - if cache_key(job[5], job[6], job[2]) not in cache + if not cache_satisfies_job( + cache.get(cache_key(job[5], job[6], job[2])), + False, + expected_block_size=job[7], + ) ] print( f"nodes={len(images)} geometries={len(jobs)} " @@ -1359,7 +1570,15 @@ def main(argv: list[str] | None = None) -> int: f"geometry_measured={done}/{len(geometry_remaining)}", flush=True, ) - geometry_completed = required & cache.keys() + geometry_completed = { + cache_key(job[5], job[6], job[2]) + for job in jobs + if cache_satisfies_job( + cache.get(cache_key(job[5], job[6], job[2])), + False, + expected_block_size=job[7], + ) + } rows = project_cache(cache, images, geometry_completed) write_csv(args.work_dir / "geometry.csv", rows) geometry_complete = geometry_completed == required @@ -1380,6 +1599,7 @@ def main(argv: list[str] | None = None) -> int: job, images, args.objective, relevant_pairs ), frozen_transport_tool_sha256, + job[7], ) ] print( @@ -1437,6 +1657,7 @@ def main(argv: list[str] | None = None) -> int: job, images, args.objective, relevant_pairs ), frozen_transport_tool_sha256, + job[7], ) for job in jobs ) diff --git a/tools/lora_ota/test_build_rak3401_compact_bundle.py b/tools/lora_ota/test_build_rak3401_compact_bundle.py index 34398671..4aa3757b 100644 --- a/tools/lora_ota/test_build_rak3401_compact_bundle.py +++ b/tools/lora_ota/test_build_rak3401_compact_bundle.py @@ -98,6 +98,109 @@ class CompactRouteInputTests(unittest.TestCase): ], } + def transport_schema2_route(self) -> dict[str, object]: + specifications = [ + ("legacy-160-raw", 1024, None), + ("v2-171-deflate", 1024, 1), + ("v2-171-raw", 2048, 2), + ("v2-171-deflate", 2048, 4), + ] + steps = [] + for number, (profile, block_size, pipeline) in enumerate( + specifications, 1 + ): + payload = 3000 + if profile == "legacy-160-raw": + cost = compact.route_search.legacy_transport_cost(payload) + elif profile == "v2-171-raw": + cost = compact.route_search.v2_raw_transport_cost( + payload, int(pipeline), block_size + ) + else: + cost = compact.route_search.v2_transport_cost( + payload, + 1500, + 1500, + (payload + block_size - 1) // block_size, + 9, + int(pipeline), + block_size, + ) + cost["linear_path_bytes"] = compact.route_search.linear_path_bytes( + int(cost["origin_mesh_bytes"]), int(cost["packets"]), 0 + ) + transport = { + field: cost[field] for field in compact.TRANSPORT_STEP_FIELDS + } + if profile == "v2-171-deflate": + transport.update({ + "payload_sha256": f"{number:064x}", + "encoder_sha256": compact.TRANSPORT_ROUTE_ENCODER_SHA256, + }) + steps.append({ + "source_node": number - 1, + "target_node": number, + "block_size": block_size, + "inplace_memory": "0x98000" if number <= 2 else "0x80000", + "reuse_baseline_package": number == 1, + "expected_container_size": compact.route_search.container_size( + payload, block_size + ), + "expected_staging_margin": 1, + "expected_target_sha256": f"{number:064x}", + "expected_target_version": f"1.0.0.{number}", + "transport": transport, + }) + return { + "schema": 2, + "status": "reachable", + "search_complete": True, + "app_base": "0x26000", + "stage_ceiling": "0xD4000", + "node_count": 5, + "objective": compact.TRANSPORT_ROUTE_OBJECTIVE, + "candidate_geometries": 4, + "feasible_edges": 4, + "shortest_package_count": 4, + "shortest_route_count": 1, + "selected_total_bytes": sum( + int(step["expected_container_size"]) for step in steps + ), + "selected_total_transport_bytes": sum( + int(step["transport"]["linear_path_bytes"]) for step in steps + ), + "transport_accounting": { + "relay_hops": 0, + "legacy_block_size": 1024, + "supported_block_sizes": [1024, 2048], + "comparison_baseline_block_size": 1024, + "source_capability_field": ( + compact.route_search.TRANSPORT_CAPABILITY + ), + "source_pipeline_field": ( + compact.route_search.TRANSPORT_PIPELINE_FIELD + ), + "source_max_block_field": ( + compact.route_search.TRANSPORT_MAX_BLOCK_FIELD + ), + "source_profile_matrix": ( + compact.route_search.TRANSPORT_PROFILE_MATRIX + ), + "first_bootstrap_profile": "legacy-160-raw", + "included": ( + "OTA messages, repeated 4-byte v2 stream IDs, 171-byte DATA " + "slicing, adaptive requests, manifest, exact per-block proofs, " + "and MeshCore framing" + ), + "excluded": ( + "discovery, retries, flood fan-out, and radio-dependent LoRa " + "PHY coding/preamble" + ), + "encoder_sha256": compact.TRANSPORT_ROUTE_ENCODER_SHA256, + }, + "steps": steps, + } + def test_existing_schema1_route_remains_accepted(self) -> None: route = Path(__file__).with_name("rak3401_compact_route.json") steps, document = compact.read_route(route) @@ -151,6 +254,91 @@ class CompactRouteInputTests(unittest.TestCase): with self.assertRaisesRegex(compact.CompactBuildError, message): compact.read_route(path) + def test_transport_route_accepts_full_capability_matrix_and_fails_closed( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + route = self.transport_schema2_route() + path = root / "route.json" + path.write_text(json.dumps(route), encoding="ascii") + steps, _document = compact.read_route(path) + self.assertEqual( + [step["transport"]["profile"] for step in steps], + [ + "legacy-160-raw", + "v2-171-deflate", + "v2-171-raw", + "v2-171-deflate", + ], + ) + + raw_only = self.transport_schema2_route() + for index in (1, 3): + block_size = int(raw_only["steps"][index]["block_size"]) + if block_size == 1024: + cost = compact.route_search.legacy_transport_cost(3000) + else: + cost = compact.route_search.v2_raw_transport_cost( + 3000, + int(raw_only["steps"][index]["transport"][ + "request_pipeline" + ]), + block_size, + ) + cost["linear_path_bytes"] = compact.route_search.linear_path_bytes( + int(cost["origin_mesh_bytes"]), int(cost["packets"]), 0 + ) + raw_only["steps"][index]["transport"] = { + field: cost[field] + for field in compact.TRANSPORT_STEP_FIELDS + } + del raw_only["transport_accounting"]["encoder_sha256"] + raw_only["selected_total_transport_bytes"] = sum( + int(step["transport"]["linear_path_bytes"]) + for step in raw_only["steps"] + ) + raw_path = root / "raw-only.json" + raw_path.write_text(json.dumps(raw_only), encoding="ascii") + raw_steps, raw_document = compact.read_route(raw_path) + self.assertNotIn( + "encoder_sha256", raw_document["transport_accounting"] + ) + self.assertEqual( + [step["transport"]["profile"] for step in raw_steps], + [ + "legacy-160-raw", + "legacy-160-raw", + "v2-171-raw", + "v2-171-raw", + ], + ) + + mutations = [] + bad = self.transport_schema2_route() + del bad["transport_accounting"]["source_profile_matrix"] + mutations.append((bad, "capability contract")) + bad = self.transport_schema2_route() + bad["steps"][2]["transport"].update({ + "payload_sha256": "a" * 64, + "encoder_sha256": compact.TRANSPORT_ROUTE_ENCODER_SHA256, + }) + mutations.append((bad, "hashes do not match")) + bad = self.transport_schema2_route() + bad["steps"][2]["block_size"] = 1024 + mutations.append((bad, "block_size disagrees")) + bad = self.transport_schema2_route() + del bad["transport_accounting"]["encoder_sha256"] + mutations.append((bad, "encoder presence")) + + for number, (document, message) in enumerate(mutations): + candidate = root / f"bad-{number}.json" + candidate.write_text(json.dumps(document), encoding="ascii") + with self.subTest(message=message), self.assertRaisesRegex( + compact.CompactBuildError, message + ): + compact.read_route(candidate) + class CompactImageInventoryTests(unittest.TestCase): def test_schema2_snapshot_freezes_manifest_and_referenced_images(self) -> None: @@ -322,25 +510,20 @@ class CompactImageInventoryTests(unittest.TestCase): class CompactSimulatorTests(unittest.TestCase): def test_exact_required_simulators_are_accepted(self) -> None: with tempfile.TemporaryDirectory() as directory: - preview5 = Path(directory) / "preview5" - current = Path(directory) / "current" - preview5.write_bytes(b"p5") - current.write_bytes(b"current") + simulator = Path(directory) / "otafix2.4" + simulator.write_bytes(b"exact deployed bootloader") hashes = { - preview5: compact.REQUIRED_SIMULATORS["preview5"]["sha256"], - current: compact.REQUIRED_SIMULATORS["current"]["sha256"], + simulator: compact.REQUIRED_SIMULATORS["otafix2.4"]["sha256"], } with mock.patch.object(compact.common, "sha256_file", side_effect=hashes.get): - result = compact.parse_simulators([ - f"preview5={preview5}", f"current={current}" - ]) - self.assertEqual([label for label, _path in result], ["preview5", "current"]) + result = compact.parse_simulators([f"otafix2.4={simulator}"]) + self.assertEqual([label for label, _path in result], ["otafix2.4"]) def test_missing_or_wrong_simulator_is_rejected(self) -> None: with tempfile.TemporaryDirectory() as directory: simulator = Path(directory) / "sim" simulator.write_bytes(b"wrong") - for values, digest in (([], None), ([f"preview5={simulator}"], "0" * 64)): + for values, digest in (([], None), ([f"otafix2.4={simulator}"], "0" * 64)): with self.subTest(values=values), mock.patch.object( compact.common, "sha256_file", return_value=digest ): @@ -446,14 +629,17 @@ class CompactGeometryEvidenceTests(unittest.TestCase): inventory = compact.route_search.load_inventory(inventory_path) rows = [] for job in compact.route_search.all_jobs(inventory): - source, target, memory, _from, _to, source_sha, target_sha = job + ( + source, target, memory, _from, _to, source_sha, target_sha, + block_size, + ) = job if source == 1 and target == 3: payload = 10 else: # A successfully generated patch may be too large for staging; # that is valid infeasible evidence. A tool exception is not. payload = 700_000 - container = compact.route_search.container_size(payload) + container = compact.route_search.container_size(payload, block_size) stage_start = compact.route_search.align_down( compact.STAGE_CEILING - container ) @@ -462,6 +648,7 @@ class CompactGeometryEvidenceTests(unittest.TestCase): rows.append({ "source": source, "target": target, "source_sha256": source_sha, "target_sha256": target_sha, "memory": memory, "payload": payload, + "block_size": block_size, "container": container, "stage_start": stage_start, "margin": margin, "feasible": feasible, "error": error, }) @@ -496,6 +683,7 @@ class CompactGeometryEvidenceTests(unittest.TestCase): "target_sha256": inventory[2]["sha256"], "memory": compact.FIXED_WORKSPACE, "payload": payload, + "block_size": compact.LEGACY_BLOCK_SIZE, "container": container, "stage_start": stage_start, "margin": margin, @@ -523,6 +711,228 @@ class CompactGeometryEvidenceTests(unittest.TestCase): with self.assertRaisesRegex(compact.CompactBuildError, "inconsistent"): compact.validate_geometry_results(root / "geometry.csv", inventory, route) + def test_selected_deflate_step_uses_2k_and_reports_1k_raw_baseline(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + payload_path = root / "payload.patch" + payload_path.write_bytes(b"A" * 3000) + payload_sha = hashlib.sha256(payload_path.read_bytes()).hexdigest() + inventory = [ + { + compact.route_search.TRANSPORT_CAPABILITY: False, + compact.route_search.TRANSPORT_PIPELINE_FIELD: None, + compact.route_search.TRANSPORT_MAX_BLOCK_FIELD: + compact.LEGACY_BLOCK_SIZE, + }, + { + compact.route_search.TRANSPORT_CAPABILITY: True, + compact.route_search.TRANSPORT_PIPELINE_FIELD: 1, + compact.route_search.TRANSPORT_MAX_BLOCK_FIELD: + compact.DEFLATE_BLOCK_SIZE, + }, + ] + selected = compact.route_search.v2_transport_cost( + 3000, 1500, 1500, 2, 10, 1, + compact.DEFLATE_BLOCK_SIZE, + ) + selected["linear_path_bytes"] = compact.route_search.linear_path_bytes( + int(selected["origin_mesh_bytes"]), int(selected["packets"]), 0 + ) + plan_step = { + "source_node": 1, + "block_size": compact.DEFLATE_BLOCK_SIZE, + "transport": { + **{ + field: selected[field] + for field in compact.TRANSPORT_STEP_FIELDS + }, + "payload_sha256": payload_sha, + "encoder_sha256": compact.TRANSPORT_ROUTE_ENCODER_SHA256, + }, + } + measured = { + "payload_sha256": payload_sha, + "transport_encoder_sha256": compact.TRANSPORT_ROUTE_ENCODER_SHA256, + "v2_wire_bytes": 1500, + "v2_deflate_bytes": 1500, + "v2_deflate_blocks": 2, + "v2_data_packets": 10, + } + with mock.patch.object( + compact.route_search, + "measure_transport_size", + return_value=measured, + ) as measure: + result = compact.validate_selected_transport( + Path("motatool"), payload_path, plan_step, inventory, 0, 3 + ) + with mock.patch.object( + compact.route_search, + "measure_transport_size", + return_value={ + **measured, + "transport_encoder_sha256": "0" * 64, + }, + ), self.assertRaisesRegex( + compact.CompactBuildError, "different encoder binary" + ): + compact.validate_selected_transport( + Path("motatool"), payload_path, plan_step, inventory, 0, 3 + ) + + measure.assert_called_once_with( + Path("motatool"), payload_path, compact.DEFLATE_BLOCK_SIZE + ) + baseline = result["one_kib_no_compression"] + self.assertEqual(result["block_size"], compact.DEFLATE_BLOCK_SIZE) + self.assertEqual(baseline["profile"], "v2-171-raw") + self.assertEqual(baseline["block_size"], compact.LEGACY_BLOCK_SIZE) + self.assertEqual(baseline["proof_packets"], 3) + self.assertGreater(baseline["packets"], result["packets"]) + + def test_selected_transport_validates_independent_capability_matrix( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + payload_path = root / "payload.patch" + payload_path.write_bytes(b"A" * 3000) + payload_sha = hashlib.sha256(payload_path.read_bytes()).hexdigest() + inventory = [ + { + compact.route_search.TRANSPORT_CAPABILITY: False, + compact.route_search.TRANSPORT_MAX_BLOCK_FIELD: 1024, + compact.route_search.TRANSPORT_PIPELINE_FIELD: None, + }, + { + compact.route_search.TRANSPORT_CAPABILITY: False, + compact.route_search.TRANSPORT_MAX_BLOCK_FIELD: 1024, + compact.route_search.TRANSPORT_PIPELINE_FIELD: None, + }, + { + compact.route_search.TRANSPORT_CAPABILITY: True, + compact.route_search.TRANSPORT_MAX_BLOCK_FIELD: 1024, + compact.route_search.TRANSPORT_PIPELINE_FIELD: 1, + }, + { + compact.route_search.TRANSPORT_CAPABILITY: False, + compact.route_search.TRANSPORT_MAX_BLOCK_FIELD: 2048, + compact.route_search.TRANSPORT_PIPELINE_FIELD: 2, + }, + { + compact.route_search.TRANSPORT_CAPABILITY: True, + compact.route_search.TRANSPORT_MAX_BLOCK_FIELD: 2048, + compact.route_search.TRANSPORT_PIPELINE_FIELD: 4, + }, + ] + + def plan(source: int) -> dict[str, object]: + block_size = compact.route_search.source_block_size( + inventory, source + ) + profile = compact.route_search.source_transport_profile( + inventory, source + ) + if profile == "v2-171-deflate": + cost = compact.route_search.v2_transport_cost( + 3000, + 1500, + 1500, + (3000 + block_size - 1) // block_size, + 9, + compact.route_search.source_transport_pipeline( + inventory, source + ), + block_size, + ) + elif profile == "v2-171-raw": + cost = compact.route_search.v2_raw_transport_cost( + 3000, + compact.route_search.source_transport_pipeline( + inventory, source + ), + block_size, + ) + else: + cost = compact.route_search.legacy_transport_cost(3000) + cost["linear_path_bytes"] = compact.route_search.linear_path_bytes( + int(cost["origin_mesh_bytes"]), int(cost["packets"]), 0 + ) + transport = { + field: cost[field] for field in compact.TRANSPORT_STEP_FIELDS + } + if profile == "v2-171-deflate": + transport.update({ + "payload_sha256": payload_sha, + "encoder_sha256": compact.TRANSPORT_ROUTE_ENCODER_SHA256, + }) + return { + "source_node": source, + "block_size": block_size, + "transport": transport, + } + + def measured( + _tool: Path, _payload: Path, block_size: int + ) -> dict[str, object]: + return { + "payload_sha256": payload_sha, + "transport_encoder_sha256": ( + compact.TRANSPORT_ROUTE_ENCODER_SHA256 + ), + "v2_wire_bytes": 1500, + "v2_deflate_bytes": 1500, + "v2_deflate_blocks": (3000 + block_size - 1) // block_size, + "v2_data_packets": 9, + } + + with mock.patch.object( + compact.route_search, + "measure_transport_size", + side_effect=measured, + ) as measure: + results = [ + compact.validate_selected_transport( + Path("motatool"), payload_path, plan(source), inventory, 0, + source, + ) + for source in range(1, 5) + ] + + self.assertEqual( + [result["profile"] for result in results], + [ + "legacy-160-raw", + "v2-171-deflate", + "v2-171-raw", + "v2-171-deflate", + ], + ) + self.assertEqual( + [call.args[2] for call in measure.call_args_list], [1024, 2048] + ) + + raw_with_hashes = plan(3) + raw_with_hashes["transport"].update({ + "payload_sha256": payload_sha, + "encoder_sha256": compact.TRANSPORT_ROUTE_ENCODER_SHA256, + }) + with self.assertRaisesRegex( + compact.CompactBuildError, "raw transport unexpectedly has encoder pins" + ): + compact.validate_selected_transport( + Path("motatool"), payload_path, raw_with_hashes, inventory, 0, 3 + ) + + wrong_profile = plan(3) + wrong_profile["transport"]["profile"] = "legacy-160-raw" + with self.assertRaisesRegex( + compact.CompactBuildError, "profile disagrees" + ): + compact.validate_selected_transport( + Path("motatool"), payload_path, wrong_profile, inventory, 0, 3 + ) + def test_missing_and_extra_geometry_are_rejected(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -574,6 +984,132 @@ class CompactGeometryEvidenceTests(unittest.TestCase): compact.validate_geometry_results(root / "geometry.csv", inventory, route) +class CompactGeneratedReportTests(unittest.TestCase): + def test_chain_csv_pins_block_and_transport_component_details(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "CHAIN.csv" + compact.write_chain(path, [{ + "step": 1, + "mota_block_size": compact.DEFLATE_BLOCK_SIZE, + "transport_relay_hops": 0, + "transport_request_pipeline": 1, + "transport_payload_sha256": "a" * 64, + "transport_encoder_sha256": "b" * 64, + "transport_request_packets": 7, + "transport_proof_packets": 9, + "transport_data_bytes": 1234, + "baseline_1k_raw_packets": 99, + }]) + lines = path.read_text(encoding="ascii").splitlines() + + header = lines[0].split(",") + values = dict(zip(header, lines[1].split(","))) + self.assertEqual(values["mota_block_size"], "2048") + self.assertEqual(values["transport_request_pipeline"], "1") + self.assertEqual(values["transport_payload_sha256"], "a" * 64) + self.assertEqual(values["transport_encoder_sha256"], "b" * 64) + self.assertEqual(values["transport_proof_packets"], "9") + self.assertEqual(values["transport_data_bytes"], "1234") + self.assertEqual(values["baseline_1k_raw_packets"], "99") + + def test_report_contains_derived_2k_vs_1k_raw_comparison(self) -> None: + legacy_payloads = [89_282, 64_756] + v2_measurements = [ + (104_543, 60_379, 60_284, 51, 381), + (148_997, 92_549, 92_549, 73, 577), + (112_123, 59_704, 59_704, 55, 375), + (121_934, 63_977, 63_977, 60, 405), + (135_068, 87_425, 87_425, 66, 547), + (176_140, 130_264, 130_252, 86, 807), + (271_013, 210_192, 210_192, 133, 1297), + ] + selected = [ + compact.route_search.legacy_transport_cost(payload) + for payload in legacy_payloads + ] + [ + compact.route_search.v2_transport_cost( + *measurement, 1, compact.DEFLATE_BLOCK_SIZE + ) + for measurement in v2_measurements + ] + [compact.route_search.legacy_transport_cost(196_843)] + baseline = [ + compact.route_search.legacy_transport_cost(payload) + for payload in legacy_payloads + ] + [ + compact.route_search.v2_raw_transport_cost( + measurement[0], 1, compact.LEGACY_BLOCK_SIZE + ) + for measurement in v2_measurements + ] + [compact.route_search.legacy_transport_cost(196_843)] + rows = [] + for number, (actual, raw) in enumerate(zip(selected, baseline), 1): + rows.append({ + "step": number, + "mota_size": compact.route_search.container_size( + int(actual["payload_bytes"]), int(actual["block_size"]) + ), + "staging_margin": 4096, + "transport_profile": actual["profile"], + "transport_relay_hops": 0, + "transport_payload_bytes": actual["payload_bytes"], + "transport_wire_bytes": actual["wire_bytes"], + "transport_deflate_bytes": actual["deflate_bytes"], + "transport_deflate_blocks": actual["deflate_blocks"], + "transport_data_packets": actual["data_packets"], + "transport_packets": actual["packets"], + "transport_linear_path_bytes": actual["origin_mesh_bytes"], + "baseline_1k_raw_packets": raw["packets"], + "baseline_1k_raw_linear_path_bytes": raw["origin_mesh_bytes"], + }) + tools = { + "motatool": { + "version": compact.MOTATOOL_VERSION, + "asserted_source_commit": compact.MOTATOOL_COMMIT, + "executable_sha256": compact.TRANSPORT_ROUTE_ENCODER_SHA256, + }, + "detools": { + "version": compact.DETOOLS_VERSION, + "launcher_sha256": "d" * 64, + }, + } + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + compact.write_docs( + root, + "a" * 40, + "1.17.1.6", + firmware(b"endpoint", "1.17.1.6"), + "1" * 64, + "2" * 64, + "3" * 64, + "4" * 64, + "5" * 64, + "6" * 64, + tools, + rows, + False, + ) + readme = (root / "README.md").read_text(encoding="ascii") + provenance = (root / "PROVENANCE.md").read_text(encoding="ascii") + + self.assertIn( + "| No DEFLATE baseline; packet profiles preserved | 1 KiB all " + "steps | 11,835 | 1,618,318 |", + readme, + ) + self.assertIn( + "| Selected capability-aware route | Per-source 1/2 KiB signed " + "geometry | 8,906 | 1,195,442 |", + readme, + ) + self.assertIn("2,929 packets\n (24.75%)", readme) + self.assertIn("422,876 bytes\n (26.13%)", readme) + self.assertIn("preserves each step's packet profile", readme) + self.assertIn("independently selected per running source", provenance) + self.assertIn("`transport_max_block_bytes` capability", provenance) + self.assertIn("DEFLATE permission is a separate capability", provenance) + + class CompactPhysicalValidationTests(unittest.TestCase): def evidence(self) -> tuple[dict[str, object], dict[str, object]]: expected = { diff --git a/tools/lora_ota/test_build_rak3401_inflate_bridges.py b/tools/lora_ota/test_build_rak3401_inflate_bridges.py index 39e3ad30..31e0700e 100644 --- a/tools/lora_ota/test_build_rak3401_inflate_bridges.py +++ b/tools/lora_ota/test_build_rak3401_inflate_bridges.py @@ -4,6 +4,9 @@ from __future__ import annotations from pathlib import Path +import os +import shutil +import subprocess import sys import tempfile import unittest @@ -12,6 +15,277 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import build_rak3401_inflate_bridges as inflate +REPO_ROOT = Path(__file__).resolve().parents[2] + + +MOCK_MANAGER_HEADER = r'''#pragma once + +#include +#include +#include + +#define OTA_MAX_BLOCK 2048 +#define OTA_FRAG_DATA 160 +#define OTA_FRAG_DATA_V2 171 +#define OTA_FETCH_PIPELINE 1 + +#include "OtaProtocol.h" + +namespace mesh { +namespace ota { + +typedef void (*OtaSend)(void*, const uint8_t*, uint16_t, bool); + +class OtaManager { +public: + void begin(uint32_t target_id, OtaSend send, void* send_ctx) { + (void)target_id; + _send = send; + _send_ctx = send_ctx; + } + + void configure(const uint8_t mid[4], uint16_t block, uint16_t block_len, + uint16_t nominal_size) { + memcpy(_mid, mid, sizeof(_mid)); + _block = block; + _block_len = block_len; + _nominal_size = nominal_size; + logical.assign(block_len, 0); + offsets.clear(); + invalid = false; + } + + bool request(uint16_t want_mask) { + if (!_send) return false; + ReqMsg request; + memcpy(request.manifest_id, _mid, sizeof(request.manifest_id)); + request.block_idx = _block; + request.want_mask = want_mask; + uint8_t wire[9]; + const uint16_t wire_len = encode_req(wire, sizeof(wire), request); + if (wire_len == 0) return false; + _send(_send_ctx, wire, wire_len, false); + return true; + } + + uint16_t requestedBlockLength(const uint8_t* manifest_id, uint16_t block, + uint16_t* nominal_size = nullptr) const { + if (nominal_size) *nominal_size = 0; + if (!manifest_id || block != _block || + memcmp(manifest_id, _mid, sizeof(_mid)) != 0) return 0; + if (nominal_size) *nominal_size = _nominal_size; + return _block_len; + } + + void on_message(const uint8_t* msg, uint16_t len) { + DataMsg data; + if (!decode_data(msg, len, data) || + memcmp(data.manifest_id, _mid, sizeof(_mid)) != 0 || + data.block_idx != _block || (data.frag_off & OTA_DATA_V2_MARK) != 0 || + data.frag_off >= _block_len) { + invalid = true; + return; + } + uint16_t expected = (uint16_t)(_block_len - data.frag_off); + if (expected > OTA_FRAG_DATA) expected = OTA_FRAG_DATA; + if (data.frag_off % OTA_FRAG_DATA != 0 || data.data_len != expected) { + invalid = true; + return; + } + memcpy(logical.data() + data.frag_off, data.data, data.data_len); + offsets.push_back(data.frag_off); + } + + std::vector logical; + std::vector offsets; + bool invalid = false; + +private: + OtaSend _send = nullptr; + void* _send_ctx = nullptr; + uint8_t _mid[4] = {0}; + uint16_t _block = 0; + uint16_t _block_len = 0; + uint16_t _nominal_size = 0; +}; + +} // namespace ota +} // namespace mesh +''' + + +MOCK_MULTIHASH_HEADER = r'''#pragma once + +#include +#include + +namespace mesh { +namespace ota { + +inline void mh4(uint8_t out[4], const uint8_t* data, size_t len) { + uint32_t hash = 2166136261u; + for (size_t i = 0; i < len; ++i) { + hash ^= data[i]; + hash *= 16777619u; + } + out[0] = (uint8_t)hash; + out[1] = (uint8_t)(hash >> 8); + out[2] = (uint8_t)(hash >> 16); + out[3] = (uint8_t)(hash >> 24); +} + +} // namespace ota +} // namespace mesh +''' + + +BRIDGE_BEHAVIOR_HARNESS = r'''#include +#include +#include +#include + +#include "Multihash.h" +#include "OtaDeflate.h" + +using namespace mesh::ota; + +namespace { + +struct Capture { + std::vector > packets; +}; + +void capture_send(void* context, const uint8_t* msg, uint16_t len, bool flood) { + (void)flood; + Capture* capture = static_cast(context); + capture->packets.push_back(std::vector(msg, msg + len)); +} + +std::vector make_payload(uint16_t total, uint16_t random_prefix) { + std::vector payload(total, 'A'); + uint32_t state = 0xc001d00du ^ total; + for (uint16_t i = 0; i < random_prefix; ++i) { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + payload[i] = (uint8_t)state; + } + return payload; +} + +std::vector make_deflate(const std::vector& payload, + uint16_t random_prefix) { + // Non-final stored block for the pseudo-random prefix, followed by a tiny + // final fixed-Huffman block for the repeated 'A' suffix. This deliberately + // keeps the 2 KiB representation long enough to exercise fragment index 9. + static const uint8_t tail_304[] = {0x73, 0x74, 0x1c, 0x05, 0xa4, 0x00, 0x00}; + static const uint8_t tail_512[] = {0x73, 0x74, 0x1c, 0x05, 0x23, 0x19, 0x00, 0x00}; + const uint8_t* tail = payload.size() == 1024 ? tail_304 : tail_512; + const size_t tail_len = payload.size() == 1024 ? sizeof(tail_304) : sizeof(tail_512); + + std::vector encoded; + encoded.reserve((size_t)random_prefix + 5u + tail_len); + encoded.push_back(0x00); // BFINAL=0, stored block, then align to the next byte. + encoded.push_back((uint8_t)random_prefix); + encoded.push_back((uint8_t)(random_prefix >> 8)); + const uint16_t complement = (uint16_t)~random_prefix; + encoded.push_back((uint8_t)complement); + encoded.push_back((uint8_t)(complement >> 8)); + encoded.insert(encoded.end(), payload.begin(), payload.begin() + random_prefix); + encoded.insert(encoded.end(), tail, tail + tail_len); + return encoded; +} + +int run_case(uint16_t block_len, uint16_t random_prefix, bool extended) { + const uint8_t mid[4] = {0x21, 0x43, 0x65, (uint8_t)(extended ? 0x87 : 0x76)}; + const uint16_t block = extended ? 9 : 4; + const uint16_t canonical_fragments = + (uint16_t)((block_len + OTA_FRAG_DATA - 1u) / OTA_FRAG_DATA); + const uint16_t original_want = + (uint16_t)((1u << canonical_fragments) - 1u); + + OtaManager manager; + manager.configure(mid, block, block_len, block_len); + Capture capture; + OtaTransportInflateReceiver receiver; + receiver.begin(manager, 0x2fa509c1u, capture_send, &capture); + if (!manager.request(original_want) || capture.packets.size() != 1) return 10; + + ReqMsg request; + if (!decode_req(capture.packets[0].data(), + (uint16_t)capture.packets[0].size(), request)) return 11; + const uint16_t expected_want = ota_req_make_v2(original_want, true, extended); + if (request.want_mask != expected_want || !ota_req_is_v2(request.want_mask) || + ota_req_v2_extended_length(request.want_mask) != extended) return 12; + if ((!extended && request.want_mask != 0xc07fu) || + (extended && request.want_mask != 0xefffu)) return 13; + + const std::vector payload = make_payload(block_len, random_prefix); + const std::vector encoded = make_deflate(payload, random_prefix); + if (encoded.size() >= payload.size()) return 14; + const uint16_t wire_fragments = + (uint16_t)((encoded.size() + OTA_FRAG_DATA_V2 - 1u) / OTA_FRAG_DATA_V2); + if ((!extended && wire_fragments != 5) || (extended && wire_fragments != 10)) return 15; + + uint8_t stream_id[4]; + mh4(stream_id, encoded.data(), encoded.size()); + // Reverse order proves that completion is driven by the descriptor bitmap, + // rather than accidentally relying on sequential arrival. + for (int fragment = (int)wire_fragments - 1; fragment >= 0; --fragment) { + const uint16_t offset = (uint16_t)(fragment * OTA_FRAG_DATA_V2); + uint16_t slice = (uint16_t)(encoded.size() - offset); + if (slice > OTA_FRAG_DATA_V2) slice = OTA_FRAG_DATA_V2; + + uint16_t descriptor = 0; + const bool packed = extended + ? ota_data_v2_pack_extended((uint8_t)fragment, + (uint16_t)encoded.size(), descriptor) + : ota_data_v2_pack((uint8_t)fragment, (uint16_t)encoded.size(), + true, descriptor); + if (!packed) return 16; + + std::vector body(OTA_DATA_V2_STREAM_ID_BYTES + slice); + memcpy(body.data(), stream_id, sizeof(stream_id)); + memcpy(body.data() + OTA_DATA_V2_STREAM_ID_BYTES, + encoded.data() + offset, slice); + DataMsg data; + memcpy(data.manifest_id, mid, sizeof(data.manifest_id)); + data.block_idx = block; + data.frag_off = descriptor; + data.data = body.data(); + data.data_len = (uint16_t)body.size(); + std::vector wire(9u + body.size()); + const uint16_t wire_len = encode_data(wire.data(), (uint16_t)wire.size(), data); + if (wire_len != wire.size()) return 17; + receiver.on_message(wire.data(), wire_len); + } + + if (manager.invalid || manager.logical != payload || + manager.offsets.size() != canonical_fragments) return 18; + for (uint16_t fragment = 0; fragment < canonical_fragments; ++fragment) { + if (manager.offsets[fragment] != fragment * OTA_FRAG_DATA) return 19; + } + return 0; +} + +} // namespace + +int main() { + int result = run_case(1024, 720, false); + if (result != 0) { + fprintf(stderr, "1 KiB bridge case failed: %d\n", result); + return result; + } + result = run_case(2048, 1536, true); + if (result != 0) { + fprintf(stderr, "2 KiB bridge case failed: %d\n", result); + return 100 + result; + } + return 0; +} +''' + + class InflateAssetTests(unittest.TestCase): def test_vendored_snapshot_assets_match_their_pins(self) -> None: metadata = inflate.inflate_asset_metadata() @@ -34,6 +308,88 @@ class InflateAssetTests(unittest.TestCase): for destination in inflate.INFLATE_ASSETS: self.assertFalse((source / destination).exists()) + def test_receiver_negotiates_both_v2_descriptor_layouts(self) -> None: + source = (inflate.INFLATE_ASSET_ROOT / "OtaDeflate.cpp").read_text( + encoding="utf-8" + ) + + self.assertIn("OTA_DATA_V2_LEGACY_MAX_ENCODED", source) + self.assertIn("ota_data_v2_unpack_extended", source) + self.assertIn("&nominal_block_size", source) + self.assertIn( + "nominal_block_size > OTA_DATA_V2_LEGACY_MAX_ENCODED", source + ) + + def test_installed_receiver_reassembles_deflated_1k_and_2k_blocks(self) -> None: + cxx = shutil.which("c++") or shutil.which("g++") + cc = shutil.which("cc") or shutil.which("gcc") + if not cxx or not cc: + self.skipTest("a host C and C++ compiler are required") + + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) + inflate.install_inflate_assets(source) + ota_dir = source / "src" / "helpers" / "ota" + for filename in ("OtaFormat.h", "OtaProtocol.h", "OtaProtocol.cpp"): + shutil.copy2(REPO_ROOT / "src" / "helpers" / "ota" / filename, + ota_dir / filename) + (ota_dir / "OtaManager.h").write_text( + MOCK_MANAGER_HEADER, encoding="ascii" + ) + (ota_dir / "Multihash.h").write_text( + MOCK_MULTIHASH_HEADER, encoding="ascii" + ) + harness = source / "bridge_behavior.cpp" + harness.write_text(BRIDGE_BEHAVIOR_HARNESS, encoding="ascii") + + tinf_object = source / "ota_tinf.o" + compile_c = subprocess.run( + [ + cc, + "-std=c99", + "-DOTA_TRANSPORT_DEFLATE_TEST", + f"-I{ota_dir}", + "-c", + str(ota_dir / "OtaTinf.c"), + "-o", + str(tinf_object), + ], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(compile_c.returncode, 0, compile_c.stderr) + + executable = source / ( + "bridge_behavior.exe" if os.name == "nt" else "bridge_behavior" + ) + compile_cpp = subprocess.run( + [ + cxx, + "-std=c++11", + "-Wall", + "-Wextra", + "-Werror", + "-DOTA_TRANSPORT_DEFLATE_TEST", + f"-I{ota_dir}", + str(ota_dir / "OtaDeflate.cpp"), + str(ota_dir / "OtaProtocol.cpp"), + str(harness), + str(tinf_object), + "-o", + str(executable), + ], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(compile_cpp.returncode, 0, compile_cpp.stderr) + + run_result = subprocess.run( + [str(executable)], capture_output=True, text=True, check=False + ) + self.assertEqual(run_result.returncode, 0, run_result.stderr) + class ExactTransformTests(unittest.TestCase): def test_replace_exact_replaces_one_anchor(self) -> None: @@ -57,10 +413,10 @@ class ExactTransformTests(unittest.TestCase): class BuilderConfigurationTests(unittest.TestCase): def test_inflate_builder_has_distinct_reproducibility_identity(self) -> None: self.assertEqual( - inflate.VERSION_SUFFIX, "halo-keymind-cascade-mota-inflate" + inflate.VERSION_SUFFIX, "halo-keymind-cascade-mota-inflate2k" ) self.assertEqual( - inflate.INFLATE_SNAPSHOT_COMMIT, + inflate.INFLATE_BASE_SNAPSHOT_COMMIT, "add51bf00c46c15ef54318ca766a6daf08a147ee", ) self.assertNotIn("6f03eae8", {target.source_commit[:8] for target in inflate.TARGETS}) diff --git a/tools/lora_ota/test_lora_ota.py b/tools/lora_ota/test_lora_ota.py index 2236c72b..a77dd7df 100644 --- a/tools/lora_ota/test_lora_ota.py +++ b/tools/lora_ota/test_lora_ota.py @@ -90,6 +90,7 @@ def target( boot_codecs: int | None = None, boot_version: str | None = None, current_version: str | None = None, + max_block_size: int = ota.MOTA_MAX_BLOCK_SIZE, ) -> ota.TargetInfo: return ota.TargetInfo( name="remote", @@ -105,6 +106,7 @@ def target( self_status="self", current_version=current_version, nrf_qspi=nrf_qspi, + max_block_size=max_block_size, ) @@ -150,10 +152,15 @@ class FormatTests(unittest.TestCase): with self.assertRaisesRegex(ota.OtaError, "invalid flags in v2"): ota.parse_mota(bytes(blob)) + def test_2k_application_block_is_accepted(self) -> None: + image = firmware(b"C" * 5000, VERSION_NEW) + parsed = ota.parse_mota(mota_blob(image, block_size_log2=11)) + self.assertEqual(parsed.block_size, 2048) + def test_block_larger_than_firmware_buffer_is_rejected(self) -> None: image = firmware(b"C" * 5000, VERSION_NEW) - with self.assertRaisesRegex(ota.OtaError, "at most 1024 bytes"): - ota.parse_mota(mota_blob(image, block_size_log2=11)) + with self.assertRaisesRegex(ota.OtaError, "at most 2048 bytes"): + ota.parse_mota(mota_blob(image, block_size_log2=12)) def test_manifest_endf_identity_mismatch_is_rejected(self) -> None: wrong_image = firmware(b"B" * 5000, VERSION_NEW, target=TARGET + 1) @@ -2097,6 +2104,38 @@ class CompatibilityTests(unittest.TestCase): nrf = target(platform="nrf52", nrf_qspi=True, boot_codecs=1) self.assertTrue(ota.compatible_mota(full, nrf)[0]) + def test_2k_mota_requires_explicit_live_target_capability(self) -> None: + package = ota.parse_mota( + mota_blob(self.new_image, block_size_log2=11) + ) + legacy = target(max_block_size=ota.LEGACY_TARGET_MAX_BLOCK_SIZE) + good, reason = ota.compatible_mota(package, legacy) + self.assertFalse(good) + self.assertIn("block size 2048", reason) + self.assertIn("maxblk:1024", reason) + self.assertTrue(ota.compatible_mota(package, target())[0]) + + def test_direct_2k_mota_is_rejected_before_any_tool_is_run(self) -> None: + package = mota_blob(self.new_image, block_size_log2=11) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "update.mota" + source.write_bytes(package) + work = root / "work" + work.mkdir() + with ( + mock.patch.object(ota, "run_checked") as run, + self.assertRaisesRegex( + ota.OtaError, r"block size 2048.*maxblk:1024" + ), + ): + ota.prepare_package( + prepare_args(source, "motatool"), + target(max_block_size=ota.LEGACY_TARGET_MAX_BLOCK_SIZE), + work, + ) + run.assert_not_called() + def test_zip_prefers_equal_version_delta(self) -> None: full = mota_blob(self.new_image) delta = mota_blob( @@ -2151,12 +2190,55 @@ class CompatibilityTests(unittest.TestCase): if call.kwargs["label"] == "build mOTA" ) self.assertEqual(build_call.kwargs["timeout"], 4321) + build_command = build_call.args[0] + self.assertEqual( + build_command[build_command.index("--block-size") + 1], "2048" + ) verify_call = next( call for call in run.call_args_list if call.kwargs["label"].startswith("verify ") ) self.assertEqual(verify_call.kwargs["timeout"], 120) + def test_raw_package_build_uses_legacy_live_block_limit(self) -> None: + image = firmware(b"legacy live block limit" * 300, VERSION_NEW) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archive_path = root / "release.zip" + with zipfile.ZipFile(archive_path, "w") as archive: + archive.writestr("firmware.bin", image) + work = root / "work" + work.mkdir() + + def run_tool( + command: list[str], *, label: str, **_kwargs: object, + ) -> subprocess.CompletedProcess[str]: + if label == "build mOTA": + output = Path(command[command.index("--out") + 1]) + output.write_bytes(mota_blob(image, block_size_log2=10)) + return subprocess.CompletedProcess( + args=command, returncode=0, stdout="OK", stderr="" + ) + + with mock.patch.object( + ota, "run_checked", side_effect=run_tool + ) as run: + _path, package, _expected = ota.prepare_package( + prepare_args(archive_path, "motatool"), + target(max_block_size=ota.LEGACY_TARGET_MAX_BLOCK_SIZE), + work, + ) + + self.assertEqual(package.block_size, 1024) + build_call = next( + call for call in run.call_args_list + if call.kwargs["label"] == "build mOTA" + ) + build_command = build_call.args[0] + self.assertEqual( + build_command[build_command.index("--block-size") + 1], "1024" + ) + def test_base_zip_selects_running_hash_not_newest_file(self) -> None: other = firmware(b"other" * 1300, VERSION_NEW) with tempfile.TemporaryDirectory() as directory: @@ -4587,6 +4669,9 @@ class ReliabilityTests(unittest.TestCase): ) self.assertEqual(result.platform, "esp32") self.assertIsNone(result.bootloader_version) + self.assertEqual( + result.max_block_size, ota.LEGACY_TARGET_MAX_BLOCK_SIZE + ) self.assertEqual(result.current_version, "v1.16.9") self.assertEqual(result.current_version_source, "ver") @@ -4625,9 +4710,11 @@ class ReliabilityTests(unittest.TestCase): def __init__(self) -> None: self.commands: list[str] = [] self.replies = iter([ - "OTA | no download | target:1234ABCD hw=RAK_3401", + "OTA | no download | target:1234ABCD | maxblk:2048 " + "| hw=RAK_3401", "> 0.9.2-OTAFIX2.4", - "self body=1 image=2 base_hash=0011223344556677 | " + "self body=1 image=2 base_hash=0011223344556677 " + "| maxblk:2048 | " "bootloader: apply OK (abi=2 codecs=0x4)", "OTA | fw v1.17.0 id=00112233", ]) @@ -4646,11 +4733,23 @@ class ReliabilityTests(unittest.TestCase): self.assertEqual(result.bootloader_version, "0.9.2-OTAFIX2.4") self.assertEqual(result.bootloader_abi, 2) self.assertEqual(result.bootloader_codecs, 0x4) + self.assertEqual(result.max_block_size, 2048) self.assertEqual( controller.commands, ["ota status", "get bootloader.ver", "ota self", "ota stats"], ) + def test_target_rejects_conflicting_or_unsupported_maxblk_markers(self) -> None: + with self.assertRaisesRegex(ota.OtaError, "conflicting maxblk"): + ota.parse_target_max_block_size( + "OTA | target:1234ABCD | maxblk:2048", + "self base_hash=0011223344556677 | maxblk:1024", + ) + with self.assertRaisesRegex(ota.OtaError, "unsupported maxblk:4096"): + ota.parse_target_max_block_size( + "OTA | target:1234ABCD | maxblk:4096", "self" + ) + def test_optional_ota_stats_loss_is_bounded_then_falls_back_to_ver( self, ) -> None: @@ -6779,6 +6878,58 @@ class Rak3401ExtractionCacheTests(unittest.TestCase): class Rak3401KnownUnsafeReleaseTests(unittest.TestCase): + @staticmethod + def live_chain_args() -> argparse.Namespace: + return argparse.Namespace( + meshcli="meshcli", + controller_serial="/dev/controller", + controller_tcp=None, + controller_ble=None, + controller_baud=115200, + reply_timeout=45, + ) + + @staticmethod + def live_chain_target(bootloader_version: str | None) -> ota.TargetInfo: + return ota.TargetInfo( + name="remote", + target_id=rak_chain.EXPECTED_TARGET_ID, + base_hash=b"12345678", + platform="nrf52", + nrf_sd=False, + hw_id=rak_chain.EXPECTED_HARDWARE, + bootloader_version=bootloader_version, + bootloader_abi=2, + bootloader_codecs=1 << ota.MOTA_CODEC_IN_PLACE, + status="status", + self_status="self", + nrf_qspi=False, + ) + + def test_live_chain_accepts_exact_deployed_bootloader_version(self) -> None: + expected = self.live_chain_target(rak_chain.EXPECTED_BOOTLOADER_VERSION) + with mock.patch.object( + rak_chain.ota, "query_target", return_value=expected + ): + actual = rak_chain.query_live_target( + mock.Mock(), self.live_chain_args(), "remote" + ) + self.assertIs(actual, expected) + + def test_live_chain_rejects_any_other_bootloader_version(self) -> None: + for version in (None, "0.9.2-OTAFIX2.3", "0.9.2-OTAFIX2.5"): + with self.subTest(version=version), mock.patch.object( + rak_chain.ota, + "query_target", + return_value=self.live_chain_target(version), + ), self.assertRaisesRegex( + ota.OtaError, + "expected exact deployed version '0.9.2-OTAFIX2.4'", + ): + rak_chain.query_live_target( + mock.Mock(), self.live_chain_args(), "remote" + ) + def test_chain_resume_uses_exact_body_hash_not_runtime_label(self) -> None: steps = [ mock.Mock(base_hash=bytes.fromhex("0011223344556677")), @@ -7026,7 +7177,22 @@ class Rak3401KnownUnsafeReleaseTests(unittest.TestCase): steps[number - 1].target_sha256 = image_sha256 with self.assertRaisesRegex( rak_chain.KnownUnsafeReleaseError, - "new b40d2e6c step 10.*bootloader simulators only", + "new 3f6eddd5 step 10.*exact OTAFIX2.4 bootloader simulator only", + ): + rak_chain.require_live_release_safe( + argparse.Namespace(accept_test_candidate=False), steps + ) + rak_chain.require_live_release_safe( + argparse.Namespace(accept_test_candidate=True), steps + ) + + def test_exact_superseded_b40_ten_step_candidate_remains_gated(self) -> None: + steps = [mock.Mock(target_sha256="") for _ in range(10)] + for number, image_sha256 in rak_chain.SUPERSEDED_B40_ANCHORS: + steps[number - 1].target_sha256 = image_sha256 + with self.assertRaisesRegex( + rak_chain.KnownUnsafeReleaseError, + "exact superseded b40d2e6c ten-step prerelease.*step 10", ): rak_chain.require_live_release_safe( argparse.Namespace(accept_test_candidate=False), steps @@ -7050,31 +7216,46 @@ class Rak3401KnownUnsafeReleaseTests(unittest.TestCase): argparse.Namespace(accept_test_candidate=True), steps ) - def test_current_and_legacy_ten_step_reports_do_not_mix_evidence(self) -> None: + def test_ten_step_reports_do_not_mix_current_and_legacy_evidence(self) -> None: current = [mock.Mock(target_sha256="") for _ in range(10)] + superseded = [mock.Mock(target_sha256="") for _ in range(10)] legacy = [mock.Mock(target_sha256="") for _ in range(10)] for number, image_sha256 in rak_chain.CURRENT_10_CANDIDATE_ANCHORS: current[number - 1].target_sha256 = image_sha256 + for number, image_sha256 in rak_chain.SUPERSEDED_B40_ANCHORS: + superseded[number - 1].target_sha256 = image_sha256 for number, image_sha256 in rak_chain.PHYSICALLY_PASSED_FD98_ANCHORS: legacy[number - 1].target_sha256 = image_sha256 current_message = rak_chain.ten_step_verification_message(current) + superseded_message = rak_chain.ten_step_verification_message(superseded) legacy_message = rak_chain.ten_step_verification_message(legacy) self.assertIn("new step 10", current_message) + self.assertIn("exact OTAFIX2.4 bootloader simulation", current_message) self.assertIn("not had a clean physical run", current_message) self.assertNotIn("endpoint passed independent SWD readback", current_message) + self.assertIn("Superseded b40d2e6c", superseded_message) + self.assertNotIn("exact OTAFIX2.4 bootloader simulation", superseded_message) self.assertIn("exact ten package transitions", legacy_message) self.assertIn("endpoint passed independent SWD readback", legacy_message) - def test_ten_step_candidates_share_only_the_pinned_physical_prefix(self) -> None: + def test_ten_step_candidates_share_the_pinned_physical_prefix(self) -> None: self.assertEqual( rak_chain.CURRENT_10_CANDIDATE_ANCHORS[:9], rak_chain.PHYSICALLY_PASSED_FD98_ANCHORS[:9], ) + self.assertEqual( + rak_chain.SUPERSEDED_B40_ANCHORS[:9], + rak_chain.PHYSICALLY_PASSED_FD98_ANCHORS[:9], + ) self.assertNotEqual( rak_chain.CURRENT_10_CANDIDATE_ANCHORS[-1], rak_chain.PHYSICALLY_PASSED_FD98_ANCHORS[-1], ) + self.assertNotEqual( + rak_chain.CURRENT_10_CANDIDATE_ANCHORS[-1], + rak_chain.SUPERSEDED_B40_ANCHORS[-1], + ) steps = [mock.Mock(target_sha256="") for _ in range(10)] for number, image_sha256 in rak_chain.CURRENT_10_CANDIDATE_ANCHORS: steps[number - 1].target_sha256 = image_sha256 @@ -7102,6 +7283,62 @@ class Rak3401KnownUnsafeReleaseTests(unittest.TestCase): ): rak_chain.download_release_asset(destination) + def test_primary_release_and_endpoint_are_pinned(self) -> None: + self.assertEqual( + rak_chain.RELEASE_URL, + "https://github.com/mikecarper/MeshCore/releases/tag/" + "rak3401-mota-v1.16.07-c1caa5ad-to-v1.17.1.6-3f6eddd5", + ) + self.assertEqual( + rak_chain.ASSET_NAME, + "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.6-3f6eddd5.zip", + ) + self.assertEqual( + rak_chain.BUNDLE_ROOT_NAME, + "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.6-3f6eddd5", + ) + self.assertEqual( + rak_chain.ASSET_SHA256, + "135783dd8777490db2422a7543f7d80e3cf03f621d5ac362da930abbc1850dc4", + ) + self.assertEqual( + rak_chain.CHECKSUM_LIST_SHA256, + "93835ed497c579be0a2322292e7f15ab095e7c8ab7892833fe7df3130dbdaba4", + ) + self.assertEqual(rak_chain.EXPECTED_FINAL_VERSION, "1.17.1.6") + self.assertEqual( + rak_chain.CURRENT_10_CANDIDATE_ANCHORS[-1], + ( + 10, + "cd1e9b819f9e918f9f2ceaf37b8a74bf242e2be78f319af2cf02a42d7729d0f4", + ), + ) + self.assertEqual( + rak_chain.PINNED_ARCHIVE_CHECKSUMS[rak_chain.ASSET_SHA256], + rak_chain.CHECKSUM_LIST_SHA256, + ) + + def test_superseded_b40_archive_and_inner_checksum_remain_pinned(self) -> None: + self.assertEqual( + rak_chain.SUPERSEDED_B40_ASSET_SHA256, + "8e7f0c2565f37d4372fa51dc3ba72b591e7f38b74be09303e48641623f99bce4", + ) + self.assertEqual( + rak_chain.SUPERSEDED_B40_CHECKSUM_LIST_SHA256, + "29a02a6c4c83eafa80168a80562b0ded5d8432e6cc02cfaca289cb2ed426681a", + ) + self.assertEqual( + rak_chain.SUPERSEDED_B40_ROOT_NAME, + "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-b40d2e6c", + ) + self.assertEqual(rak_chain.SUPERSEDED_B40_FINAL_VERSION, "1.17.1.5") + self.assertEqual( + rak_chain.PINNED_ARCHIVE_CHECKSUMS[ + rak_chain.SUPERSEDED_B40_ASSET_SHA256 + ], + rak_chain.SUPERSEDED_B40_CHECKSUM_LIST_SHA256, + ) + def test_fd98_archive_and_inner_checksum_remain_pinned(self) -> None: self.assertEqual( rak_chain.PINNED_ARCHIVE_CHECKSUMS[ @@ -7134,6 +7371,30 @@ class Rak3401KnownUnsafeReleaseTests(unittest.TestCase): rak_chain.PHYSICALLY_PASSED_FD98_ASSET_SHA256, ) + def test_locate_bundle_accepts_exact_superseded_b40_archive(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archive = root / rak_chain.SUPERSEDED_B40_ASSET_NAME + archive.write_bytes(b"fixture is identified by the mocked digest") + extracted = root / rak_chain.SUPERSEDED_B40_ROOT_NAME + args = argparse.Namespace(bundle=archive) + with ( + mock.patch.object( + rak_chain, + "sha256_file_limited", + return_value=rak_chain.SUPERSEDED_B40_ASSET_SHA256, + ), + mock.patch.object( + rak_chain, "extract_bundle", return_value=extracted + ) as extract_bundle, + ): + self.assertEqual(rak_chain.locate_bundle(args, root / "work"), extracted) + extract_bundle.assert_called_once_with( + archive.resolve(), + root / "work" / "bundle", + rak_chain.SUPERSEDED_B40_ASSET_SHA256, + ) + def test_compact_9_step_release_rejects_changed_anchor(self) -> None: steps = [mock.Mock(target_sha256="") for _ in range(9)] for number, image_sha256 in rak_chain.COMPACT_RELEASE_ANCHORS: diff --git a/tools/lora_ota/test_rak3401_route_search.py b/tools/lora_ota/test_rak3401_route_search.py index 85a3d13b..d33c95ec 100644 --- a/tools/lora_ota/test_rak3401_route_search.py +++ b/tools/lora_ota/test_rak3401_route_search.py @@ -33,6 +33,7 @@ class RouteSearchTests(unittest.TestCase): self, root: Path, payloads: list[bytes], *, baseline_size: int = 100, versions: list[str] | None = None, transport_capabilities: list[bool] | None = None, + transport_block_sizes: list[int] | None = None, transport_pipelines: list[int | None] | None = None, version_ranks: list[int] | None = None, ) -> tuple[Path, list[dict[str, object]]]: @@ -60,6 +61,14 @@ class RouteSearchTests(unittest.TestCase): if transport_capabilities is not None else False ) record[search.TRANSPORT_CAPABILITY] = capability + record[search.TRANSPORT_MAX_BLOCK_FIELD] = ( + transport_block_sizes[node] + if transport_block_sizes is not None + else ( + search.DEFLATE_BLOCK_SIZE + if capability else search.LEGACY_BLOCK_SIZE + ) + ) record[search.TRANSPORT_PIPELINE_FIELD] = ( transport_pipelines[node] if transport_pipelines is not None else None @@ -69,6 +78,10 @@ class RouteSearchTests(unittest.TestCase): manifest.write_text(json.dumps({ "capability_contract": { search.TRANSPORT_CAPABILITY: "required boolean on every record", + search.TRANSPORT_MAX_BLOCK_FIELD: ( + "required signed-container block size on every record; " + "independent of ota_transport_deflate" + ), }, "edge_policy": { "require_target_version_rank_gt_source_version_rank": True, @@ -120,6 +133,49 @@ class RouteSearchTests(unittest.TestCase): ) self.assertEqual((projected[0]["source"], projected[0]["target"]), (2, 3)) + def test_cache_migration_reuses_payload_but_recomputes_2k_geometry(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest, records = self.write_inventory( + root, + [b"start", b"capable", b"endpoint"], + transport_capabilities=[False, True, False], + transport_pipelines=[None, 1, None], + ) + payload = 3000 + old_total = search.container_size(payload, search.LEGACY_BLOCK_SIZE) + old_stage = search.align_down(search.STAGE_CEILING - old_total) + cache = root / "old-1k.csv" + with cache.open("w", newline="", encoding="ascii") as output: + writer = csv.DictWriter(output, fieldnames=search.FIELDS) + writer.writeheader() + writer.writerow({ + "source": 1, + "target": 2, + "source_sha256": records[1]["sha256"], + "target_sha256": records[2]["sha256"], + "memory": search.FIXED_MEMORY, + "payload": payload, + # Blank means a pre-per-step cache with 1 KiB geometry. + "block_size": "", + "container": old_total, + "stage_start": old_stage, + "margin": old_stage - (search.APP_BASE + search.FIXED_MEMORY), + "feasible": True, + "error": "", + }) + + migrated = search.migrate_csv(manifest, cache) + row = next(iter(migrated.values())) + + self.assertEqual(row["payload"], payload) + self.assertEqual(row["block_size"], search.DEFLATE_BLOCK_SIZE) + self.assertEqual( + row["container"], + search.container_size(payload, search.DEFLATE_BLOCK_SIZE), + ) + self.assertFalse(search.transport_stats_complete(row)) + def test_all_jobs_covers_every_valid_page_workspace(self) -> None: with tempfile.TemporaryDirectory() as directory: manifest, _ = self.write_inventory( @@ -216,7 +272,7 @@ class RouteSearchTests(unittest.TestCase): 1, 2, search.FIXED_MEMORY, str(Path(directory) / "from.bin"), str(Path(directory) / "to.bin"), - "1" * 64, "2" * 64, directory, + "1" * 64, "2" * 64, search.LEGACY_BLOCK_SIZE, directory, )) def test_snapshot_inventory_is_immune_to_later_source_mutation(self) -> None: @@ -272,7 +328,7 @@ class RouteSearchTests(unittest.TestCase): 1, 2, search.FIXED_MEMORY, str(inventory_root / str(records[1]["path"])), str(inventory_root / str(records[2]["path"])), - source_sha, target_sha, + source_sha, target_sha, search.LEGACY_BLOCK_SIZE, ) def measured(args: tuple[object, ...]) -> dict[str, object]: @@ -285,6 +341,7 @@ class RouteSearchTests(unittest.TestCase): "source_sha256": source_sha, "target_sha256": target_sha, "memory": memory, "payload": payload, + "block_size": search.LEGACY_BLOCK_SIZE, "container": total, "stage_start": stage, "margin": stage - (search.APP_BASE + int(memory)), "feasible": True, "error": "", @@ -400,8 +457,14 @@ class RouteSearchTests(unittest.TestCase): # The receive-inflate compatibility bridges are compiled with a single # block in flight. Account for their five individual request frames, # rather than the full RAK3401 receiver's clean [1, 2, 2] flights. - full = search.v2_transport_cost(5 * 1024, 5 * 1024, 0, 0, 30, 4) - bridge = search.v2_transport_cost(5 * 1024, 5 * 1024, 0, 0, 30, 1) + full = search.v2_transport_cost( + 5 * 1024, 5 * 1024, 0, 0, 30, 4, + search.LEGACY_BLOCK_SIZE, + ) + bridge = search.v2_transport_cost( + 5 * 1024, 5 * 1024, 0, 0, 30, 1, + search.LEGACY_BLOCK_SIZE, + ) self.assertEqual(full["request_packets"], 3) self.assertEqual(full["request_bytes"], 41) self.assertEqual(bridge["request_packets"], 5) @@ -410,6 +473,27 @@ class RouteSearchTests(unittest.TestCase): int(bridge["origin_mesh_bytes"]) - int(full["origin_mesh_bytes"]), 14 ) + def test_deflate_geometry_uses_2k_blocks_and_raw_baseline_uses_1k(self) -> None: + payload = 4096 + self.assertEqual( + search.container_size(payload, search.LEGACY_BLOCK_SIZE), + payload + 210 + 4 * 4, + ) + self.assertEqual( + search.container_size(payload, search.DEFLATE_BLOCK_SIZE), + payload + 210 + 4 * 2, + ) + selected = search.v2_transport_cost( + payload, 2000, 2000, 2, 12, 1 + ) + baseline = search.v2_raw_transport_cost(payload, 1) + self.assertEqual(selected["block_size"], search.DEFLATE_BLOCK_SIZE) + self.assertEqual(selected["proof_packets"], 2) + self.assertEqual(baseline["profile"], "v2-171-raw") + self.assertEqual(baseline["block_size"], search.LEGACY_BLOCK_SIZE) + self.assertEqual(baseline["proof_packets"], 4) + self.assertGreater(baseline["packets"], selected["packets"]) + def test_published_legacy_chain_and_new_bootstrap_cost_regressions(self) -> None: old_payloads = [ 89282, 45974, 105407, 149133, 110666, @@ -429,6 +513,59 @@ class RouteSearchTests(unittest.TestCase): with self.assertRaisesRegex(search.RouteSearchError, "between 0 and 8"): search.linear_path_bytes(100, 1, 9) + def test_current_route_2k_and_1k_raw_projection_regressions(self) -> None: + # Exact current-route detools payloads and pinned motatool 2 KiB + # measurements. Steps 1, 2, and 10 remain legacy; steps 3-9 negotiate + # v2 and use a one-block request pipeline. + legacy_payloads = [89_282, 64_756, 196_843] + v2_measurements = [ + (104_543, 60_379, 60_284, 51, 381), + (148_997, 92_549, 92_549, 73, 577), + (112_123, 59_704, 59_704, 55, 375), + (121_934, 63_977, 63_977, 60, 405), + (135_068, 87_425, 87_425, 66, 547), + (176_140, 130_264, 130_252, 86, 807), + (271_013, 210_192, 210_192, 133, 1297), + ] + selected = [ + search.legacy_transport_cost(payload) for payload in legacy_payloads + ] + [ + search.v2_transport_cost(*measurement, 1, search.DEFLATE_BLOCK_SIZE) + for measurement in v2_measurements + ] + baseline = [ + search.legacy_transport_cost(payload) for payload in legacy_payloads + ] + [ + search.v2_raw_transport_cost( + measurement[0], 1, search.LEGACY_BLOCK_SIZE + ) + for measurement in v2_measurements + ] + + self.assertEqual( + ( + sum(int(cost["origin_mesh_bytes"]) for cost in selected), + sum(int(cost["packets"]) for cost in selected), + ), + (1_195_442, 8_906), + ) + self.assertEqual( + ( + sum(int(cost["origin_mesh_bytes"]) for cost in baseline), + sum(int(cost["packets"]) for cost in baseline), + ), + (1_618_318, 11_835), + ) + self.assertEqual( + ( + 1_618_318 - 1_195_442, + 11_835 - 8_906, + ), + (422_876, 2_929), + ) + self.assertEqual(round(100 * 422_876 / 1_618_318, 2), 26.13) + self.assertEqual(round(100 * 2_929 / 11_835, 2), 24.75) + def test_proof_sizing_matches_promote_odd_merkle_implementation(self) -> None: for count in range(1, 33): leaves = [bytes([index]) * 4 for index in range(count)] @@ -491,6 +628,22 @@ class RouteSearchTests(unittest.TestCase): self.assertEqual( transport["steps"][2]["transport"]["request_pipeline"], 1 ) + self.assertEqual( + [step["block_size"] for step in transport["steps"]], + [ + search.LEGACY_BLOCK_SIZE, + search.LEGACY_BLOCK_SIZE, + search.DEFLATE_BLOCK_SIZE, + ], + ) + self.assertEqual( + transport["transport_accounting"]["source_max_block_field"], + search.TRANSPORT_MAX_BLOCK_FIELD, + ) + self.assertEqual( + transport["transport_accounting"]["source_profile_matrix"], + search.TRANSPORT_PROFILE_MATRIX, + ) self.assertEqual( transport["selected_total_bytes"], sum(step["expected_container_size"] for step in transport["steps"]), @@ -509,6 +662,162 @@ class RouteSearchTests(unittest.TestCase): ) self.assertEqual(container["nodes"], [0, 1, 3, 4]) + def test_block_size_and_deflate_capabilities_form_independent_matrix( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest, _ = self.write_inventory( + root, [b"bootstrap", b"legacy", b"deflate-1k", b"raw-2k", b"deflate-2k"], + transport_capabilities=[False, False, True, False, True], + transport_block_sizes=[ + search.LEGACY_BLOCK_SIZE, + search.LEGACY_BLOCK_SIZE, + search.LEGACY_BLOCK_SIZE, + search.DEFLATE_BLOCK_SIZE, + search.DEFLATE_BLOCK_SIZE, + ], + transport_pipelines=[None, None, 1, 2, 4], + ) + images = search.load_inventory(manifest) + + self.assertEqual( + [search.source_block_size(images, node) for node in range(1, 5)], + [1024, 1024, 2048, 2048], + ) + self.assertEqual( + [ + search.source_transport_profile(images, node) + for node in range(1, 5) + ], + [ + "legacy-160-raw", + "v2-171-deflate", + "v2-171-raw", + "v2-171-deflate", + ], + ) + jobs = [ + ( + node, 9, search.FIXED_MEMORY, "from", "to", + "1" * 64, "2" * 64, + search.source_block_size(images, node), + ) + for node in range(1, 5) + ] + self.assertEqual( + [ + search.job_needs_transport_stats(job, images, "transport") + for job in jobs + ], + [False, True, False, True], + ) + + def edge(source: int, include_measurement: bool) -> dict[str, object]: + block_size = search.source_block_size(images, source) + payload = 3000 + row: dict[str, object] = { + "source": source, + "target": 4, + "memory": 0x1000, + "payload": payload, + "block_size": block_size, + "container": search.container_size(payload, block_size), + "stage_start": 0xC0000, + "margin": 100, + "feasible": True, + } + if include_measurement: + row.update({ + "payload_sha256": "a" * 64, + "transport_encoder_sha256": "b" * 64, + "v2_wire_bytes": 1500, + "v2_deflate_bytes": 1500, + "v2_deflate_blocks": ( + payload + block_size - 1 + ) // block_size, + "v2_data_packets": 9, + }) + return row + + self.assertEqual( + search.edge_transport_cost(edge(1, False), images, 0)["profile"], + "legacy-160-raw", + ) + with self.assertRaisesRegex(search.RouteSearchError, "missing transport"): + search.edge_transport_cost(edge(2, False), images, 0) + self.assertEqual( + search.edge_transport_cost(edge(2, True), images, 0)["profile"], + "v2-171-deflate", + ) + # Raw 2 KiB v2 accounting is derived and needs no encoder evidence. + raw_2k = search.edge_transport_cost(edge(3, False), images, 0) + self.assertEqual(raw_2k["profile"], "v2-171-raw") + self.assertEqual(raw_2k["block_size"], search.DEFLATE_BLOCK_SIZE) + self.assertEqual( + search.edge_transport_cost(edge(4, True), images, 0)["profile"], + "v2-171-deflate", + ) + + def test_raw_2k_route_requires_no_transport_encoder_evidence(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + baseline = search.container_size(64) + manifest, _ = self.write_inventory( + root, [b"bootstrap", b"raw-2k", b"endpoint"], + baseline_size=baseline, + transport_capabilities=[False, False, False], + transport_block_sizes=[1024, 2048, 1024], + transport_pipelines=[None, 2, None], + ) + images = search.load_inventory(manifest) + payload = 3000 + row = { + "source": 1, + "target": 2, + "memory": search.FIXED_MEMORY, + "payload": payload, + "block_size": 2048, + "container": search.container_size(payload, 2048), + "stage_start": 0xC0000, + "margin": 100, + "feasible": True, + } + + result = search.select_route( + [row], images, baseline, root / "route.json", True, + objective="transport", + ) + + self.assertEqual( + result["steps"][1]["transport"]["profile"], "v2-171-raw" + ) + self.assertNotIn("payload_sha256", result["steps"][1]["transport"]) + self.assertNotIn("encoder_sha256", result["steps"][1]["transport"]) + self.assertNotIn("encoder_sha256", result["transport_accounting"]) + + def test_transport_image_without_explicit_block_capability_fails_closed( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest, _ = self.write_inventory( + root, [b"start", b"capable", b"endpoint"], + transport_capabilities=[False, True, False], + transport_pipelines=[None, 1, None], + ) + document = json.loads(manifest.read_text(encoding="ascii")) + del document["capability_contract"][search.TRANSPORT_MAX_BLOCK_FIELD] + for image in document["images"]: + del image[search.TRANSPORT_MAX_BLOCK_FIELD] + manifest.write_text(json.dumps(document), encoding="ascii") + + with self.assertRaisesRegex( + search.RouteSearchError, + "transport_max_block_bytes is required by the inventory", + ): + search.load_inventory(manifest) + def test_transport_stats_are_required_only_on_the_shortest_hop_dag(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -609,7 +918,7 @@ class RouteSearchTests(unittest.TestCase): tool = root / "motatool" tool.write_bytes(b"exact encoder binary") measured = { - "schema": 1, "payload_bytes": 300, "block_size": 1024, + "schema": 1, "payload_bytes": 300, "block_size": 2048, "block_count": 1, "wire_bytes": 200, "deflate_bytes": 200, "deflate_blocks": 1, "data_packets": 2, } @@ -720,16 +1029,33 @@ class RouteSearchTests(unittest.TestCase): del bad["images"][1][search.TRANSPORT_CAPABILITY] cases.append((bad, "required by the inventory capability contract")) bad = json.loads(manifest.read_text()) + del bad["images"][1][search.TRANSPORT_MAX_BLOCK_FIELD] + cases.append((bad, "required by the inventory capability contract")) + bad = json.loads(manifest.read_text()) + bad["images"][1][search.TRANSPORT_MAX_BLOCK_FIELD] = 4096 + cases.append((bad, "must be 1024 or 2048")) + bad = json.loads(manifest.read_text()) bad["images"][1][search.TRANSPORT_CAPABILITY_ALIAS] = True cases.append((bad, "disagrees")) bad = json.loads(manifest.read_text()) bad["images"][1][search.TRANSPORT_CAPABILITY] = True + bad["images"][1][search.TRANSPORT_MAX_BLOCK_FIELD] = ( + search.DEFLATE_BLOCK_SIZE + ) cases.append((bad, "is required")) bad = json.loads(manifest.read_text()) bad["images"][1][search.TRANSPORT_CAPABILITY] = True + bad["images"][1][search.TRANSPORT_MAX_BLOCK_FIELD] = ( + search.DEFLATE_BLOCK_SIZE + ) bad["images"][1][search.TRANSPORT_PIPELINE_FIELD] = 0 cases.append((bad, "must be between")) bad = json.loads(manifest.read_text()) + bad["images"][1][search.TRANSPORT_MAX_BLOCK_FIELD] = ( + search.DEFLATE_BLOCK_SIZE + ) + cases.append((bad, "required for v2 transport")) + bad = json.loads(manifest.read_text()) del bad["images"][1]["version_rank"] cases.append((bad, "required by the inventory edge policy")) bad = json.loads(manifest.read_text()) diff --git a/tools/mota/README.md b/tools/mota/README.md index bbcf7695..d04c8adf 100644 --- a/tools/mota/README.md +++ b/tools/mota/README.md @@ -58,6 +58,8 @@ family derived from the environment name, and `fw_version` = parsed from `FIRMWA ## Privileged nRF52 bootloader reference builder Ordinary calls to `build_manifest()` continue to emit v2 application packages. +New application packages default to 2048-byte blocks; pass 1024 explicitly when +targeting a deployed receiver that predates the extended transport descriptor. Passing `bootloader=True` is a deliberately narrow reference-only path: it requires a nonzero version, a signed exact 40 KiB OTAFIX region, 1024-byte blocks, full codec, zero base hash, a sane vector table, an exact embedded diff --git a/tools/mota/gen_vectors.py b/tools/mota/gen_vectors.py index 70227e56..f563640b 100644 --- a/tools/mota/gen_vectors.py +++ b/tools/mota/gen_vectors.py @@ -11,6 +11,7 @@ Run: ./meshcore/bin/python tools/mota/gen_vectors.py from __future__ import annotations import random +import zlib from pathlib import Path import motalib as ml @@ -23,6 +24,11 @@ def _carr(name, data: bytes) -> str: return f"static const uint8_t {name}[{len(data)}] = {{{body}}};\n" +def _u16arr(name: str, values: list[int]) -> str: + body = ",".join(str(value) for value in values) + return f"static const uint16_t {name}[{len(values)}] = {{{body}}};\n" + + def build_full(): random.seed(42) fw = bytes(random.getrandbits(8) for _ in range(5 * 1024 + 137)) # 6 blocks, last short @@ -132,6 +138,38 @@ def main(): lines.append(f"static const uint32_t SIM_MOTA_1K_LEN = {len(sim1k_blob)};") lines.append(f"static const uint32_t SIM_MOTA_1K_BLOCKS = {sm1k.block_count};") + # 2 KiB-block signed .mota: one full block plus a short final block. The payload is deliberately + # compressible so its independently generated raw-RFC1951 streams exercise both extended-length + # DEFLATE descriptors, including the rule that a short tail still uses the manifest's 2 KiB profile. + sim2k_fw_len = 2 * 1024 + 137 + sim2k_phrase = b"MeshCore 2 KiB transport DEFLATE vector. " + sim2k_fw = (sim2k_phrase * ((sim2k_fw_len + len(sim2k_phrase) - 1) // len(sim2k_phrase)))[:sim2k_fw_len] + sim2k_image, _ = ml.ensure_endf(sim2k_fw) + sm2k = ml.build_manifest(target_id=0xCAFEBABE, fw_version=ml.pack_version("3.1.0"), + image_size=len(sim2k_image), payload=sim2k_image, block_size=2048, + image_hash=ml.mh32(sim2k_image), codec_id=ml.CODEC_FULL, + is_full=True, sign_priv=sim_priv) + sim2k_blob = ml.build_container(sm2k, sim2k_image) + compressed = bytearray() + compressed_offsets = [] + compressed_lengths = [] + for offset in range(0, len(sim2k_image), 2048): + block = sim2k_image[offset:offset + 2048] + encoder = zlib.compressobj(level=9, wbits=-15) + encoded = encoder.compress(block) + encoder.flush() + assert 0 < len(encoded) < len(block), (len(block), len(encoded)) + compressed_offsets.append(len(compressed)) + compressed_lengths.append(len(encoded)) + compressed.extend(encoded) + assert len(compressed_offsets) == sm2k.block_count + lines.append("// 2 KiB-block signed .mota: one full block plus a short tail") + lines.append(_carr("SIM_MOTA_2K", sim2k_blob)) + lines.append(f"static const uint32_t SIM_MOTA_2K_LEN = {len(sim2k_blob)};") + lines.append(f"static const uint32_t SIM_MOTA_2K_BLOCKS = {sm2k.block_count};") + lines.append(_carr("SIM_MOTA_2K_DEFLATED", bytes(compressed))) + lines.append(_u16arr("SIM_MOTA_2K_DEFLATED_OFFSETS", compressed_offsets)) + lines.append(_u16arr("SIM_MOTA_2K_DEFLATED_LENGTHS", compressed_lengths)) + # detools sequential+crle delta vector: base image, the real detools 0.53.0 patch, and the # expected target image. The native test applies DT_PATCH to DT_BASE with the *vendored detools # C decoder* (src/helpers/ota/detools) and must reproduce DT_TARGET byte-for-byte -- proving the diff --git a/tools/mota/motalib.py b/tools/mota/motalib.py index 837cab70..e7a65673 100644 --- a/tools/mota/motalib.py +++ b/tools/mota/motalib.py @@ -56,7 +56,9 @@ CODEC_NAMES = {CODEC_FULL: "full", CODEC_DETOOLS_SEQUENTIAL: "detools-sequential APPROVAL_NOT = b"\xff\xff\xff\xff" # erased = not approved APPROVAL_YES = b"APRV" # 41 50 52 56 = approved -DEFAULT_BLOCK_SIZE = 1024 +DEFAULT_BLOCK_SIZE = 2048 +MAX_APP_BLOCK_SIZE = 2048 +BOOTLOADER_BLOCK_SIZE = 1024 # Conservative fallback for firmware built before the layout record below existed. Keep in sync with # OtaFlashLayout_nrf52.h and motatool's format.rs. @@ -503,7 +505,7 @@ class Manifest: fw_version: int = 0 image_size: int = 0 payload_size: int = 0 - block_size_log2: int = 10 + block_size_log2: int = 11 merkle_root: bytes = b"\0\0\0\0" image_hash: bytes = b"\0" * 32 codec_id: int = CODEC_FULL @@ -859,6 +861,9 @@ def build_manifest(*, target_id: int, fw_version: int, image_size: int, payload: base_hash: Optional[bytes] = None, sign_priv=None, hw_id=None, bootloader: bool = False) -> Manifest: assert (block_size & (block_size - 1)) == 0, "block_size must be a power of two" + if not 1 < block_size <= MAX_APP_BLOCK_SIZE: + raise ValueError( + f"application block size must be at most {MAX_APP_BLOCK_SIZE} bytes") leaves = leaf_hashes(payload, block_size) if bootloader: identity = validate_bootloader_image(payload, target_id) @@ -868,7 +873,7 @@ def build_manifest(*, target_id: int, fw_version: int, image_size: int, payload: if (not bootloader_version_valid(fw_version) or fw_version != identity.boot_version or not is_full or codec_id != CODEC_FULL or image_size != XIAO_BOOT_IMAGE_SIZE or - block_size != DEFAULT_BLOCK_SIZE or base_hash not in (None, b"\0" * 8)): + block_size != BOOTLOADER_BLOCK_SIZE or base_hash not in (None, b"\0" * 8)): raise ValueError("bootloader package version must equal BLM2 metadata and use a 40 KiB CODEC_FULL image with 1 KiB blocks") if sign_priv is None: raise ValueError("bootloader package must be signed") @@ -950,6 +955,12 @@ def parse_container(blob: bytes) -> Parsed: m.hash_algo = take(1)[0] m.target_id, m.fw_version, m.image_size, m.payload_size = struct.unpack("